diff --git a/.gitignore b/.gitignore index 75f5c4633874..990195b9d6a9 100644 --- a/.gitignore +++ b/.gitignore @@ -35,4 +35,9 @@ data/ .test token_estimator_test.go skills-lock.json +generated-videos/ +docs/*.local.md +tmp/ +test-*.mp4 +web/**/pnpm-lock.yaml .playwright-mcp diff --git a/common/api_type.go b/common/api_type.go index 39c1fe9a5406..8f4414e6be52 100644 --- a/common/api_type.go +++ b/common/api_type.go @@ -75,6 +75,8 @@ func ChannelType2APIType(channelType int) (int, bool) { apiType = constant.APITypeReplicate case constant.ChannelTypeCodex: apiType = constant.APITypeCodex + case constant.ChannelTypeListenHub: + apiType = constant.APITypeListenHub } if apiType == -1 { return constant.APITypeOpenAI, false diff --git a/common/endpoint_type.go b/common/endpoint_type.go index a5e2ff8412e8..d945b83c847c 100644 --- a/common/endpoint_type.go +++ b/common/endpoint_type.go @@ -28,8 +28,10 @@ func GetEndpointTypesByChannelType(channelType int, modelName string) []constant endpointTypes = []constant.EndpointType{constant.EndpointTypeOpenAI} case constant.ChannelTypeXai: endpointTypes = []constant.EndpointType{constant.EndpointTypeOpenAI, constant.EndpointTypeOpenAIResponse} - case constant.ChannelTypeSora: + case constant.ChannelTypeSora, constant.ChannelTypeOpenAIVideo: endpointTypes = []constant.EndpointType{constant.EndpointTypeOpenAIVideo} + case constant.ChannelTypeListenHub: + endpointTypes = []constant.EndpointType{constant.EndpointTypeImageGeneration} default: if IsOpenAIResponseOnlyModel(modelName) { endpointTypes = []constant.EndpointType{constant.EndpointTypeOpenAIResponse} diff --git a/common/model.go b/common/model.go index 4ebc7b532d74..7bd6c9843b2f 100644 --- a/common/model.go +++ b/common/model.go @@ -10,10 +10,26 @@ var ( "o4-mini-deep-research", } ImageGenerationModels = []string{ + "dall-e", "dall-e-3", "dall-e-2", "gpt-image-1", + "gpt-image-2", + "gpt-image-2(线路xf)", + "gr-image-2", "prefix:imagen-", + "gemini-2.5-flash-image", + "gemini-2.5-flash-image-preview", + "gemini-3-pro-image-preview", + "gemini-3.1-flash-image-preview", + "gemini_3.0_pro_image_preview", + "gemini_3.1_flash_image_preview", + "nano-banana", + "nano-banana-hd", + "nano-banana-pro", + "baidu/ernie-image-turbo", + "qwen/qwen-image", + "tongyi-mai/z-image", "flux-", "flux.1-", } diff --git a/common/model_test.go b/common/model_test.go new file mode 100644 index 000000000000..8b518858f567 --- /dev/null +++ b/common/model_test.go @@ -0,0 +1,41 @@ +package common + +import ( + "testing" + + "github.com/QuantumNous/new-api/constant" +) + +func TestIsImageGenerationModelIncludesConfiguredImageModels(t *testing.T) { + models := []string{ + "gpt-image-2(线路XF)", + "gr-image-2", + "gemini-2.5-flash-image", + "gemini-2.5-flash-image-preview", + "gemini-3-pro-image-preview", + "gemini-3.1-flash-image-preview", + "nano-banana", + "nano-banana-hd", + "nano-banana-pro", + } + + for _, model := range models { + t.Run(model, func(t *testing.T) { + if !IsImageGenerationModel(model) { + t.Fatalf("IsImageGenerationModel(%q) = false, want true", model) + } + }) + } +} + +func TestImageModelsEnableImageGenerationEndpoint(t *testing.T) { + endpoints := GetEndpointTypesByChannelType(constant.ChannelTypeOpenAIVideo, "gr-image-2") + if len(endpoints) == 0 || endpoints[0] != constant.EndpointTypeImageGeneration { + t.Fatalf("OpenAI video image model endpoints = %#v, want image generation first", endpoints) + } + + endpoints = GetEndpointTypesByChannelType(constant.ChannelTypeOpenAI, "nano-banana-pro") + if len(endpoints) == 0 || endpoints[0] != constant.EndpointTypeImageGeneration { + t.Fatalf("OpenAI image model endpoints = %#v, want image generation first", endpoints) + } +} diff --git a/constant/api_type.go b/constant/api_type.go index 536ebd2c7198..59e2f314335b 100644 --- a/constant/api_type.go +++ b/constant/api_type.go @@ -36,5 +36,6 @@ const ( APITypeMiniMax APITypeReplicate APITypeCodex + APITypeListenHub APITypeDummy // this one is only for count, do not add any channel after this ) diff --git a/constant/channel.go b/constant/channel.go index 48502bedc52c..8ad331bc6977 100644 --- a/constant/channel.go +++ b/constant/channel.go @@ -55,6 +55,8 @@ const ( ChannelTypeSora = 55 ChannelTypeReplicate = 56 ChannelTypeCodex = 57 + ChannelTypeOpenAIVideo = 58 + ChannelTypeListenHub = 59 ChannelTypeDummy // this one is only for count, do not add any channel after this ) @@ -115,9 +117,11 @@ var ChannelBaseURLs = []string{ "https://api.vidu.cn", //52 "https://llm.submodel.ai", //53 "https://ark.cn-beijing.volces.com", //54 - "https://api.openai.com", //55 - "https://api.replicate.com", //56 - "https://chatgpt.com", //57 + "https://api.openai.com", // 55 + "https://api.replicate.com", // 56 + "https://chatgpt.com", // 57 + "", // 58 + "https://api.marswave.ai/openapi", // 59 } var ChannelTypeNames = map[int]string{ @@ -175,6 +179,8 @@ var ChannelTypeNames = map[int]string{ ChannelTypeSora: "Sora", ChannelTypeReplicate: "Replicate", ChannelTypeCodex: "Codex", + ChannelTypeOpenAIVideo: "OpenAIVideo", + ChannelTypeListenHub: "ListenHub", } func GetChannelTypeName(channelType int) string { diff --git a/constant/context_key.go b/constant/context_key.go index c28ad202514b..a99a5d725b6a 100644 --- a/constant/context_key.go +++ b/constant/context_key.go @@ -26,6 +26,7 @@ const ( ContextKeyChannelCreateTime ContextKey = "channel_create_time" ContextKeyChannelBaseUrl ContextKey = "base_url" ContextKeyChannelType ContextKey = "channel_type" + ContextKeyChannelOther ContextKey = "channel_other" ContextKeyChannelSetting ContextKey = "channel_setting" ContextKeyChannelOtherSetting ContextKey = "channel_other_setting" ContextKeyChannelParamOverride ContextKey = "param_override" diff --git a/controller/channel-billing.go b/controller/channel-billing.go index 751ee3600ac9..725caae9a592 100644 --- a/controller/channel-billing.go +++ b/controller/channel-billing.go @@ -361,6 +361,24 @@ func updateChannelBalance(channel *model.Channel) (float64, error) { if channel.GetBaseURL() == "" { channel.BaseURL = &baseURL } + // 优先按 base_url/Other hint 走余额 provider（覆盖中转站，不依赖 channel.Type） + if result, handled, err := queryBalanceByProvider(channel); handled { + if err != nil { + return 0, err + } + switch result.Kind { + case BalanceKindConsoleOnly: + return 0, fmt.Errorf("该上游不支持 API 余额查询，仅 web 控制台可查") + case BalanceKindSpendOnly: + // 拿不到钱包余额，返回累计消费供展示；保持为正避免被余额<=0 自动禁用误伤 + if result.Used > 0 { + return result.Used, nil + } + return 1, nil + default: + return result.Remaining, nil + } + } switch channel.Type { case constant.ChannelTypeOpenAI: if channel.GetBaseURL() != "" { @@ -444,10 +462,16 @@ func UpdateChannelBalance(c *gin.Context) { common.ApiError(c, err) return } + info := channel.GetOtherInfo() c.JSON(http.StatusOK, gin.H{ - "success": true, - "message": "", - "balance": balance, + "success": true, + "message": "", + "balance": balance, + "kind": info["balance_kind"], + "unit": info["balance_unit"], + "used": info["balance_used"], + "remaining": info["balance_remaining"], + "provider": info["balance_provider"], }) } diff --git a/controller/channel.go b/controller/channel.go index c59e492a5a02..ee056c0276c6 100644 --- a/controller/channel.go +++ b/controller/channel.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "io" "net/http" "strconv" "strings" @@ -190,6 +191,13 @@ func buildFetchModelsHeaders(channel *model.Channel, key string) (http.Header, e switch channel.Type { case constant.ChannelTypeAnthropic: headers = GetClaudeAuthHeader(key) + case constant.ChannelTypeOpenAIVideo: + if isXBSoraModelsAPI(channel.GetBaseURL()) { + headers = http.Header{} + headers.Set("X-API-Key", key) + } else { + headers = GetAuthHeader(key) + } default: headers = GetAuthHeader(key) } @@ -212,6 +220,93 @@ func buildFetchModelsHeaders(channel *model.Channel, key string) (http.Header, e return headers, nil } +func isXBSoraModelsAPI(baseURL string) bool { + baseURL = strings.ToLower(strings.TrimRight(strings.TrimSpace(baseURL), "/")) + return strings.Contains(baseURL, "xb-sora2") || + strings.Contains(baseURL, "xbsora2") || + strings.Contains(baseURL, "xb-sora") || + strings.Contains(baseURL, "xbsora") || + strings.HasSuffix(baseURL, "/api/v1") || + strings.HasSuffix(baseURL, "/v1") +} + +func xbSoraModelsURL(baseURL string) string { + baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/") + if strings.HasSuffix(baseURL, "/api/v1") || strings.HasSuffix(baseURL, "/v1") { + return baseURL + "/models" + } + return baseURL + "/api/v1/models" +} + +func parseXBSoraModelIDs(body []byte) ([]string, error) { + var result map[string]any + if err := common.Unmarshal(body, &result); err != nil { + return nil, err + } + if !xbSoraFetchCodeOK(result["code"]) { + return nil, fmt.Errorf("xb-sora2 models fetch failed: %s", xbSoraFetchMessage(result)) + } + models := findXBSoraModelIDs(result, 0) + if len(models) == 0 { + return nil, fmt.Errorf("xb-sora2 models response contains no models") + } + return models, nil +} + +func findXBSoraModelIDs(value any, depth int) []string { + if depth > 4 { + return nil + } + switch v := value.(type) { + case map[string]any: + if models, ok := v["models"].([]any); ok { + ids := make([]string, 0, len(models)) + for _, item := range models { + model, ok := item.(map[string]any) + if !ok { + continue + } + id, _ := model["id"].(string) + if strings.TrimSpace(id) != "" { + ids = append(ids, id) + } + } + return ids + } + if data, ok := v["data"]; ok { + return findXBSoraModelIDs(data, depth+1) + } + } + return nil +} + +func xbSoraFetchCodeOK(code any) bool { + switch v := code.(type) { + case nil: + return true + case int: + return v == 0 || v == http.StatusOK + case int64: + return v == 0 || v == http.StatusOK + case float64: + return v == 0 || v == http.StatusOK + case string: + v = strings.TrimSpace(v) + return v == "" || v == "0" || v == "0000" || v == "200" + default: + return false + } +} + +func xbSoraFetchMessage(result map[string]any) string { + for _, key := range []string{"message", "msg", "error"} { + if message, ok := result[key].(string); ok && strings.TrimSpace(message) != "" { + return message + } + } + return "unknown error" +} + func FetchUpstreamModels(c *gin.Context) { id, err := strconv.Atoi(c.Param("id")) if err != nil { @@ -1056,6 +1151,10 @@ func FetchModels(c *gin.Context) { client := &http.Client{} url := fmt.Sprintf("%s/v1/models", baseURL) + useXBSoraModelsAPI := req.Type == constant.ChannelTypeOpenAIVideo && isXBSoraModelsAPI(baseURL) + if useXBSoraModelsAPI { + url = xbSoraModelsURL(baseURL) + } request, err := http.NewRequest("GET", url, nil) if err != nil { @@ -1066,7 +1165,11 @@ func FetchModels(c *gin.Context) { return } - request.Header.Set("Authorization", "Bearer "+key) + if useXBSoraModelsAPI { + request.Header.Set("X-API-Key", key) + } else { + request.Header.Set("Authorization", "Bearer "+key) + } response, err := client.Do(request) if err != nil { @@ -1086,6 +1189,30 @@ func FetchModels(c *gin.Context) { } defer response.Body.Close() + if useXBSoraModelsAPI { + body, err := io.ReadAll(response.Body) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{ + "success": false, + "message": err.Error(), + }) + return + } + models, err := parseXBSoraModelIDs(body) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{ + "success": false, + "message": err.Error(), + }) + return + } + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": models, + }) + return + } + var result struct { Data []struct { ID string `json:"id"` diff --git a/controller/channel_balance_provider.go b/controller/channel_balance_provider.go new file mode 100644 index 000000000000..c2040e79f9e2 --- /dev/null +++ b/controller/channel_balance_provider.go @@ -0,0 +1,580 @@ +package controller + +import ( + "bytes" + "fmt" + "net/http" + "strings" + "time" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/constant" + "github.com/QuantumNous/new-api/model" + "github.com/QuantumNous/new-api/service" + + "github.com/gin-gonic/gin" +) + +// 下游平台余额查询 provider —— 按 base_url / channel.Other hint 路由（不依赖 channel.Type）， +// 与视频 provider 模式（relay/channel/task/openaivideo）同构。 +// 调研结论与三档语义详见 docs/channel-balance-query.md。 + +// 余额可查性三档 +const ( + BalanceKindBalance = "balance" // 真实剩余余额 + BalanceKindSpendOnly = "spend_only" // 仅累计消费（无限额度 key 查不到钱包余额） + BalanceKindConsoleOnly = "console_only" // 仅 web 控制台可查 +) + +// new-api 无限额度令牌在 /v1/dashboard/billing/subscription 返回的哨兵额度 +const newapiUnlimitedSentinel = 100000000.0 + +// new-api 内部 quota → USD 换算单位（common.QuotaPerUnit 同值，避免 import 循环风险这里直接取常量） +const newapiQuotaPerUnit = 500000.0 + +// BalanceQueryResult 余额查询统一结果 +type BalanceQueryResult struct { + Kind string `json:"kind"` // balance | spend_only | console_only + Remaining float64 `json:"remaining"` // 剩余（Kind=balance 时有效） + Used float64 `json:"used"` // 累计消费（尽量填） + Unit string `json:"unit"` // USD | CNY | 算力 | credits + ExpiresAt int64 `json:"expires_at,omitempty"` // 订阅到期（listenhub 等） + Provider string `json:"provider"` // 命中的 provider 名 +} + +type balanceProvider struct { + name string + match func(ch *model.Channel) bool + query func(ch *model.Channel) (*BalanceQueryResult, error) +} + +var balanceProviders = []balanceProvider{ + {name: "lk888", match: matchLK888, query: queryLK888Balance}, + {name: "listenhub", match: matchListenHub, query: queryListenHubBalance}, + {name: "newapi", match: matchNewAPIStation, query: queryNewAPIStationBalance}, +} + +// resolveBalanceProvider 按渠道特征解析余额 provider；未命中返回 nil（由调用方退回 type switch）。 +func resolveBalanceProvider(ch *model.Channel) *balanceProvider { + if ch.GetSetting().BalanceQuery != nil && ch.GetSetting().BalanceQuery.Mode == "disabled" { + return nil + } + for i := range balanceProviders { + if balanceProviders[i].match(ch) { + return &balanceProviders[i] + } + } + return nil +} + +// queryBalanceByProvider 尝试用 provider 查询余额；handled=false 表示未命中任何 provider。 +func queryBalanceByProvider(ch *model.Channel) (result *BalanceQueryResult, handled bool, err error) { + p := resolveBalanceProvider(ch) + if p == nil { + return nil, false, nil + } + result, err = p.query(ch) + if err != nil { + return nil, true, err + } + result.Provider = p.name + storeBalanceResult(ch, result) + return result, true, nil +} + +// storeBalanceResult 把结果写回渠道：Balance 列存"可对外排序的数值"，明细存 OtherInfo。 +func storeBalanceResult(ch *model.Channel, r *BalanceQueryResult) { + display := r.Remaining + if r.Kind == BalanceKindSpendOnly { + // 拿不到钱包余额：若用户填了累计充值，估算剩余；否则用累计消费占位（保持为正，避免被余额<=0 自动禁用逻辑误伤） + if rc := rechargedOf(ch); rc > 0 { + display = rc - r.Used + } else { + display = r.Used + } + } + ch.UpdateBalance(display) + + info := ch.GetOtherInfo() + info["balance_kind"] = r.Kind + info["balance_unit"] = r.Unit + info["balance_used"] = r.Used + info["balance_remaining"] = r.Remaining + info["balance_provider"] = r.Provider + info["balance_expires_at"] = r.ExpiresAt + info["balance_checked_time"] = time.Now().Unix() + ch.SetOtherInfo(info) + _ = ch.SaveChannelInfo() +} + +func rechargedOf(ch *model.Channel) float64 { + if bq := ch.GetSetting().BalanceQuery; bq != nil { + return bq.Recharged + } + return 0 +} + +// ---------------- 通用 HTTP 工具 ---------------- + +// balanceHTTPGet 发 GET 请求，返回 body 与状态码（不像 GetResponseBody 那样把非 200 当错误， +// 便于上层按状态码判断"该上游是否支持此接口"）。 +func balanceHTTPGet(ch *model.Channel, url string, headers http.Header) ([]byte, int, error) { + return balanceHTTPDo(ch, http.MethodGet, url, headers, nil) +} + +func balanceHTTPDo(ch *model.Channel, method, url string, headers http.Header, body []byte) ([]byte, int, error) { + var reqBody *bytes.Reader + if body != nil { + reqBody = bytes.NewReader(body) + } else { + reqBody = bytes.NewReader(nil) + } + req, err := http.NewRequest(method, url, reqBody) + if err != nil { + return nil, 0, err + } + for k := range headers { + req.Header.Set(k, headers.Get(k)) + } + client, err := service.NewProxyHttpClient(ch.GetSetting().Proxy) + if err != nil { + return nil, 0, err + } + resp, err := client.Do(req) + if err != nil { + return nil, 0, err + } + defer resp.Body.Close() + respBody, err := readAllLimited(resp) + if err != nil { + return nil, resp.StatusCode, err + } + return respBody, resp.StatusCode, nil +} + +func readAllLimited(resp *http.Response) ([]byte, error) { + const max = 1 << 20 // 1MB 足够余额响应 + buf := make([]byte, 0, 4096) + tmp := make([]byte, 4096) + for { + n, err := resp.Body.Read(tmp) + if n > 0 { + buf = append(buf, tmp[:n]...) + if len(buf) > max { + break + } + } + if err != nil { + break + } + } + return buf, nil +} + +// apiBaseOf 去掉 base_url 末尾的 /v1 后缀，得到站点根（控制台与 billing 接口都挂在站点根下）。 +func apiBaseOf(base string) string { + b := strings.TrimRight(base, "/") + b = strings.TrimSuffix(b, "/v1") + return strings.TrimRight(b, "/") +} + +// ---------------- lk888（A 档：真实余额，单位算力） ---------------- + +func matchLK888(ch *model.Channel) bool { + other := strings.ToLower(strings.TrimSpace(ch.Other)) + if other == "lk888" { + return true + } + return strings.Contains(strings.ToLower(ch.GetBaseURL()), "lk888") +} + +func queryLK888Balance(ch *model.Channel) (*BalanceQueryResult, error) { + // lk888 的 base_url 已含 /api，余额接口为 {base}/v1/skills/balance + url := fmt.Sprintf("%s/v1/skills/balance", strings.TrimRight(ch.GetBaseURL(), "/")) + body, status, err := balanceHTTPGet(ch, url, GetAuthHeader(ch.Key)) + if err != nil { + return nil, err + } + if status != http.StatusOK { + return nil, fmt.Errorf("lk888 余额接口返回状态码 %d: %s", status, snippet(body)) + } + var resp struct { + Balance float64 `json:"balance"` + Unit string `json:"unit"` + APIKeyQuota struct { + Limit float64 `json:"limit"` + Used float64 `json:"used"` + } `json:"api_key_quota"` + } + if err := common.Unmarshal(body, &resp); err != nil { + return nil, fmt.Errorf("lk888 余额响应解析失败: %v (%s)", err, snippet(body)) + } + unit := resp.Unit + if unit == "" { + unit = "算力" + } + return &BalanceQueryResult{ + Kind: BalanceKindBalance, + Remaining: resp.Balance, + Used: resp.APIKeyQuota.Used, + Unit: unit, + }, nil +} + +// ---------------- listenhub / marswave（A 档：真实余额，单位 credits） ---------------- + +func matchListenHub(ch *model.Channel) bool { + if ch.Type == constant.ChannelTypeListenHub { + return true + } + other := strings.ToLower(strings.TrimSpace(ch.Other)) + if other == "listenhub" || other == "marswave" { + return true + } + return strings.Contains(strings.ToLower(ch.GetBaseURL()), "marswave.ai") +} + +func queryListenHubBalance(ch *model.Channel) (*BalanceQueryResult, error) { + // base_url 为 https://api.marswave.ai/openapi，余额接口为 {base}/v1/user/subscription + url := fmt.Sprintf("%s/v1/user/subscription", strings.TrimRight(ch.GetBaseURL(), "/")) + body, status, err := balanceHTTPGet(ch, url, GetAuthHeader(ch.Key)) + if err != nil { + return nil, err + } + if status != http.StatusOK { + return nil, fmt.Errorf("listenhub 余额接口返回状态码 %d: %s", status, snippet(body)) + } + var resp struct { + Code int `json:"code"` + Data struct { + TotalAvailableCredits float64 `json:"totalAvailableCredits"` + SubscriptionExpiresAt int64 `json:"subscriptionExpiresAt"` + } `json:"data"` + Message string `json:"message"` + } + if err := common.Unmarshal(body, &resp); err != nil { + return nil, fmt.Errorf("listenhub 余额响应解析失败: %v (%s)", err, snippet(body)) + } + if resp.Code != 0 { + return nil, fmt.Errorf("listenhub 余额接口返回 code=%d message=%s", resp.Code, resp.Message) + } + return &BalanceQueryResult{ + Kind: BalanceKindBalance, + Remaining: resp.Data.TotalAvailableCredits, + Unit: "credits", + ExpiresAt: resp.Data.SubscriptionExpiresAt / 1000, // 毫秒→秒 + }, nil +} + +// ---------------- new-api 套壳站（B 档 spend_only / 登录态 balance） ---------------- + +// matchNewAPIStation 匹配 new-api/one-api 套壳中转站： +// - 显式配了 BalanceQuery（console/system_token/auto）；或 +// - 渠道是中转常见类型（OpenAI / Custom / Gemini / OpenAI Video）且配了 base_url。 +// +// 这样不靠厂商域名硬编码即可覆盖 bltcy/apexer/xgapi/qilin/manxiaobai； +// 非 new-api 上游（如 hongniao）billing 接口会 404，query 优雅降级为 console_only。 +func matchNewAPIStation(ch *model.Channel) bool { + if bq := ch.GetSetting().BalanceQuery; bq != nil && bq.Mode != "" && bq.Mode != "auto" { + return true + } + if ch.GetBaseURL() == "" { + return false + } + switch ch.Type { + case constant.ChannelTypeOpenAI, constant.ChannelTypeCustom, + constant.ChannelTypeGemini, constant.ChannelTypeOpenAIVideo: + return true + } + return false +} + +func queryNewAPIStationBalance(ch *model.Channel) (*BalanceQueryResult, error) { + // 配了登录凭据 → 走登录态拿真实钱包余额 + if bq := ch.GetSetting().BalanceQuery; bq != nil { + switch bq.Mode { + case "newapi_console": + return queryNewAPIConsoleBalance(ch, bq.Username, bq.Password) + case "system_token": + return nil, fmt.Errorf("system_token 模式暂未实现，请使用 newapi_console（账密）") + } + } + // 否则仅凭 API key：只能拿累计消费 + return queryNewAPISpend(ch) +} + +// queryNewAPISpend 仅凭 API key 查 /v1/dashboard/billing/subscription + usage。 +// 无限额度令牌 → spend_only（只报累计消费）；正常额度 → balance（hardLimit - used）。 +func queryNewAPISpend(ch *model.Channel) (*BalanceQueryResult, error) { + apiBase := apiBaseOf(ch.GetBaseURL()) + subBody, status, err := balanceHTTPGet(ch, apiBase+"/v1/dashboard/billing/subscription", GetAuthHeader(ch.Key)) + if err != nil { + return nil, err + } + if status == http.StatusNotFound { + return &BalanceQueryResult{Kind: BalanceKindConsoleOnly, Unit: "USD"}, nil + } + if status != http.StatusOK { + return nil, fmt.Errorf("billing/subscription 返回状态码 %d: %s", status, snippet(subBody)) + } + var sub OpenAISubscriptionResponse + if err := common.Unmarshal(subBody, &sub); err != nil || sub.Object != "billing_subscription" { + // 不是 new-api billing 结构（如 hongniao 用 200 返回 {"code":"404"} 错误体或 SPA HTML）→ 仅控制台可查 + return &BalanceQueryResult{Kind: BalanceKindConsoleOnly, Unit: "USD"}, nil + } + + // 拉累计消费 + now := time.Now() + startDate := fmt.Sprintf("%s-01", now.Format("2006-01")) + endDate := now.Format("2006-01-02") + if !sub.HasPaymentMethod { + startDate = now.AddDate(0, 0, -100).Format("2006-01-02") + } + usageURL := fmt.Sprintf("%s/v1/dashboard/billing/usage?start_date=%s&end_date=%s", apiBase, startDate, endDate) + usageBody, _, err := balanceHTTPGet(ch, usageURL, GetAuthHeader(ch.Key)) + if err != nil { + return nil, err + } + var usage OpenAIUsageResponse + _ = common.Unmarshal(usageBody, &usage) + used := usage.TotalUsage / 100 // total_usage 单位 0.01 USD + + if sub.HardLimitUSD >= newapiUnlimitedSentinel { + // 无限额度令牌：钱包余额查不到，只能报累计消费 + return &BalanceQueryResult{Kind: BalanceKindSpendOnly, Used: used, Unit: "USD"}, nil + } + return &BalanceQueryResult{ + Kind: BalanceKindBalance, + Remaining: sub.HardLimitUSD - used, + Used: used, + Unit: "USD", + }, nil +} + +// queryNewAPIConsoleBalance 用账密登录 new-api 套壳站控制台，再查 /api/user/self 拿真实钱包余额。 +func queryNewAPIConsoleBalance(ch *model.Channel, username, password string) (*BalanceQueryResult, error) { + if username == "" || password == "" { + return nil, fmt.Errorf("newapi_console 模式需要配置 username 和 password") + } + apiBase := apiBaseOf(ch.GetBaseURL()) + + loginBody, _ := common.Marshal(map[string]string{"username": username, "password": password}) + headers := http.Header{} + headers.Set("Content-Type", "application/json") + respBody, status, err := balanceHTTPDo(ch, http.MethodPost, apiBase+"/api/user/login", headers, loginBody) + if err != nil { + return nil, err + } + if status != http.StatusOK { + return nil, fmt.Errorf("登录 %s 失败，状态码 %d: %s", apiBase, status, snippet(respBody)) + } + // 解析登录响应拿 user id（/api/user/self 需要 New-Api-User 头） + var login struct { + Success bool `json:"success"` + Message string `json:"message"` + Data struct { + Id int `json:"id"` + } `json:"data"` + } + if err := common.Unmarshal(respBody, &login); err != nil { + return nil, fmt.Errorf("登录响应解析失败: %v (%s)", err, snippet(respBody)) + } + if !login.Success { + return nil, fmt.Errorf("登录失败: %s", login.Message) + } + + // 需要把登录返回的 session cookie 带到 /api/user/self + // balanceHTTPDo 用的是共享 client（无 cookie jar），这里单独发一次带 cookie 的请求。 + cookie, err := loginSessionCookie(ch, apiBase, loginBody) + if err != nil { + return nil, err + } + selfHeaders := http.Header{} + selfHeaders.Set("New-Api-User", fmt.Sprintf("%d", login.Data.Id)) + if cookie != "" { + selfHeaders.Set("Cookie", cookie) + } + selfBody, selfStatus, err := balanceHTTPGet(ch, apiBase+"/api/user/self", selfHeaders) + if err != nil { + return nil, err + } + if selfStatus != http.StatusOK { + return nil, fmt.Errorf("/api/user/self 返回状态码 %d: %s", selfStatus, snippet(selfBody)) + } + var self struct { + Success bool `json:"success"` + Data struct { + Quota int64 `json:"quota"` + UsedQuota int64 `json:"used_quota"` + } `json:"data"` + Message string `json:"message"` + } + if err := common.Unmarshal(selfBody, &self); err != nil { + return nil, fmt.Errorf("/api/user/self 解析失败: %v (%s)", err, snippet(selfBody)) + } + if !self.Success { + return nil, fmt.Errorf("/api/user/self 失败: %s", self.Message) + } + return &BalanceQueryResult{ + Kind: BalanceKindBalance, + Remaining: float64(self.Data.Quota) / newapiQuotaPerUnit, + Used: float64(self.Data.UsedQuota) / newapiQuotaPerUnit, + Unit: "USD", + }, nil +} + +// loginSessionCookie 发一次登录请求并从 Set-Cookie 提取会话 cookie。 +func loginSessionCookie(ch *model.Channel, apiBase string, loginBody []byte) (string, error) { + req, err := http.NewRequest(http.MethodPost, apiBase+"/api/user/login", bytes.NewReader(loginBody)) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", "application/json") + client, err := service.NewProxyHttpClient(ch.GetSetting().Proxy) + if err != nil { + return "", err + } + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + var parts []string + for _, c := range resp.Cookies() { + parts = append(parts, c.Name+"="+c.Value) + } + return strings.Join(parts, "; "), nil +} + +// ---------------- 余额总览聚合接口 ---------------- + +// ChannelBalanceOverviewItem 一个上游账号（按 base_url+key 去重）的余额概览 +type ChannelBalanceOverviewItem struct { + BaseURL string `json:"base_url"` + ChannelIds []int `json:"channel_ids"` + ChannelNames []string `json:"channel_names"` + Provider string `json:"provider,omitempty"` + Kind string `json:"kind"` // balance | spend_only | console_only | unknown + Remaining float64 `json:"remaining"` + Used float64 `json:"used"` + Unit string `json:"unit"` + ExpiresAt int64 `json:"expires_at,omitempty"` + Recharged float64 `json:"recharged,omitempty"` + EstRemaining *float64 `json:"est_remaining,omitempty"` // spend_only 档：recharged - used + CheckedTime int64 `json:"checked_time"` + Error string `json:"error,omitempty"` +} + +// GetChannelBalanceOverview 一次性查出所有下游平台余额（按 base_url+key 去重，避免同账号多渠道重复查询）。 +// GET /api/channel/balance_overview +// +// ?cached=true 只读已存储的余额（不发起上游请求，秒回） +// ?include_disabled=true 连同已禁用渠道一起查（默认仅启用） +func GetChannelBalanceOverview(c *gin.Context) { + cached := c.Query("cached") == "true" + includeDisabled := c.Query("include_disabled") == "true" + + channels, err := model.GetAllChannels(0, 0, true, false) + if err != nil { + common.ApiError(c, err) + return + } + + type group struct { + rep *model.Channel + ids []int + names []string + } + groups := make(map[string]*group) + var order []string + for _, ch := range channels { + if ch.ChannelInfo.IsMultiKey { + continue // 多密钥渠道不支持余额查询 + } + if !includeDisabled && ch.Status != common.ChannelStatusEnabled { + continue + } + dedupKey := ch.GetBaseURL() + "\x00" + ch.Key + g, ok := groups[dedupKey] + if !ok { + g = &group{rep: ch} + groups[dedupKey] = g + order = append(order, dedupKey) + } + g.ids = append(g.ids, ch.Id) + g.names = append(g.names, ch.Name) + } + + items := make([]ChannelBalanceOverviewItem, 0, len(order)) + for _, k := range order { + g := groups[k] + item := ChannelBalanceOverviewItem{ + BaseURL: g.rep.GetBaseURL(), + ChannelIds: g.ids, + ChannelNames: g.names, + Recharged: rechargedOf(g.rep), + } + if !cached { + if _, qErr := updateChannelBalance(g.rep); qErr != nil { + item.Error = qErr.Error() + } + } + fillOverviewFromChannel(&item, g.rep) + items = append(items, item) + } + + c.JSON(http.StatusOK, gin.H{ + "success": true, + "message": "", + "data": items, + }) +} + +// fillOverviewFromChannel 从渠道存储的余额元数据（OtherInfo）填充概览项。 +// provider 路径会写入 balance_* 元数据；type switch 路径只更新 Balance 列，此处兜底为 balance/USD。 +func fillOverviewFromChannel(item *ChannelBalanceOverviewItem, ch *model.Channel) { + info := ch.GetOtherInfo() + kind, _ := info["balance_kind"].(string) + if kind == "" { + // 未经 provider（如 siliconflow/deepseek 等 type switch 渠道）：以 Balance 列为剩余余额 + item.Kind = BalanceKindBalance + item.Remaining = ch.Balance + item.Unit = "USD" + item.CheckedTime = ch.BalanceUpdatedTime + if item.Error == "" && ch.BalanceUpdatedTime == 0 { + item.Kind = "unknown" + } + return + } + item.Kind = kind + item.Provider, _ = info["balance_provider"].(string) + item.Unit, _ = info["balance_unit"].(string) + item.Remaining = otherInfoFloat(info, "balance_remaining") + item.Used = otherInfoFloat(info, "balance_used") + item.ExpiresAt = int64(otherInfoFloat(info, "balance_expires_at")) + item.CheckedTime = int64(otherInfoFloat(info, "balance_checked_time")) + if kind == BalanceKindSpendOnly && item.Recharged > 0 { + est := item.Recharged - item.Used + item.EstRemaining = &est + } +} + +func otherInfoFloat(info map[string]interface{}, key string) float64 { + switch v := info[key].(type) { + case float64: + return v + case int64: + return float64(v) + case int: + return float64(v) + } + return 0 +} + +func snippet(b []byte) string { + s := strings.TrimSpace(string(b)) + if len(s) > 160 { + return s[:160] + } + return s +} diff --git a/controller/channel_balance_provider_test.go b/controller/channel_balance_provider_test.go new file mode 100644 index 000000000000..868b0e7c71b9 --- /dev/null +++ b/controller/channel_balance_provider_test.go @@ -0,0 +1,114 @@ +package controller + +import ( + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/constant" + "github.com/QuantumNous/new-api/dto" + "github.com/QuantumNous/new-api/model" + "github.com/gin-gonic/gin" +) + +// 实网验证：直接调用各 provider 的 query 函数（不经 storeBalanceResult，避免依赖 DB）。 +// 默认跳过；用 LIVE_BALANCE_TEST=1 打开。 +// +// 凭据一律从环境变量读取，绝不硬编码进仓库（避免泄露生产 key/密码）。 +// 每个用例缺失对应 env 时单独跳过。示例： +// +// LIVE_BALANCE_TEST=1 \ +// LK888_BASE=https://api.lk888.ai/api LK888_KEY=sk-xxx \ +// LISTENHUB_BASE=https://api.marswave.ai/openapi LISTENHUB_KEY=lh_sk_xxx \ +// NEWAPI_SPEND_BASE=https://api.bltcy.ai NEWAPI_SPEND_KEY=sk-xxx \ +// CONSOLE_BASE=https://api.manxiaobai.online CONSOLE_USER=xxx CONSOLE_PASS=xxx \ +// go test ./controller/ -run 'TestLiveBalance|TestLiveConsole' -v +func ptr(s string) *string { return &s } + +func skipUnlessLive(t *testing.T) { + if os.Getenv("LIVE_BALANCE_TEST") != "1" { + t.Skip("set LIVE_BALANCE_TEST=1 to run live balance probes") + } +} + +func TestLiveBalanceProviders(t *testing.T) { + skipUnlessLive(t) + + cases := []struct { + name string + baseEnv string + keyEnv string + typ int + other string + fn func(*model.Channel) (*BalanceQueryResult, error) + }{ + {"lk888", "LK888_BASE", "LK888_KEY", constant.ChannelTypeOpenAIVideo, "lk888", queryLK888Balance}, + {"listenhub", "LISTENHUB_BASE", "LISTENHUB_KEY", constant.ChannelTypeListenHub, "", queryListenHubBalance}, + {"newapi-spend", "NEWAPI_SPEND_BASE", "NEWAPI_SPEND_KEY", constant.ChannelTypeOpenAI, "", queryNewAPISpend}, + {"console-only", "CONSOLE_ONLY_BASE", "CONSOLE_ONLY_KEY", constant.ChannelTypeOpenAIVideo, "", queryNewAPISpend}, + } + + for _, tc := range cases { + base, key := os.Getenv(tc.baseEnv), os.Getenv(tc.keyEnv) + if base == "" || key == "" { + t.Logf("skip %s: set %s and %s to run", tc.name, tc.baseEnv, tc.keyEnv) + continue + } + t.Run(tc.name, func(t *testing.T) { + ch := &model.Channel{Type: tc.typ, Other: tc.other, BaseURL: ptr(base), Key: key} + r, err := tc.fn(ch) + if err != nil { + t.Fatalf("%s query error: %v", tc.name, err) + } + t.Logf("%s => kind=%s remaining=%.4f used=%.4f unit=%s expires=%d", tc.name, r.Kind, r.Remaining, r.Used, r.Unit, r.ExpiresAt) + }) + } +} + +// 登录态拿真实钱包余额（new-api 套壳站） +func TestLiveConsoleBalance(t *testing.T) { + skipUnlessLive(t) + base, user, pass := os.Getenv("CONSOLE_BASE"), os.Getenv("CONSOLE_USER"), os.Getenv("CONSOLE_PASS") + if base == "" || user == "" || pass == "" { + t.Skip("set CONSOLE_BASE/CONSOLE_USER/CONSOLE_PASS to run console balance test") + } + ch := &model.Channel{Type: constant.ChannelTypeOpenAI, BaseURL: ptr(base), Key: os.Getenv("CONSOLE_KEY")} + ch.SetSetting(dto.ChannelSettings{ + BalanceQuery: &dto.BalanceQuerySetting{Mode: "newapi_console", Username: user, Password: pass}, + }) + r, err := queryNewAPIStationBalance(ch) + if err != nil { + t.Fatalf("console query error: %v", err) + } + t.Logf("console => kind=%s remaining=%.4f(USD) used=%.4f unit=%s", r.Kind, r.Remaining, r.Used, r.Unit) +} + +// 集成测试：用本地 sqlite 副本初始化 DB，直接调用 GetChannelBalanceOverview（绕过 AdminAuth）， +// 验证一次性查出全部下游余额。需 LIVE_BALANCE_TEST=1 且 OVERVIEW_DB 指向 sqlite 文件。 +func TestLiveBalanceOverview(t *testing.T) { + skipUnlessLive(t) + dbPath := os.Getenv("OVERVIEW_DB") + if dbPath == "" { + t.Skip("set OVERVIEW_DB=/path/to/one-api.db to run the overview integration test") + } + os.Setenv("SQLITE_PATH", dbPath) + common.InitEnv() + if err := model.InitDB(); err != nil { + t.Fatalf("InitDB: %v", err) + } + + gin.SetMode(gin.TestMode) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + c.Request = httptest.NewRequest(http.MethodGet, "/api/channel/balance_overview", nil) + + GetChannelBalanceOverview(c) + + t.Logf("HTTP %d", rec.Code) + t.Logf("response:\n%s", rec.Body.String()) + if rec.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", rec.Code) + } +} diff --git a/controller/channel_upstream_update.go b/controller/channel_upstream_update.go index 77a1e3c817a8..f741004d6454 100644 --- a/controller/channel_upstream_update.go +++ b/controller/channel_upstream_update.go @@ -293,6 +293,12 @@ func fetchChannelUpstreamModelIDs(channel *model.Channel) ([]string, error) { switch channel.Type { case constant.ChannelTypeAli: url = fmt.Sprintf("%s/compatible-mode/v1/models", baseURL) + case constant.ChannelTypeOpenAIVideo: + if isXBSoraModelsAPI(baseURL) { + url = xbSoraModelsURL(baseURL) + } else { + url = fmt.Sprintf("%s/v1/models", baseURL) + } case constant.ChannelTypeZhipu_v4: if plan, ok := constant.ChannelSpecialBases[baseURL]; ok && plan.OpenAIBaseURL != "" { url = fmt.Sprintf("%s/models", plan.OpenAIBaseURL) @@ -331,6 +337,14 @@ func fetchChannelUpstreamModelIDs(channel *model.Channel) ([]string, error) { return nil, err } + if channel.Type == constant.ChannelTypeOpenAIVideo && isXBSoraModelsAPI(baseURL) { + models, err := parseXBSoraModelIDs(body) + if err != nil { + return nil, err + } + return normalizeModelNames(models), nil + } + var result OpenAIModelsResponse if err := common.Unmarshal(body, &result); err != nil { return nil, err diff --git a/controller/relay.go b/controller/relay.go index 1d14dcc6f880..89f1d62da1ef 100644 --- a/controller/relay.go +++ b/controller/relay.go @@ -340,6 +340,9 @@ func shouldRetry(c *gin.Context, openaiErr *types.NewAPIError, retryTimes int) b if _, ok := c.Get("specific_channel_id"); ok { return false } + if isRetryableUpstreamQuotaError(openaiErr) { + return true + } code := openaiErr.StatusCode if code >= 200 && code < 300 { return false @@ -353,8 +356,47 @@ func shouldRetry(c *gin.Context, openaiErr *types.NewAPIError, retryTimes int) b return operation_setting.ShouldRetryByStatusCode(code) } +func isRetryableUpstreamQuotaError(openaiErr *types.NewAPIError) bool { + // 本地额度错误（调用方钱包不足等）均带 SkipRetry 标记，先排除； + // 上游透传的 OpenAI 风格错误会把 errorCode 设为上游 code（如 + // insufficient_user_quota），不能用 ErrorCodeBadResponseStatusCode 过滤， + // 因此把上游 code 一并纳入关键词匹配。 + if openaiErr == nil || types.IsSkipRetryError(openaiErr) { + return false + } + switch openaiErr.StatusCode { + case http.StatusBadRequest, http.StatusPaymentRequired, http.StatusForbidden: + default: + return false + } + + message := openaiErr.Error() + " " + string(openaiErr.GetErrorCode()) + return operation_setting.IsUpstreamQuotaErrorMessage(message) +} + +// isRetryableUpstreamQuotaTaskError 视频任务版本：上游余额/额度不足时换渠道重试。 +func isRetryableUpstreamQuotaTaskError(taskErr *dto.TaskError) bool { + if taskErr == nil || taskErr.LocalError { + return false + } + switch taskErr.StatusCode { + case http.StatusBadRequest, http.StatusPaymentRequired, http.StatusForbidden: + default: + return false + } + message := taskErr.Message + if taskErr.Error != nil { + message += " " + taskErr.Error.Error() + } + return operation_setting.IsUpstreamQuotaErrorMessage(message) +} + func processChannelError(c *gin.Context, channelError types.ChannelError, err *types.NewAPIError) { logger.LogError(c, fmt.Sprintf("channel error (channel #%d, status code: %d): %s", channelError.ChannelId, err.StatusCode, common.LocalLogPreview(err.Error()))) + // 余额/额度不足：渠道进入短暂冷却，冷却期内选路自动跳过，到期自动恢复 + if isRetryableUpstreamQuotaError(err) { + model.SetChannelQuotaCooldown(channelError.ChannelId) + } // 不要使用context获取渠道信息，异步处理时可能会出现渠道信息不一致的情况 // do not use context to get channel info, there may be inconsistent channel info when processing asynchronously if service.ShouldDisableChannel(err) && channelError.AutoBan { @@ -629,6 +671,9 @@ func shouldRetryTaskRelay(c *gin.Context, channelId int, taskErr *dto.TaskError, if taskErr.StatusCode == 307 { return true } + if isRetryableUpstreamQuotaTaskError(taskErr) { + return true + } if taskErr.StatusCode/100 == 5 { // 超时不重试 if operation_setting.IsAlwaysSkipRetryStatusCode(taskErr.StatusCode) { diff --git a/controller/relay_retry_test.go b/controller/relay_retry_test.go new file mode 100644 index 000000000000..9913558d6a98 --- /dev/null +++ b/controller/relay_retry_test.go @@ -0,0 +1,89 @@ +package controller + +import ( + "errors" + "net/http" + "testing" + + "github.com/QuantumNous/new-api/dto" + "github.com/QuantumNous/new-api/types" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +func TestShouldRetryUpstreamQuotaError(t *testing.T) { + gin.SetMode(gin.TestMode) + ctx, _ := gin.CreateTestContext(nil) + + err := types.NewOpenAIError( + errors.New("Insufficient credits for Image generation"), + types.ErrorCodeBadResponseStatusCode, + http.StatusBadRequest, + ) + + require.True(t, shouldRetry(ctx, err, 1)) +} + +func TestShouldRetryDoesNotRetryOrdinaryBadRequest(t *testing.T) { + gin.SetMode(gin.TestMode) + ctx, _ := gin.CreateTestContext(nil) + + err := types.NewOpenAIError( + errors.New("invalid image size"), + types.ErrorCodeBadResponseStatusCode, + http.StatusBadRequest, + ) + + require.False(t, shouldRetry(ctx, err, 1)) +} + +func TestShouldRetryUpstreamRelayedQuotaErrorCode(t *testing.T) { + gin.SetMode(gin.TestMode) + + // 上游 OpenAI 风格错误体透传：errorCode 为上游 code，而非 bad_response_status_code + err := types.WithOpenAIError(types.OpenAIError{ + Message: "当前模型暂时不可用，请稍后重试或联系管理员。", + Type: "new_api_error", + Code: "insufficient_user_quota", + }, http.StatusForbidden) + + require.True(t, isRetryableUpstreamQuotaError(err)) +} + +func TestShouldRetryTaskRelayUpstreamQuotaError(t *testing.T) { + gin.SetMode(gin.TestMode) + ctx, _ := gin.CreateTestContext(nil) + + taskErr := &dto.TaskError{ + Message: "Insufficient credits for video generation", + StatusCode: http.StatusBadRequest, + } + + require.True(t, shouldRetryTaskRelay(ctx, 1, taskErr, 1)) +} + +func TestShouldRetryTaskRelayOrdinaryBadRequest(t *testing.T) { + gin.SetMode(gin.TestMode) + ctx, _ := gin.CreateTestContext(nil) + + taskErr := &dto.TaskError{ + Message: "invalid aspect ratio", + StatusCode: http.StatusBadRequest, + } + + require.False(t, shouldRetryTaskRelay(ctx, 1, taskErr, 1)) +} + +func TestShouldRetryDoesNotRetryLocalUserQuotaError(t *testing.T) { + gin.SetMode(gin.TestMode) + ctx, _ := gin.CreateTestContext(nil) + + err := types.NewErrorWithStatusCode( + errors.New("用户额度不足, 剩余额度: 0"), + types.ErrorCodeInsufficientUserQuota, + http.StatusForbidden, + types.ErrOptionWithSkipRetry(), + ) + + require.False(t, shouldRetry(ctx, err, 1)) +} diff --git a/controller/task_video.go b/controller/task_video.go index 8be2e1769ba4..ea682e5a4298 100644 --- a/controller/task_video.go +++ b/controller/task_video.go @@ -81,8 +81,11 @@ func updateVideoSingleTask(ctx context.Context, adaptor channel.TaskAdaptor, cha key = privateData.Key } resp, err := adaptor.FetchTask(baseURL, key, map[string]any{ - "task_id": taskId, - "action": task.Action, + "task_id": task.GetUpstreamTaskID(), + "action": task.Action, + "channel_other": channel.Other, + "origin_model_name": task.Properties.OriginModelName, + "upstream_model_name": task.Properties.UpstreamModelName, }, proxy) if err != nil { return fmt.Errorf("fetchTask failed for task %s: %w", taskId, err) diff --git a/controller/video_proxy.go b/controller/video_proxy.go index 520d313a312d..07b672387b26 100644 --- a/controller/video_proxy.go +++ b/controller/video_proxy.go @@ -14,6 +14,7 @@ import ( "github.com/QuantumNous/new-api/constant" "github.com/QuantumNous/new-api/logger" "github.com/QuantumNous/new-api/model" + "github.com/QuantumNous/new-api/relay/channel/task/taskcommon" "github.com/QuantumNous/new-api/service" "github.com/QuantumNous/new-api/setting/system_setting" @@ -109,12 +110,33 @@ func VideoProxy(c *gin.Context) { case constant.ChannelTypeOpenAI, constant.ChannelTypeSora: videoURL = fmt.Sprintf("%s/v1/videos/%s/content", baseURL, task.GetUpstreamTaskID()) req.Header.Set("Authorization", "Bearer "+channel.Key) + case constant.ChannelTypeOpenAIVideo: + videoURL = task.GetResultURL() + if isHongniaoVideoChannel(baseURL, channel.Other) { + req.Header.Set("X-API-Key", channel.Key) + } else if strings.TrimSpace(videoURL) == "" || videoURL == taskcommon.BuildProxyURL(task.TaskID) { + // 标准 OpenAI Video 上游（如漫小白）轮询响应不含直链， + // ResultURL 会被填成本服务的对外代理地址（自指）； + // 回源时改用上游鉴权 content 端点代理下载 + videoURL = fmt.Sprintf("%s/v1/videos/%s/content", strings.TrimRight(baseURL, "/"), task.GetUpstreamTaskID()) + req.Header.Set("Authorization", "Bearer "+channel.Key) + } default: // Video URL is stored in PrivateData.ResultURL (fallback to FailReason for old data) videoURL = task.GetResultURL() } videoURL = strings.TrimSpace(videoURL) + skipSSRFValidation := false + if channel.Type == constant.ChannelTypeOpenAIVideo && strings.HasPrefix(videoURL, "runway:") { + filePath := strings.TrimSpace(strings.TrimPrefix(videoURL, "runway:")) + if !strings.HasPrefix(filePath, "/files/") { + videoProxyError(c, http.StatusBadGateway, "server_error", "Invalid Runway file URL") + return + } + videoURL = strings.TrimRight(baseURL, "/") + filePath + skipSSRFValidation = true + } if videoURL == "" { logger.LogError(c.Request.Context(), fmt.Sprintf("Video URL is empty for task %s", taskID)) videoProxyError(c, http.StatusBadGateway, "server_error", "Failed to fetch video content") @@ -130,10 +152,12 @@ func VideoProxy(c *gin.Context) { } fetchSetting := system_setting.GetFetchSetting() - if err := common.ValidateURLWithFetchSetting(videoURL, fetchSetting.EnableSSRFProtection, fetchSetting.AllowPrivateIp, fetchSetting.DomainFilterMode, fetchSetting.IpFilterMode, fetchSetting.DomainList, fetchSetting.IpList, fetchSetting.AllowedPorts, fetchSetting.ApplyIPFilterForDomain); err != nil { - logger.LogError(c.Request.Context(), fmt.Sprintf("Video URL blocked for task %s: %v", taskID, err)) - videoProxyError(c, http.StatusForbidden, "server_error", fmt.Sprintf("request blocked: %v", err)) - return + if !skipSSRFValidation { + if err := common.ValidateURLWithFetchSetting(videoURL, fetchSetting.EnableSSRFProtection, fetchSetting.AllowPrivateIp, fetchSetting.DomainFilterMode, fetchSetting.IpFilterMode, fetchSetting.DomainList, fetchSetting.IpList, fetchSetting.AllowedPorts, fetchSetting.ApplyIPFilterForDomain); err != nil { + logger.LogError(c.Request.Context(), fmt.Sprintf("Video URL blocked for task %s: %v", taskID, err)) + videoProxyError(c, http.StatusForbidden, "server_error", fmt.Sprintf("request blocked: %v", err)) + return + } } req.URL, err = url.Parse(videoURL) @@ -171,6 +195,15 @@ func VideoProxy(c *gin.Context) { } } +func isHongniaoVideoChannel(baseURL, channelOther string) bool { + baseURL = strings.ToLower(strings.TrimSpace(baseURL)) + channelOther = strings.ToLower(strings.TrimSpace(channelOther)) + return strings.Contains(baseURL, "open.hongniaoai.com") || + strings.Contains(channelOther, "hongniao") || + strings.Contains(channelOther, "xb-sora") || + strings.Contains(channelOther, "xbsora") +} + func writeVideoDataURL(c *gin.Context, dataURL string) error { parts := strings.SplitN(dataURL, ",", 2) if len(parts) != 2 { diff --git a/controller/xb_sora_models_test.go b/controller/xb_sora_models_test.go new file mode 100644 index 000000000000..7589a2137d55 --- /dev/null +++ b/controller/xb_sora_models_test.go @@ -0,0 +1,123 @@ +package controller + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/QuantumNous/new-api/constant" + "github.com/QuantumNous/new-api/model" + "github.com/gin-gonic/gin" +) + +func TestXBSoraModelsHelpers(t *testing.T) { + if !isXBSoraModelsAPI("https://example.com/api/v1") { + t.Fatalf("expected /api/v1 base URL to use xb-sora models API") + } + if !isXBSoraModelsAPI("https://localhost:3000/v1") { + t.Fatalf("expected documented localhost /v1 base URL to use xb-sora models API") + } + if !isXBSoraModelsAPI("https://xb-sora2.example.com") { + t.Fatalf("expected xb-sora2 host to use xb-sora models API") + } + if isXBSoraModelsAPI("https://xgapi.top") { + t.Fatalf("xgapi should keep OpenAI-compatible models API") + } + + if got := xbSoraModelsURL("https://example.com/api/v1/"); got != "https://example.com/api/v1/models" { + t.Fatalf("xbSoraModelsURL api base = %q", got) + } + if got := xbSoraModelsURL("https://localhost:3000/v1"); got != "https://localhost:3000/v1/models" { + t.Fatalf("xbSoraModelsURL v1 base = %q", got) + } + if got := xbSoraModelsURL("https://example.com"); got != "https://example.com/api/v1/models" { + t.Fatalf("xbSoraModelsURL host root = %q", got) + } +} + +func TestParseXBSoraModelIDs(t *testing.T) { + models, err := parseXBSoraModelIDs([]byte(`{"code":200,"message":"ok","data":{"models":[{"id":"openai-sora-2"},{"id":"sora-2-image-to-video"},{"id":""}]}}`)) + if err != nil { + t.Fatalf("parseXBSoraModelIDs error: %v", err) + } + if len(models) != 2 || models[0] != "openai-sora-2" || models[1] != "sora-2-image-to-video" { + t.Fatalf("models = %#v", models) + } + + models, err = parseXBSoraModelIDs([]byte(`{"code":"0000","msg":"success","data":{"code":200,"message":"ok","data":{"models":[{"id":"ss-sora-2"},{"id":"xb-sora2"}]}}}`)) + if err != nil { + t.Fatalf("parse nested parseXBSoraModelIDs error: %v", err) + } + if len(models) != 2 || models[0] != "ss-sora-2" || models[1] != "xb-sora2" { + t.Fatalf("nested models = %#v", models) + } + + _, err = parseXBSoraModelIDs([]byte(`{"code":401,"message":"bad key","data":{"models":[]}}`)) + if err == nil { + t.Fatalf("expected error for non-%d code", http.StatusOK) + } +} + +func TestFetchModelsUsesXBSoraModelsAPI(t *testing.T) { + gin.SetMode(gin.TestMode) + + var gotPath string + var gotAPIKey string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotAPIKey = r.Header.Get("X-API-Key") + if auth := r.Header.Get("Authorization"); auth != "" { + t.Fatalf("Authorization should not be sent, got %q", auth) + } + _, _ = w.Write([]byte(`{"code":200,"message":"ok","data":{"models":[{"id":"openai-sora-2"},{"id":"future-video-model"}]}}`)) + })) + defer server.Close() + + body := []byte(`{"base_url":"` + server.URL + `/api/v1","type":58,"key":"sk_test"}`) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodPost, "/api/channel/fetch_models", bytes.NewReader(body)) + c.Request.Header.Set("Content-Type", "application/json") + + FetchModels(c) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", w.Code, w.Body.String()) + } + if gotPath != "/api/v1/models" { + t.Fatalf("upstream path = %q", gotPath) + } + if gotAPIKey != "sk_test" { + t.Fatalf("X-API-Key = %q", gotAPIKey) + } + + var resp struct { + Success bool `json:"success"` + Data []string `json:"data"` + } + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("unmarshal response: %v", err) + } + if !resp.Success || len(resp.Data) != 2 || resp.Data[1] != "future-video-model" { + t.Fatalf("response = %+v", resp) + } +} + +func TestBuildFetchModelsHeadersUsesXAPIKeyForXBSora(t *testing.T) { + baseURL := "https://example.com/api/v1" + headers, err := buildFetchModelsHeaders(&model.Channel{ + Type: constant.ChannelTypeOpenAIVideo, + BaseURL: &baseURL, + }, "sk_test") + if err != nil { + t.Fatalf("buildFetchModelsHeaders error: %v", err) + } + if got := headers.Get("X-API-Key"); got != "sk_test" { + t.Fatalf("X-API-Key = %q", got) + } + if got := headers.Get("Authorization"); got != "" { + t.Fatalf("Authorization should be empty, got %q", got) + } +} diff --git a/docs/api-usage.md b/docs/api-usage.md new file mode 100644 index 000000000000..81df6a343322 --- /dev/null +++ b/docs/api-usage.md @@ -0,0 +1,1624 @@ +# API 调用文档 + +本文档面向合作方接入和 AI agent 自动调用，覆盖视频生成、图片生成、图像编辑、文本对话、模型列表、错误处理和价格参考。所有接口兼容 OpenAI API 格式，可直接使用 OpenAI SDK 或兼容客户端调用。 + +> **最后验证：2026-06-11**。当前分支已部署 quota 冷却机制与漫小白（manxiaobai）渠道；`/api/status`、`/v1/models`、`gpt-image-2` xgapi 主路径与漫小白兜底轮换、`gpt-image-2` 漫小白参考图编辑、`gpt-image-2-1k` 高分档位、`gemini-3.1-flash-image-preview` 漫小白 Gemini 原生路径、`grok-imagine-video` 视频全流程（提交/轮询/下载）均已通过远端真实接口验证。 + +--- + +## 连接信息 + +| 项目 | 值 | +|------|-----| +| Base URL | `http://192.129.209.36:3001/v1` | +| 认证方式 | HTTP Header `Authorization: Bearer ` | +| 兼容协议 | OpenAI API (Chat Completions, Models, Images, Video Generations) | +| 内部测试 API Key | `EW93ybOP6Zr1axAPYNEu8VpehQzdTkZBTATszAGYEDiwpCmJ` | +| 测试 Key 额度 | 当前测试 Key 为本服务侧无限额度（unlimited_quota）；生产 Key 以实际配置为准 | + +当前入口运行在 2026-05-26 迁移后的新服务器上，由 Coolify 资源 `new-api-video-gateway` 管理。2026-05-28 完成 upstream 合并（78 commits）后重新部署并完成视频全模型回归测试；2026-06-06 再次合并 upstream 最新代码并完成 SiliconFlow 图片模型实测；2026-06-10 合并 upstream 27 commits 并部署 quota 冷却故障转移机制；2026-06-11 接入漫小白（manxiaobai）渠道并完成图片/参考图/视频全链路真实验证。 + +所有请求必须在 HTTP Header 中携带 API Key： + +``` +Authorization: Bearer EW93ybOP6Zr1axAPYNEu8VpehQzdTkZBTATszAGYEDiwpCmJ +``` + +本文档保留的是本服务内部测试 Key，供联调和 AI agent 读取。生产 Key、上游供应商 Key 和个人 Key 不要写入代码、日志、Prompt 或截图。 + +注意：上表的测试 Key 额度只表示本服务侧 token/user 钱包额度，不等于 ListenHub、xgapi、SiliconFlow 等上游供应商账号余额。若上游 channel 自身返回 credit/balance 不足，本服务会按模型能力表尝试切换到其它可用 channel。 + +--- + +## 给 AI agent 的读取规则 + +- 模型名必须逐字使用表格中的值，不要翻译、改大小写、替换斜杠或自动补后缀。 +- 上游调用方优先使用推荐的标准模型名；不要为了指定供应商自行拼接线路后缀或平台名。 +- 同一个对外模型名可能由多个内部 channel 承载，服务会按配置自动路由、重试和做模型名映射。 +- 带“线路”或供应商风格的模型名通常是历史兼容或排障别名，只有表格明确推荐时才给业务方使用。 +- 所有请求统一使用 `http://192.129.209.36:3001/v1` 作为 Base URL，并携带上方 `Authorization` Header。 +- 图片生成返回可能是 `data[0].url` 或 `data[0].b64_json`；客户端必须同时兼容两种字段，不要假设只有一种。 +- 视频生成是异步任务：先 `POST /v1/videos` 获取 `task_id`，再 `GET /v1/videos/{task_id}` 轮询，完成后用 `GET /v1/videos/{task_id}/content` 下载。 +- 不要在日志中打印完整 `b64_json`、签名图片 URL、视频下载 URL 或 API Key；排障只记录模型名、HTTP 状态、耗时、`task_id` 和返回字段是否存在。 +- 优先调用“现有可用模型合集”中的推荐模型；“暂不推荐/不建议”的模型不要主动推荐给业务方。 +- 如果 `/v1/models` 中出现本文档未列出的模型，先小流量真实验证，再更新本文档。 + +--- + +## 现有可用模型合集（2026-06-11） + +本节是给人和 AI agent 的快速索引。详细参数、轮询方式、错误处理和价格见后续章节。 + +### 模型命名与自动路由 + +对外模型名是本服务给上游的稳定调用名，不等同于下游供应商内部模型名。调用方只需要传推荐模型名，系统会自动选择可用 channel，并在需要时把模型名映射成下游实际名称。 + +| 场景 | 上游推荐传参 | 内部处理 | +|------|--------------|----------| +| `gpt-image-2` 直接生图 | `model: "gpt-image-2"` | 优先走 xgapi 直出生图，漫小白为第一兜底（quota/失败自动转移并冷却坏渠道 10 分钟），ListenHub 垫底 | +| `gpt-image-2` 带参考图 | `model: "gpt-image-2"` + `image` / `images`（或 `POST /v1/images/edits`） | 首选漫小白参考图接口（2026-06-11 真实验证，约 43 秒），ListenHub 为兜底。大体积 `data:image/...;base64,...` 参考图需压缩或改用 URL | +| 历史图片别名 | `gpt-image-2(线路XF)` / `gr-image-2` / `nano-banana-pro` | 作为兼容别名映射到 xgapi 上游 `gpt-image-2` | +| `grok-video-3` 视频 | `model: "grok-video-3"` | 走 LK888 视频 channel；`/v1/models` 已暴露该标准名 | + +除非是在排障或兼容旧调用方，不建议上游主动选择带线路后缀的别名。 + +### 视频模型 + +| 推荐模型 | 入口 | 类型 | 远端实测 | 返回方式 | 适用场景 | +|----------|------|------|----------|----------|----------| +| `veo3.1-fast` | `POST /v1/videos` | 文生视频、图生视频 | 约 1.5 分钟 | `task_id` 后轮询 | 默认首选，速度和成本均衡 | +| `xb-sora2` | `POST /v1/videos` | 文生视频、参考图视频 | 约 3.5 分钟 | `task_id` 后轮询 | Sora 2 主路径 | +| `grok-imagine-1.0-video` | `POST /v1/videos` | 文生视频、参考图视频 | 约 2 分钟 | `task_id` 后轮询 | Grok Imagine；建议使用稳定尺寸 | +| `ss-sora-2` | `POST /v1/videos` | 文生视频 | 约 3 分钟 | `task_id` 后轮询 | Sora 2 备用路径 | +| `veo3.1-4k` | `POST /v1/videos` | 文生视频、图生视频 | 约 4 分钟 | `task_id` 后轮询 | 4K 高质量输出 | +| `grok-imagine-video` | `POST /v1/videos` | 文生视频、参考图视频（10s） | 约 2-3 分钟 | `task_id` 后轮询 | 漫小白渠道，已真实验证（注意 `seconds` 必须传字符串）；下载走 `/v1/videos/{task_id}/content` | +| `grok-imagine-video-1.5-preview` | `POST /v1/videos` | 参考图视频（必须参考图，10/15s） | 待实测 | `task_id` 后轮询 | 漫小白渠道；参考图预上传协议待验证 | + +### 图片生成与编辑模型 + +| 推荐模型 | 入口 | 类型 | 远端实测耗时 | 返回字段 | 适用场景 | +|----------|------|------|--------------|----------|----------| +| `Tongyi-MAI/Z-Image` | `POST /v1/images/generations` | 生图 | 12.20 秒 | `data[0].url` | SiliconFlow 通义图片路径，当前实测最快 | +| `Qwen/Qwen-Image` | `POST /v1/images/generations` | 生图 | 18.76 秒 | `data[0].url` | SiliconFlow Qwen 生图，高质量通用 | +| `baidu/ERNIE-Image-Turbo` | `POST /v1/images/generations` | 生图 | 20.89 秒 | `data[0].url` | SiliconFlow 文心快速生图 | +| `Qwen/Qwen-Image-Edit-2509` | `POST /v1/images/edits` | 图像编辑 | 24.60 秒 | `data[0].url` | SiliconFlow 图像编辑、风格转换 | +| `gemini_3.1_flash_image_preview` | `POST /v1/images/generations` | 生图 | 约 29 秒 | `data[0].b64_json` | Apexer 快速生图 | +| `gemini_3.0_pro_image_preview` | `POST /v1/images/generations` | 生图 | 约 58 秒 | `data[0].b64_json` | Apexer 高质量图片、产品图 | +| `gemini_3.1_flash_image_preview_4K` | `POST /v1/images/generations` | 生图 | 约 65 秒 | `data[0].b64_json` | Apexer 快速高清输出 | +| `gemini_3.0_pro_image_preview_4K` | `POST /v1/images/generations` | 生图 | 约 383 秒 | `data[0].b64_json` | Apexer 4K 高质量，耗时较长 | +| `gemini-3.1-flash-image-preview` | `POST /v1/images/generations` | 生图 | 约 22-93 秒 | `data[0].b64_json` | 横线命名快速生图；主路径 Apexer（映射 `gemini_3.1_flash_image_preview`），漫小白第二兜底（Gemini 原生入口实测 22 秒），ListenHub 垫底 | +| `gemini-3-pro-image-preview` | `POST /v1/images/generations` | 生图 | 约 58-67 秒 | `data[0].b64_json` | 横线命名高质量生图；主路径 Apexer（映射 `gemini_3.0_pro_image_preview`），漫小白第二兜底，ListenHub 垫底 | +| `gpt-image-2` | `POST /v1/images/generations` | 生图 | 约 45-90 秒 | `data[0].url` | 主路径 xgapi 直出生图，漫小白第一兜底；带参考图走 `/v1/images/edits`（漫小白，约 43 秒） | +| `gpt-image-2-1k` / `-2k` / `-4k` | `POST /v1/images/generations` | 生图（高分辨率档位） | 1k 实测约 138 秒 | `data[0].url` | 漫小白独家档位，已真实验证；需按档位传对应尺寸（如 1k 16:9 传 `1824x1024`，4k 传 `3840x2160`） | +| `gpt-image-2(线路XF)` | `POST /v1/images/generations` | 生图 | 48-50 秒 | `data[0].url` | 映射到 xgapi `gpt-image-2` | +| `gr-image-2` | `POST /v1/images/generations` | 生图 | 46-55 秒 | `data[0].url` | 映射到 xgapi `gpt-image-2` | +| `nano-banana` | `POST /v1/images/generations` | 生图 | 8-9 秒 | `data[0].url` | bltcy 快速生图 | +| `nano-banana-hd` | `POST /v1/images/generations` | 生图 | 10-11 秒 | `data[0].url` | bltcy 高清生图 | +| `nano-banana-pro` | `POST /v1/images/generations` | 生图 | 46-48 秒 | `data[0].url` | 映射到 xgapi `gpt-image-2` 兜底 | + +### 文本模型 + +| 推荐模型 | 入口 | 类型 | 说明 | +|----------|------|------|------| +| `gemini-2.5-flash` | `POST /v1/chat/completions` | 文本对话 | 快速文本对话，响应格式与 OpenAI Chat Completions 一致 | + +### 暂不推荐直接调用的模型 + +| 模型 | 原因 | 替代建议 | +|------|------|----------| +| `openai-sora-2`、`sora-2-image-to-video`、`sora-2-pro-text-to-video`、`sora-2(线路BF)` | 真实创建失败或下游未开放 OpenAPI | 使用 `xb-sora2` 或 `ss-sora-2` | +| `grok-video-3(线路W)` | 下游未开放 OpenAPI | 使用 `grok-imagine-1.0-video` | +| `veo3.1-lite`、`全能视频2.0` | 远端创建失败 | 使用 `veo3.1-fast` 或 `veo3.1-4k` | +| `seedance-*`、`gen4-*`、`wan-*`、`kling-*`、`happyhorse-*`、`pixverse`、`vidu` | Runway 私有适配器当前未就绪 | 等 Runway 渠道上线后再验证 | +| `gemini-2.5-flash-image*` | 模型列表可能暴露，但未完成本服务真实生成验证 | 使用上表已验证图片模型 | + +--- + +## 快速开始 + +以下是已通过真实验证的最小调用示例，可直接替换 API Key 后调用。视频提交后返回 `task_id`，轮询 `GET /v1/videos/{task_id}` 即可获取生成结果；图片接口同步返回 `data` 数组。 + +### SiliconFlow 生图 — Qwen 图片生成（实测 18.76 秒） + +```bash +curl -s "http://192.129.209.36:3001/v1/images/generations" \ + -H "Authorization: Bearer your-api-key-here" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "Qwen/Qwen-Image", + "prompt": "A clean product photo of a white ceramic coffee cup on a wooden desk, soft studio lighting", + "size": "1024x1024", + "n": 1 + }' +``` + +### SiliconFlow 图像编辑 — Qwen Image Edit（实测 24.60 秒） + +```bash +curl -s "http://192.129.209.36:3001/v1/images/edits" \ + -H "Authorization: Bearer your-api-key-here" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "Qwen/Qwen-Image-Edit-2509", + "prompt": "Turn the input image into a clean watercolor illustration while preserving the main subject", + "image": "https://example.com/input.png", + "n": 1 + }' +``` + +### veo3.1-fast — 快速视频生成（≈1.5 分钟，$0.30/次） + +```bash +curl -s "http://192.129.209.36:3001/v1/videos" \ + -H "Authorization: Bearer your-api-key-here" \ + -H "Content-Type: application/json" \ + -d '{"model":"veo3.1-fast","prompt":"A golden retriever running on a beach at sunset, cinematic quality"}' +``` + +### xb-sora2 — Sora 2 主路径（≈3.5 分钟，$0.40/次） + +```bash +curl -s "http://192.129.209.36:3001/v1/videos" \ + -H "Authorization: Bearer your-api-key-here" \ + -H "Content-Type: application/json" \ + -d '{"model":"xb-sora2","prompt":"A cat walking through a neon-lit cyberpunk alley at night"}' +``` + +### grok-imagine-1.0-video — Grok 视频（≈2 分钟，$0.025/次） + +```bash +curl -s "http://192.129.209.36:3001/v1/videos" \ + -H "Authorization: Bearer your-api-key-here" \ + -H "Content-Type: application/json" \ + -d '{"model":"grok-imagine-1.0-video","prompt":"A green sphere floating over a white table, clean studio lighting","seconds":"6","size":"720x1280"}' +``` + +> **⚠️ 尺寸约束**：Grok 稳定验证过的尺寸是 `720x1280`、`1280x720`、`1024x1024`、`1024x1792`、`1792x1024`。`aspect_ratio` 也会映射 `4:3`、`3:4`、`21:9`，但这 3 个比例未完成同等生产抽检。 + +### ss-sora-2 — Sora 2 备用路径（≈3 分钟，$0.40/次） + +```bash +curl -s "http://192.129.209.36:3001/v1/videos" \ + -H "Authorization: Bearer your-api-key-here" \ + -H "Content-Type: application/json" \ + -d '{"model":"ss-sora-2","prompt":"A drone flying over a misty mountain landscape at sunrise"}' +``` + +### veo3.1-4k — 4K 高质量（≈4 分钟，$1.50/次） + +```bash +curl -s "http://192.129.209.36:3001/v1/videos" \ + -H "Authorization: Bearer your-api-key-here" \ + -H "Content-Type: application/json" \ + -d '{"model":"veo3.1-4k","prompt":"Aerial view of a tropical island with crystal clear water, cinematic 4K quality"}' +``` + +轮询查询、完整参数说明和 Python SDK 示例见下方章节。 + +--- + +## 一、视频生成（OpenAI Video 兼容） + +视频生成采用**异步任务模式**：先提交任务获取 task_id，然后轮询任务状态，直到视频生成完成。 + +### 1.1 提交视频生成任务 + +**请求：** + +``` +POST {Base URL}/videos +Content-Type: application/json +Authorization: Bearer +``` + +`POST {Base URL}/video/generations` 仍保留兼容，但新接入方推荐统一使用 `/videos`。 + +Sora/Hongniao 渠道的专项说明见 [Sora 视频生成渠道调用文档](./sora-video-api.md)。AI 聚合站 / LK888 的 `grok-video-3` 线路说明见 [AI 聚合站 / LK888 视频渠道接入文档](./lk888-video-api.md)。 + +> **2026-05-28 全模型回归验证结论**：当前推荐上游使用 `veo3.1-fast`、`xb-sora2`、`grok-imagine-1.0-video`、`ss-sora-2`、`veo3.1-4k`。这 5 个模型已通过真实创建、轮询完成和 `/content` 视频下载验证。`grok-video-3` 在 2026-05-24 可用但今天 LK888 上游参数验证失败，暂降级为"尝试"状态。Runway 系列（seedance/gen4/wan/kling/happyhorse/runway）均未就绪。`openai-sora-2`、`sora-2(线路BF)`、`grok-video-3(线路W)`、`veo3.1-lite`、`全能视频2.0` 虽然可能出现在模型列表中，但真实创建失败，见 [1.3 可用视频模型](#13-可用视频模型)。 + +#### 1.1.1 文生视频 + +最基础的调用方式，仅提供文字描述即可生成视频。 + +```json +{ + "model": "veo3.1-fast", + "prompt": "A golden retriever running on a beach at sunset, cinematic quality, slow motion" +} +``` + +#### 1.1.2 图生视频（首帧） + +提供一张图片作为视频的首帧，模型会基于图片内容生成后续视频。 + +```json +{ + "model": "veo3.1", + "prompt": "The character starts walking forward slowly", + "images": [ + "https://example.com/first_frame.jpg" + ] +} +``` + +#### 1.1.3 图生视频（首尾帧） + +提供两张图片分别作为视频的首帧和尾帧，模型会生成从首帧过渡到尾帧的视频。**仅部分模型支持首尾帧**（见下方模型列表）。 + +```json +{ + "model": "veo3.1", + "prompt": "Smooth transition from the first pose to the second pose", + "images": [ + "https://example.com/first_frame.jpg", + "https://example.com/last_frame.jpg" + ] +} +``` + +#### 1.1.4 多图参考（Components 模式） + +提供 1-3 张参考图片，模型会将这些图片作为视频中的元素融合生成。使用 `veo3.1-components` 或 `veo3.1-fast-components` 模型。 + +```json +{ + "model": "veo3.1-components", + "prompt": "A person wearing the outfit in front of the building", + "images": [ + "https://example.com/person.jpg", + "https://example.com/outfit.jpg", + "https://example.com/building.jpg" + ] +} +``` + +#### 1.1.5 带额外参数 + +```json +{ + "model": "veo3.1", + "prompt": "A cat walking across the room", + "images": [ + "https://example.com/first_frame.jpg" + ], + "aspect_ratio": "16:9", + "enhance_prompt": true +} +``` + +#### 1.1.6 Grok 视频（多参、参考图、首尾帧） + +937qq / Qilin 的 Grok 视频模型已按统一 OpenAI Video 入口接入。上游仍然传 JSON，不需要知道 937qq 的真实接口、令牌或返回字段。 + +**文生视频 + 多参数：** + +```json +{ + "model": "grok-imagine-1.0-video", + "prompt": "A green sphere floating over a white table, clean studio lighting", + "seconds": "6", + "size": "1792x1024", + "quality": "standard" +} +``` + +**单参考图：** + +```json +{ + "model": "grok-imagine-1.0-video", + "prompt": "Use the provided reference image as the visual basis and animate it subtly", + "seconds": "6", + "images": [ + "https://example.com/reference.png" + ] +} +``` + +**首尾帧：** + +```json +{ + "model": "grok-imagine-1.0-video", + "prompt": "Create a smooth transition from the first frame to the last frame", + "seconds": "6", + "images": [ + "https://example.com/start.png", + "https://example.com/end.png" + ] +} +``` + +`images` 也支持 `data:image/png;base64,...` 形式。2026-05-15 已用 base64 红圆首帧 + 蓝方块尾帧做抽帧验证，确认参考图和首尾帧视觉生效。 + +本服务会把上游常用参考图字段自动转换成 937qq/Grok 更偏好的 `image_reference` 结构。普通调用方继续传 `images` 即可，不需要直接依赖 937qq 私有字段。2026-05-16 用只包含 `images` 的医生参考图请求复测，任务 `task_QFcwttd20S49mJUdM9Y7wTDNM5XhBdtM` 输出 720×1280，抽帧确认参考图身份、黑色服装、诊室场景和指膝腿动作生效。 + +真实医生讲解 query 建议按参考图优先写法改造：明确写出 `elderly Chinese woman`、`gray hair`、`black traditional Chinese medical clothing`、`indoor clinic room`，并明确排除 `man` / `white-coat western doctor`。不要用 `him` / `his` 描述医生。2026-05-16 复测任务 `task_k6Id9R1pS3LbK22GHLLnDbUHFVPfsF5x` 输出 720×1280，抽帧确认灰发老年女性、黑色中式服装、诊室环境和指背/指脸/指膝腿动作保留较好。 + +Grok 渠道注意事项： + +- `aspect_ratio: "9:16"` 或 `ratio: "9:16"` 会自动补 `size: "720x1280"`。 +- `aspect_ratio: "16:9"` 或 `ratio: "16:9"` 会自动补 `size: "1280x720"`。 +- `aspect_ratio: "1:1"` 或 `ratio: "1:1"` 会自动补 `size: "1024x1024"`。 +- `aspect_ratio: "4:3"`、`"3:4"`、`"21:9"` 会按新版麒麟插件分别补 `size: "1152x864"`、`"864x1152"`、`"1680x720"`。 +- 本服务会同时补齐 `seconds` 和 Grok 官方风格的 `duration`，默认补 `resolution: "720p"`，并按 `resolution` 补 `quality`。 +- `grok-imagine-1.0-video` 传 `duration` / `seconds` 为 20 或 30 秒时，会自动转发到 `grok-imagine-1.0-video-20s` / `grok-imagine-1.0-video-30s`；直接请求这两个模型时会锁定对应时长。 +- 实测 `720x1280` 可以输出 720×1280 或 416×752 这类竖屏结果，`1280x720` 输出过 752×416 横屏结果，`1:1` 映射后任务 `task_EocEzfLxfQGPZ04Y7nYKgga7l0hYnpZ6` 输出 960×960；下游可能按自身编码规格缩放，不保证像素级严格等于目标尺寸。 +- `4:3`、`3:4`、`21:9` 已按新版麒麟插件映射透传，但还没有像 9:16 / 16:9 / 1:1 一样完成生产视频抽检。 +- 人物参考图是软约束，适合保留构图/动作/颜色等显著视觉特征；对“同一个人/医生身份完全一致”的锁定能力不稳定。 + +### 1.2 请求参数 + +| 参数名 | 类型 | 必填 | 说明 | +|--------|------|------|------| +| model | string | 是 | 视频生成模型名称，见下方模型列表 | +| prompt | string | 是 | 视频内容描述，建议用英文，描述越详细效果越好 | +| images | array[string] | 否 | 参考图片 URL 或 base64 编码。Grok 新版插件上限为 7 张；传图后自动启用图生视频/参考图模式 | +| aspect_ratio | string | 否 | 视频比例，可选 `16:9`、`9:16`、`1:1`、`4:3`、`3:4`、`21:9`。Grok 渠道会自动映射为像素 `size` | +| enhance_prompt | boolean | 否 | 是否优化提示词。由于 Veo 只支持英文提示词，开启后会自动将中文提示词翻译为英文并优化。默认 false | +| enable_upsample | boolean | 否 | 是否提升分辨率至 1080p。仅文生视频支持。默认 false | +| seconds | string | 否 | 视频时长。Grok 支持 `6`、`10`、`15`、`20`、`30` | +| duration | integer | 否 | 视频时长（秒）。Grok 渠道会和 `seconds` 互补；20/30 秒会自动转长时长传输模型 | +| ratio | string | 否 | 兼容麒麟插件字段。Grok 渠道未传 `size` 时会按 `aspect_ratio` 同样规则映射 | +| resolution | string | 否 | Grok 渠道未传时默认 `720p` | +| quality | string | 否 | Qilin/Grok 原生画质字段。未传时按 `resolution` 自动补 `high` 或 `standard` | +| size | string | 否 | 输出尺寸。Grok 横屏建议 `1280x720`，竖屏建议 `720x1280`，方形建议 `1024x1024`；`aspect_ratio=4:3/3:4/21:9` 会映射 `1152x864`、`864x1152`、`1680x720`，但未完成同等生产抽检。不要直接传未验证尺寸（如 `1920x1080`） | + +### 1.3 可用视频模型 + +#### 真实验证可用模型（推荐上游使用） + +以下模型已做过真实生成测试：提交任务成功、轮询到 `completed`、并且 `GET /v1/videos/{task_id}/content` 返回 `200 video/mp4`。 + +| 推荐模型 | 下游链路 | 本次验证 task_id | 结果 | 说明 | +|----------|----------|------------------|------|------| +| `veo3.1-fast` | Apexer / Veo | `task_kPRJVkUnFmkznGZbKaUAY8UAy5daQTaS` | ✅ 完成并可下载 | 当前推荐的 Veo 快速模型，约 1.5 分钟完成 | +| `xb-sora2` | Hongniao / Sora | `task_AnRb9zA2TNPKnUl3WjK0ep2yvbBgdaoD` | ✅ 完成并可下载 | 当前推荐的 Sora 主路径，约 3.5 分钟完成 | +| `grok-imagine-1.0-video` | 937qq / Qilin Grok | `task_0N4mwgTkQS8mlV8iYiTa1D385u2o2CRf` | ✅ 完成并可下载 | 推荐的 Grok Imagine 路径；稳定验证尺寸见 [1.2 请求参数](#12-请求参数) | +| `grok-video-3` | LK888 / AI 聚合站 | `task_fjCxJlZ18U0eQIQOfXy077K4HHMzNHum` | ✅ 完成并可下载 | 2026-06-07 复测完成，`/content` 返回 `video/mp4` | +| `ss-sora-2` | Hongniao / Sora | `task_s4H8Mwn0LwsUMviZTWviEH2GVBHvC7V4` | ✅ 完成并可下载 | Sora 2 备用路径，约 3 分钟完成 | +| `veo3.1-4k` | Apexer / Veo 4K | `task_mDFMyYk4fXPREqIZad9ZFTSNvEhQ46Wz` | ✅ 完成并可下载 | 4K 高质量，约 4 分钟完成，$1.5/次 | +| `grok-imagine-video` | 漫小白 / Grok 1.0 | `task_hW0IUCjIRm4fpZjtyqiCKMaU1ZrgLcdL` | ✅ 完成并可下载 | 2026-06-11 验证，10s 视频约 2-3 分钟完成，$0.25/次；`seconds` 必须传字符串 | + +下载抽查结果： + +| 模型 | `/content` 状态 | Content-Type | 下载大小 | +|------|-----------------|--------------|----------| +| `veo3.1-fast` | `200` | `video/mp4` | 约 3.3 MB | +| `xb-sora2` | `200` | `video/mp4` | 约 6.4 MB | +| `grok-imagine-1.0-video` | `200` | `video/mp4` | 约 3.8 MB | +| `ss-sora-2` | `200` | `video/mp4` | 约 7.8 MB | +| `veo3.1-4k` | `200` | `video/mp4` | 约 23 MB | +| `grok-imagine-video` | `200` | `video/mp4` | 约 18.5 MB | + +#### 可尝试但未逐一真实验证的同族模型 + +这些模型属于当前可用链路的同族模型，可能出现在 `/v1/models` 中，但本次没有逐个消耗额度真实生成。业务上建议先使用上方 5 个推荐模型；如需使用下列模型，请先小流量单独验证。 + +| 模型名 | 链路 | 说明 | +|--------|------|------| +| `veo3.1`、`veo3.1-pro`、`veo3.1-4k`、`veo3.1-fast-4k`、`veo3.1-pro-4k` | Apexer / Veo | 同属 Veo/Apexer 链路；高质量和 4K 成本更高 | +| `veo3.1-components`、`veo3.1-fast-components`、`veo3.1-components-4k`、`veo3.1-fast-components-4k` | Apexer / Veo | 多图参考/Components 模式，本次未做真实生成 | +| `ss-sora-2`、`je-grok`、`全能视频2.0` | Hongniao | `ss-sora-2` 已升级为验证模型；`je-grok` 今天 429 限流；`全能视频2.0` 今天上游返回模型不存在 | +| `grok-imagine-1.0-video-20s`、`grok-imagine-1.0-video-30s` | 937qq / Qilin Grok | 长时长 Grok 模型，成本按秒增加，但今天 `20s` 返回 `model_not_found`（渠道未注册） | + +#### 暴露但当前不建议上游调用的模型 + +| 模型名 | 本次真实结果 | 处理建议 | +|--------|--------------|----------| +| `openai-sora-2` | 创建失败：请求 8 秒仍被兼容层归一化成 10 秒，下游返回“仅支持 8 秒、12 秒” | 不建议上游使用；请直接用 `xb-sora2` | +| `sora-2-image-to-video` | 与 `openai-sora-2` 属同一兼容映射链路 | 不建议上游使用；请直接用 `xb-sora2` | +| `sora-2-pro-text-to-video` | 兼容映射到 Hongniao BF 线路；BF 线路本次创建被下游拒绝 | 暂不建议上游使用 | +| `sora-2(线路BF)` | 创建失败：下游返回“当前未开放给 OpenAPI 使用” | 不要直接调用 | +| `grok-video-3(线路W)` | 创建失败：下游返回“当前未开放给 OpenAPI 使用” | 不要直接调用；需要 Grok 请用 `grok-video-3` 或 `grok-imagine-1.0-video` | +| `veo3.1-lite` | 创建失败：`multipart: NextPart: EOF` | 暂不建议上游使用 | +| `全能视频2.0` | 创建失败：上游返回"Model does not exist or is not available" | 不要推荐给上游 | +| `seedance-2`、`gen4-turbo`、`gen4.5`、`wan-2.6*`、`kling-*`、`happyhorse-1`、`pixverse`、`vidu` | Runway 私有适配器系列，当前未部署，不具备可用性 | 不要推荐给上游 | +| `香蕉2(线路V)`、`香蕉pro(线路G)` | 模型列表暴露，但未完成真实 OpenAPI 生成验证 | 不要推荐给上游 | + +> **2026-05-28 补充**：Runway 渠道当前未配置。代码中已注册的新 Kling 3.0/O3 系列模型（`kling-3.0-pro`、`kling-3.0-standard`、`kling-3.0-4k`、`kling-3.0-motion-control`、`kling-o3-pro`、`kling-o3-standard`、`kling-o3-4k`、`kling-2.6-motion-control`）以及 `qilin-video-storyboard-pro` 均在 `constants.go` 注册了但模型列表未暴露，待 Runway 就绪后统一上线。 + +### 1.4 模型自动映射规则 + +系统根据请求中是否包含 `images` 字段，自动将基础模型映射为合适的下游模型： + +| 基础模型 | 无 images（文生视频） | 有 images（图生视频） | +|----------|---------------------|---------------------| +| veo3.1 | veo3.1 | veo3.1（自带首尾帧支持） | +| veo3.1-fast | veo3.1-fast | veo3.1-fast（自带首尾帧支持） | +| veo3.1-pro | veo3.1-pro | veo3.1-pro（自带首尾帧支持） | +| veo3.1-components | veo3.1-components | veo3.1-components（多图参考） | +| veo3 | veo3 | veo3-pro-frames | +| veo3-fast | veo3-fast | veo3-fast-frames | +| veo2-fast | veo2-fast | veo2-fast-frames | +| xb-sora2 | xb-sora2 | xb-sora2 | +| openai-sora-2 | xb-sora2（当前不建议使用该别名） | xb-sora2（当前不建议使用该别名） | +| sora-2-image-to-video | xb-sora2（当前不建议使用该别名） | xb-sora2（当前不建议使用该别名） | +| sora-2-pro-text-to-video | sora-2-pro(线路BF)（当前不可用） | sora-2-pro(线路BF)（当前不可用） | + +> **设计原则**：上游调用方无需感知下游中转站的模型命名、端点和参数差异。只需使用基础模型名 + `images` 字段，系统自动处理路由、模型映射以及 Apexer 的 `type=1/2/3` 参数。后续对接新的中转站时，只需在内部映射表中添加规则，上游调用方式不变。 + +#### Hongniao AI / xb-sora2 接入说明 + +Hongniao AI 使用独立接口协议，当前通过 OpenAI Video 类型 58 的 `xb-sora2` Provider 适配： + +| 项目 | 配置 | +|------|------| +| Base URL | `https://open.hongniaoai.com/v1` | +| 鉴权 | `X-API-Key` | +| 创建任务 | `POST /videos/generate` | +| 查询任务 | `GET /videos/{task_id}` | +| 模型发现 | `GET /models` | + +调用方仍使用本项目统一的 `/v1/videos` 和 `/v1/videos/{task_id}`。Provider 内部会处理： + +- `Authorization: Bearer <用户 token>` → 下游 `X-API-Key` +- 下游外层响应 `{"code":"0000","data":{"code":200,"data":...}}` → 本项目任务状态 +- `seconds` / `duration` → 下游 `duration` +- `aspect_ratio` / `ratio` / `size` → 下游 `orientation` +- `images` / `image` / `input_reference` / `image_url` → 下游 `images` + +参考图能力：Hongniao 文档说明 `images` 最多 5 张；本项目已把统一参考图字段收敛为下游 `images` 数组。当前已验证 `xb-sora2` 文生视频生产链路，以及带 1 张 `images` 参考图的生产链路。2026-05-24 追加真实验证 `xb-sora2` 文生视频任务 `task_DT2laJX2fCTBFeg8VvIx7DxTCllJZpOG`，最终 `completed` 且 `/content` 可下载。具体“身份一致性/首尾帧效果”仍取决于 Hongniao 下游模型本身。 + +### 1.5 成功响应（HTTP 200） + +```json +{ + "id": "task_cIfhoNBQFqDcgxcpr969DQVXw0ApwGpH", + "task_id": "task_cIfhoNBQFqDcgxcpr969DQVXw0ApwGpH" +} +``` + +返回的 `task_id` 用于后续查询任务状态。 + +### 1.6 查询视频生成状态 + +**请求：** + +``` +GET {Base URL}/videos/{task_id} +Authorization: Bearer +``` + +`GET {Base URL}/video/generations/{task_id}` 仍保留兼容。 + +将 `{task_id}` 替换为提交任务时返回的 task_id。 + +**响应（生成中）：** + +```json +{ + "id": "task_cIfhoNBQFqDcgxcpr969DQVXw0ApwGpH", + "object": "video", + "model": "grok-imagine-1.0-video", + "status": "in_progress", + "progress": 50, + "created_at": 1778855922 +} +``` + +**响应（生成成功）：** + +```json +{ + "id": "task_cIfhoNBQFqDcgxcpr969DQVXw0ApwGpH", + "object": "video", + "model": "grok-imagine-1.0-video", + "status": "completed", + "progress": 100, + "video_url": "https://example.com/video.mp4", + "created_at": 1778855922, + "completed_at": 1778855936 +} +``` + +**响应（生成失败）：** + +```json +{ + "id": "task_xxx", + "object": "video", + "model": "grok-imagine-1.0-video", + "status": "failed", + "progress": 0, + "error": { + "message": "Content policy violation", + "code": "generation_error" + } +} +``` + +**任务状态流转：** + +``` +queued → in_progress → completed + → failed +``` + +| 状态 | 含义 | 是否终态 | +|------|------|----------| +| queued | 任务排队中，等待处理 | 否 | +| in_progress | 视频正在生成中 | 否 | +| completed | 生成成功，视频 URL 在 `video_url` | 是 | +| failed | 生成失败，失败原因在 `error.message` | 是 | + +**轮询建议：** 每隔 10-15 秒查询一次状态，veo3.1-fast 通常 30-60 秒完成，veo3.1-pro 可能需要 2-5 分钟。 + +### 1.7 完整调用示例（Python） + +```python +import requests +import time + +BASE_URL = "http://192.129.209.36:3001/v1" +API_KEY = "your-api-key-here" + +headers = { + "Authorization": f"Bearer {API_KEY}", + "Content-Type": "application/json" +} + +def generate_video(prompt, model="veo3.1-fast", images=None, aspect_ratio=None, enhance_prompt=False, poll_interval=15, max_wait=600): + """ + 提交视频生成任务并等待完成。 + + Args: + prompt: 视频描述（英文效果更好） + model: 模型名称，默认 veo3.1-fast + images: 参考图片 URL 列表。1张=首帧，2张=首尾帧（需模型支持），3张=元素参考（需 components 模型） + aspect_ratio: 视频比例 "16:9" 或 "9:16" + enhance_prompt: 是否自动优化/翻译提示词 + poll_interval: 轮询间隔（秒），默认 15 秒 + max_wait: 最大等待时间（秒），默认 600 秒（10 分钟） + + Returns: + 成功时返回视频 URL，失败时返回 None + """ + body = {"model": model, "prompt": prompt} + if images: + body["images"] = images + if aspect_ratio: + body["aspect_ratio"] = aspect_ratio + if enhance_prompt: + body["enhance_prompt"] = True + + submit_resp = requests.post( + f"{BASE_URL}/videos", + headers=headers, + json=body + ) + submit_data = submit_resp.json() + + if "task_id" not in submit_data: + print(f"提交失败: {submit_data}") + return None + + task_id = submit_data["task_id"] + print(f"任务已提交，task_id: {task_id}") + + start_time = time.time() + while time.time() - start_time < max_wait: + time.sleep(poll_interval) + + poll_resp = requests.get( + f"{BASE_URL}/videos/{task_id}", + headers=headers + ) + poll_data = poll_resp.json() + status = poll_data.get("status", "unknown") + progress = poll_data.get("progress", 0) + print(f"状态: {status}, 进度: {progress}") + + if status == "completed": + video_url = poll_data.get("video_url", "") + print(f"视频生成成功: {video_url}") + return video_url + + elif status == "failed": + fail_reason = poll_data.get("error", {}).get("message", "未知原因") + print(f"视频生成失败: {fail_reason}") + return None + + print("超时，视频未在指定时间内完成") + return None + +# 文生视频 +video_url = generate_video("A golden retriever running on a beach at sunset") + +# 图生视频（首帧） +video_url = generate_video( + "The character starts walking forward", + model="veo3.1", + images=["https://example.com/first_frame.jpg"] +) + +# 图生视频（首尾帧） +video_url = generate_video( + "Smooth transition from sitting to standing", + model="veo3.1", + images=["https://example.com/sitting.jpg", "https://example.com/standing.jpg"] +) + +# 多图参考 +video_url = generate_video( + "A person wearing the outfit in front of the building", + model="veo3.1-components", + images=["https://example.com/person.jpg", "https://example.com/outfit.jpg", "https://example.com/building.jpg"] +) + +# 带中文提示词 + 自动翻译 +video_url = generate_video( + "一只金毛犬在日落的海滩上奔跑", + model="veo3.1-fast", + enhance_prompt=True +) + +# Grok 首尾帧 +video_url = generate_video( + "Create a smooth transition from the first frame to the last frame", + model="grok-imagine-1.0-video", + images=["https://example.com/start.png", "https://example.com/end.png"] +) +``` + +### 1.8 完整调用示例（cURL） + +```bash +#!/bin/bash +API_KEY="your-api-key-here" +BASE_URL="http://192.129.209.36:3001/v1" + +# 文生视频 +echo "提交视频生成任务..." +TASK_ID=$(curl -s "${BASE_URL}/videos" \ + -H "Authorization: Bearer ${API_KEY}" \ + -H "Content-Type: application/json" \ + -d '{"model":"veo3.1-fast","prompt":"A cat playing piano in a jazz bar"}' \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['task_id'])") + +echo "Task ID: ${TASK_ID}" + +# 轮询任务状态 +while true; do + sleep 15 + RESULT=$(curl -s "${BASE_URL}/videos/${TASK_ID}" \ + -H "Authorization: Bearer ${API_KEY}") + + STATUS=$(echo "$RESULT" | python3 -c "import sys,json; print(json.load(sys.stdin).get('status'))") + PROGRESS=$(echo "$RESULT" | python3 -c "import sys,json; print(json.load(sys.stdin).get('progress',''))") + echo "状态: ${STATUS}, 进度: ${PROGRESS}" + + if [ "$STATUS" = "completed" ]; then + VIDEO_URL=$(echo "$RESULT" | python3 -c "import sys,json; print(json.load(sys.stdin).get('video_url',''))") + echo "视频 URL: ${VIDEO_URL}" + break + elif [ "$STATUS" = "failed" ]; then + echo "生成失败" + break + fi +done +``` + +#### cURL 图生视频示例（首尾帧） + +```bash +curl -s "${BASE_URL}/videos" \ + -H "Authorization: Bearer ${API_KEY}" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "veo3.1", + "prompt": "Smooth transition from the first pose to the second pose", + "images": [ + "https://example.com/first_frame.jpg", + "https://example.com/last_frame.jpg" + ], + "aspect_ratio": "16:9" + }' +``` + +--- + +## 二、图片生成 + +图片生成推荐使用 OpenAI 兼容的 **Images** 接口调用，统一入口是 `/v1/images/generations`；图像编辑使用 `/v1/images/edits`。当前内部主用三类模型： + +- 下划线模型：`gemini_3.*_image_preview`，走 Apexer OpenAI 兼容通道。 +- 横线模型：`gemini-3.*-image-preview` 主路径为 Apexer（model_mapping 映射到下划线命名），漫小白第二兜底，ListenHub 垫底；`gpt-image-2` 优先走 xgapi 直出生图，漫小白第一兜底，带参考图首选漫小白 edits。 +- SiliconFlow 模型：`Qwen/Qwen-Image`、`baidu/ERNIE-Image-Turbo`、`Tongyi-MAI/Z-Image` 和 `Qwen/Qwen-Image-Edit-2509`，返回 `data[0].url`。 + +Chat Completions 形式仍保留兼容：部分上游会把图片 URL 放在 `choices[0].message.content` 的 Markdown 图片语法中返回。新接入和业务调用优先使用 Images 接口。 + +### 2.1 请求 + +``` +POST {Base URL}/images/generations +Content-Type: application/json +Authorization: Bearer +``` + +**请求体（JSON）：** + +```json +{ + "model": "gemini_3.1_flash_image_preview", + "prompt": "Generate an image of a cute cat wearing a tiny hat, studio lighting", + "n": 1, + "size": "1024x1024" +} +``` + +**请求参数：** + +| 参数名 | 类型 | 必填 | 说明 | +|--------|------|------|------| +| model | string | 是 | 图片生成模型名称，见下方模型列表 | +| prompt | string | 是 | 图片描述，建议用英文，描述越详细效果越好 | +| n | integer | 否 | 生成图片数量，默认 1 | +| size | string | 否 | 输出尺寸，常用 `1024x1024`；4K 模型建议使用模型默认高清能力 | +| response_format | string | 否 | OpenAI 兼容字段，支持情况取决于下游；调用方需要同时兼容 `data[0].b64_json` 和 `data[0].url` | +| image / images | string / array | 否 | 图像编辑或参考图。SiliconFlow 支持 URL、`data:image/...;base64,...`，最多映射到 `image`、`image2`、`image3` | +| extra_body | object | 否 | 下游扩展参数。SiliconFlow 支持 `seed`、`num_inference_steps`、`guidance_scale`、`cfg`、`negative_prompt`、`image_size`、`batch_size` 等 | +| extra_body.google.image_config.aspect_ratio | string | 否 | Apexer 兼容参数，例如 `1:1`、`16:9`、`9:16` | +| extra_body.google.image_config.image_size | string | 否 | Apexer 兼容参数，例如 `1K`、`4K` | + +**当前已通过统一入口验证的图片模型（更新至 2026-06-07）：** + +| 模型名 | 通道 | 单次价格 | 本次耗时 | 返回 | 建议场景 | +|--------|------|----------|----------|------|----------| +| `gemini_3.1_flash_image_preview` | Apexer OpenAI 兼容 | $0.25 | 约 29 秒 | `b64_json` | 首选快速生图 | +| `gemini_3.0_pro_image_preview` | Apexer OpenAI 兼容 | $0.3 | 约 58 秒 | `b64_json` | 高质量图片、产品图 | +| `gemini_3.1_flash_image_preview_4K` | Apexer OpenAI 兼容 | $0.3 | 约 65 秒 | `b64_json` | 快速高清输出 | +| `gemini_3.0_pro_image_preview_4K` | Apexer OpenAI 兼容 | $0.35 | 约 383 秒 | `b64_json` | 4K 高质量，耗时明显更长 | +| `gemini-3.1-flash-image-preview` | Apexer 主路径 / 漫小白兜底 | $0.2 | 约 22-93 秒 | `b64_json` | 横线命名快速生图 | +| `gemini-3-pro-image-preview` | Apexer 主路径 / 漫小白兜底 | $0.3 | 约 58-67 秒 | `b64_json` | 横线命名高质量生图 | +| `gpt-image-2` | xgapi 主路径 / 漫小白兜底 | $0.5 | 约 45-90 秒 | `url` | 直出生图；带参考图走 `/v1/images/edits`（漫小白，约 43 秒） | +| `gpt-image-2-1k` / `-2k` / `-4k` | 漫小白（manxiaobai） | $0.55 / $0.6 / $0.65 | 1k 实测约 138 秒 | `url` | 高分辨率档位，需按档位传对应尺寸 | +| `gpt-image-2(线路XF)` | xgapi-images | $0.3 | 48-50 秒 | `url` | 映射到 xgapi `gpt-image-2` | +| `gr-image-2` | xgapi-images | $0.3 | 46-55 秒 | `url` | 映射到 xgapi `gpt-image-2` | +| `nano-banana` | bltcy-images | $0.18 | 8-9 秒 | `url` | 快速生图 | +| `nano-banana-hd` | bltcy-images | $0.22 | 10-11 秒 | `url` | 高清生图 | +| `nano-banana-pro` | xgapi-images | $0.3 | 46-48 秒 | `url` | 映射到 xgapi `gpt-image-2` 兜底 | +| `baidu/ERNIE-Image-Turbo` | SiliconFlow | 按后台配置 | 20.89 秒 | `url` | 快速通用生图 | +| `Qwen/Qwen-Image` | SiliconFlow | 按后台配置 | 18.76 秒 | `url` | 通用高质量生图 | +| `Tongyi-MAI/Z-Image` | SiliconFlow | 按后台配置 | 12.20 秒 | `url` | 通义图像模型路径 | +| `Qwen/Qwen-Image-Edit-2509` | SiliconFlow | 按后台配置 | 24.60 秒 | `url` | 图像编辑、风格转换 | + +> **2026-06-02 远端验证方式**：使用远端测试 Key 直接请求 `POST http://192.129.209.36:3001/v1/images/generations`，上述 7 个非 SiliconFlow 图片模型均返回 HTTP 200，`data` 数组长度为 1。4K 响应体可能超过 20 MB，不要在日志或终端中直接打印完整 `b64_json`。 +> +> **2026-06-06 SiliconFlow 远端验证方式**：通过本服务统一入口验证 4 个 SiliconFlow 模型；`baidu/ERNIE-Image-Turbo`、`Qwen/Qwen-Image`、`Tongyi-MAI/Z-Image` 走 `/v1/images/generations`，`Qwen/Qwen-Image-Edit-2509` 走 `/v1/images/edits`，均返回 HTTP 200，`data` 数组长度为 1，结果在 `data[0].url`。本次实测耗时分别为 20.89 秒、18.76 秒、12.20 秒、24.60 秒。 +> +> **2026-06-07 xgapi 图片兜底验证方式**：修复部署后，通过公网统一入口连续 2 轮验证 `gpt-image-2`、`gpt-image-2(线路XF)`、`gr-image-2`、`nano-banana`、`nano-banana-hd`、`nano-banana-pro`，共 12 次请求全部 HTTP 200，均返回标准 `data[0].url`。其中 `gpt-image-2`、`gpt-image-2(线路XF)`、`gr-image-2`、`nano-banana-pro` 命中 `xgapi-images` 渠道，后三个别名映射到上游 `gpt-image-2`。 +> +> **2026-06-07 xgapi 比例与参考图兼容验证**：远端部署后复测 `gpt-image-2`，直接生图请求 `size=1792x1024` 且 prompt 不写比例，命中 `channel_id=14`，HTTP 200，46.33 秒，返回 PNG `1659x948`，比例约 `1.75`。服务端会把 `size` / `aspect_ratio` 推导出的比例自动追加到 xgapi 上游 prompt。带 `image` 参考图字段的同模型请求自动避开 xgapi，命中 ListenHub `channel_id=12`，HTTP 200，45.08 秒，返回 `b64_json` PNG `2048x2048`。 +> +> **2026-06-07 生产自测补充**：修剪 Apexer channel 6/7 的 `gpt-image-2` 后再次验证当前路径：`gpt-image-2` 直接生图命中 channel 14，46.51 秒，PNG `1659x948`；`gpt-image-2` 带 `image` 参考图命中 channel 12，93.63 秒，PNG `2048x2048`；`Qwen/Qwen-Image-Edit-2509` 图像编辑命中 channel 13，29.97 秒，返回 PNG；`grok-video-3` 视频任务命中 channel 11，轮询到 `completed`，`/content` 返回 `200 video/mp4`。 +> +> **2026-06-07 ListenHub 专项复测**：按 ListenHub 文档格式直连 `https://api.marswave.ai/openapi/v1/images/generation`，`provider=openai`、`model=gpt-image-2`、`imageConfig.aspectRatio=1:1`、`imageConfig.imageSize=1K` 连续 10 次全部 HTTP 200，均返回 `candidates[].content.parts[].inlineData`，平均 21.41 秒。通过本服务统一入口强制 channel 12 复测 `gpt-image-2` 直接生图 10 次全部 HTTP 200，均返回标准 `data[0].b64_json`，平均 40.13 秒；再用 `image` data URI 参考图强制 channel 12 复测 3 次全部 HTTP 200，平均 24.48 秒。同期普通 `poc_key` 日志仍出现 channel 12 的 `413 request entity too large`，说明 ListenHub 对大请求体/大参考图需要控制输入体积；本轮小图和标准请求未复现 504。追加大参考图实测：`2048x2048` JPEG 约 3.33 MB（JSON 请求体约 4.44 MB）成功，耗时 82.19 秒；`4096x4096` JPEG 约 13.32 MB（JSON 请求体约 17.75 MB）返回 `413 request entity too large`，耗时 9.5 秒。 +> +> **2026-06-07 ListenHub 优先级切换验证**：生产配置已调整为 ListenHub channel 12 priority `140`、xgapi channel 14 priority `130`。重启刷新 channel cache 后，普通入口 `model=gpt-image-2` 连续 10 次客户端请求全部 HTTP 200，均返回标准 `data[0].b64_json`，平均 25.08 秒；服务端成功日志均命中 channel 12。xgapi channel 14 仍保留在 `gpt-image-2` 能力表中，作为无参考图直出生图兜底。 +> +> **2026-06-09 上游余额失败路由验证**：使用本文档内部测试 Key 真实调用 `POST /v1/images/generations`、`model=gpt-image-2`。ListenHub channel 12 先返回 `400 Insufficient credits for Image generation`；服务端识别为上游 channel 余额/额度类错误后自动重试 xgapi channel 14，最终客户端收到 HTTP 200，标准 Images 响应 `data[0].url`，耗时 131.86 秒。该测试说明文档 Key 本服务侧额度可用；本服务侧额度与上游供应商账号余额是两套独立额度。 + +**SiliconFlow 图片平台：** + +SiliconFlow 已作为既有渠道类型 `SiliconFlow` / `type=40` 扩展图片能力，远端渠道配置如下： + +| 配置项 | 值 | +|--------|----| +| 渠道名称 | `siliconflow-images` | +| 渠道 ID | `13` | +| Base URL | `https://api.siliconflow.cn` | +| 上游接口 | `/v1/images/generations` | +| 对外生图入口 | `/v1/images/generations` | +| 对外编辑入口 | `/v1/images/edits` | +| 返回格式 | OpenAI Images 兼容，图片在 `data[0].url` | + +SiliconFlow 支持模型： + +| 模型名 | 类型 | 说明 | +|--------|------|------| +| `baidu/ERNIE-Image-Turbo` | 生图 | 文心图像快速模型 | +| `Qwen/Qwen-Image` | 生图 | Qwen 图片生成模型；常用 OpenAI 尺寸会自动映射到 SiliconFlow 推荐尺寸 | +| `Tongyi-MAI/Z-Image` | 生图 | 通义 Z-Image 图片生成模型 | +| `Qwen/Qwen-Image-Edit-2509` | 图像编辑 | SiliconFlow 使用 `/v1/images/generations` 上游接口接收 `image`、`image2`、`image3` | + +SiliconFlow 参数映射： + +| 上游 OpenAI 兼容参数 | SiliconFlow 参数 | +|----------------------|------------------| +| `model` | `model` | +| `prompt` | `prompt` | +| `n` | `batch_size` | +| `size` | `image_size`；`Qwen/Qwen-Image` 会将常见 OpenAI 尺寸映射到官方推荐尺寸 | +| `output_format` | `output_format` | +| `image` / `images` / multipart `image` | `image`、`image2`、`image3`，最多 3 张 | +| `extra_body.seed` | `seed` | +| `extra_body.num_inference_steps` | `num_inference_steps` | +| `extra_body.guidance_scale` | `guidance_scale` | +| `extra_body.cfg` | `cfg` | +| `extra_body.negative_prompt` | `negative_prompt` | + +SiliconFlow 生图示例： + +```bash +curl -s "http://192.129.209.36:3001/v1/images/generations" \ + -H "Authorization: Bearer your-api-key-here" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "Qwen/Qwen-Image", + "prompt": "A simple red square icon on a white background", + "size": "1024x1024", + "n": 1, + "extra_body": { + "seed": 1, + "cfg": 4, + "num_inference_steps": 20 + } + }' +``` + +SiliconFlow 图像编辑示例： + +```bash +curl -s "http://192.129.209.36:3001/v1/images/edits" \ + -H "Authorization: Bearer your-api-key-here" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "Qwen/Qwen-Image-Edit-2509", + "prompt": "Turn the image into a clean watercolor style while preserving the main subject", + "image": "https://example.com/input.png", + "n": 1, + "extra_body": { + "seed": 2, + "num_inference_steps": 20 + } + }' +``` + +**ListenHub 图片平台：** + +ListenHub 已作为独立渠道类型接入，后台创建渠道时使用： + +| 配置项 | 值 | +|--------|----| +| 渠道类型 | `ListenHub` / `type=59` | +| Base URL | `https://api.marswave.ai/openapi` | +| 上游接口 | `/v1/images/generation` | +| 对外入口 | `/v1/images/generations` | +| 返回格式 | OpenAI Images 兼容，图片在 `data[0].b64_json` | + +支持模型： + +| 模型名 | ListenHub provider | 说明 | +|--------|--------------------|------| +| gemini-3-pro-image-preview | google | 默认高质量图片模型 | +| gemini-3.1-flash-image-preview | google | 更快，支持额外长宽比 | +| gpt-image-2 | openai | OpenAI 图片模型，最多 4 张参考图 | + +ListenHub 验证状态： + +| 检查项 | 状态 | 说明 | +|--------|------|------| +| 本项目渠道适配 | 已接入 | 新增 `type=59`，对外走 `/v1/images/generations` | +| 远端统一入口 | 已跑通 | 2026-06-07 已将 `gpt-image-2` 切为 ListenHub 优先；普通入口 10 次客户端请求全部成功，均返回 `data[0].b64_json` | +| 返回格式 | OpenAI Images 兼容 | 当前返回 `data[0].b64_json`，不是 URL | +| 参考图限制 | 已验证 | 小图和约 4.44 MB JSON 请求体可成功；约 17.75 MB JSON 请求体返回 `413 request entity too large`，大图建议压缩或改用 URL | + +ListenHub 上游直连验证结果（2026-06-01，历史记录）： + +| provider | 模型 | 结果 | 耗时 | 返回 | +|----------|------|------|------|------| +| google | gemini-3-pro-image-preview | ✅ 成功 | 约 21 秒 | 1 张 PNG base64 | +| google | gemini-3.1-flash-image-preview | ✅ 成功 | 约 14 秒 | 1 张 PNG base64 | +| openai | gpt-image-2 | ✅ 成功 | 约 22 秒 | 1 张 PNG base64 | + +ListenHub 参数映射： + +| 上游 OpenAI 兼容参数 | ListenHub 参数 | +|----------------------|----------------| +| `prompt` | `prompt` | +| `model=gemini-3-pro-image-preview` | `provider=google`, `model=gemini-3-pro-image-preview` | +| `model=gemini-3.1-flash-image-preview` | `provider=google`, `model=gemini-3.1-flash-image-preview` | +| `model=gpt-image-2` | `provider=openai`, `model=gpt-image-2` | +| `size=1024x1024` | `imageConfig.aspectRatio=1:1` | +| `size=1792x1024` | `imageConfig.aspectRatio=16:9` | +| `size=1024x1792` | `imageConfig.aspectRatio=9:16` | +| `quality=1K/2K/4K` | `imageConfig.imageSize=1K/2K/4K` | +| `extra_body.listenhub.imageConfig` | 覆盖 `imageConfig` | +| `image` / `images` / `referenceImages` | `referenceImages`，支持 URL 和 `data:image/...;base64,...` | + +Apexer 统一入口真实探测结果（2026-06-02）： + +| 渠道 | 模型 | 端点 | 结果 | 耗时 | +|------|------|------|------|------| +| apexer-images-openai | `gemini_3.1_flash_image_preview` | `/v1/images/generations` | ✅ 成功，返回 `b64_json` | 约 29 秒 | +| apexer-images-openai | `gemini_3.0_pro_image_preview` | `/v1/images/generations` | ✅ 成功，返回 `b64_json` | 约 58 秒 | +| apexer-images-openai | `gemini_3.1_flash_image_preview_4K` | `/v1/images/generations` | ✅ 成功，返回 `b64_json` | 约 65 秒 | +| apexer-images-openai | `gemini_3.0_pro_image_preview_4K` | `/v1/images/generations` | ✅ 成功，返回 `b64_json` | 约 383 秒 | + +> 2026-06-07 生产日志显示 `gpt-image-2` 在 Apexer 图片 OpenAI/Gemini 渠道上会分别出现上游 distributor 503 和 `only imagen models are supported`，因此已从 channel 6/7 的模型列表和能力表移除。`gpt-image-2` 当前由 ListenHub 优先承载，xgapi 仅作为无参考图直出生图兜底。 + +暂不推荐模型： + +| 模型名 | 当前状态 | +|--------|----------| +| `gemini-2.5-flash-image` / `gemini-2.5-flash-image-preview` | 模型列表暴露，但本次未做统一入口真实生成；如需使用先单独验证 | + +ListenHub 调用示例： + +```bash +curl -s "http://192.129.209.36:3001/v1/images/generations" \ + -H "Authorization: Bearer your-api-key-here" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gemini-3.1-flash-image-preview", + "prompt": "A serene mountain landscape at sunset with a reflective lake", + "quality": "2K", + "size": "1792x1024" + }' +``` + +带参考图示例： + +```json +{ + "model": "gpt-image-2", + "prompt": "Transform this scene into a watercolor painting style", + "images": ["https://example.com/my-photo.jpg"], + "extra_body": { + "listenhub": { + "imageConfig": { + "aspectRatio": "1:1", + "imageSize": "2K" + } + } + } +} +``` + +**Apexer 图片接口兼容层：** + +上游可以继续使用统一入口，不需要感知 Apexer 的 Google 原生/OpenAI 兼容格式差异： + +| 上游入口 | 下游渠道 | 说明 | +|----------|----------|------| +| `/v1beta/models/{model}:generateContent` | `apexer-images-gemini` | Gemini 原生格式，支持 `generationConfig.imageConfig`，图生图使用 `inlineData` | +| `/v1/chat/completions` | `apexer-images-openai` | OpenAI 对话格式，最多 3 张 `image_url` 参考图 | +| `/v1/images/generations` | `apexer-images-openai` | OpenAI 图片格式，支持 1 张 `image` 参考图；`extra_body.google.image_config` 会透传给下游 | + +路由层会按端点类型选择通道：Gemini 原生入口固定选择 Gemini 类型通道；OpenAI 对话和图片入口优先选择 OpenAI 兼容通道，避免同名模型在不同下游格式之间随机分发。 + +参数映射规则： + +| 上游参数 | 下游处理 | +|----------|----------| +| `extra_body.google.image_config.aspect_ratio` | 透传到 Apexer OpenAI 兼容接口 | +| `extra_body.google.image_config.image_size` | 透传到 Apexer OpenAI 兼容接口 | +| `generationConfig.imageConfig.aspectRatio` | Gemini 原生格式原样透传 | +| `generationConfig.imageConfig.imageSize` | Gemini 原生格式原样透传 | +| `size` / `quality` / `output_format` / `background` | GPT Image 系列在 `/v1/images/generations` 中原样透传 | + +**Images 成功响应（HTTP 200）：** + +```json +{ + "created": 1770000000, + "data": [ + { + "b64_json": "iVBORw0KGgoAAAANSUhEUg..." + } + ] +} +``` + +SiliconFlow 等下游会返回 URL： + +```json +{ + "created": 1770000000, + "data": [ + { + "url": "https://example.com/generated-image.png" + } + ] +} +``` + +调用方应同时兼容 `data[0].b64_json` 和 `data[0].url` 两种格式。 + +如果使用 Chat Completions 兼容入口，成功响应通常如下： + +```json +{ + "id": "chatcmpl-xxx", + "object": "chat.completion", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Here is the image you requested:\n\n![image1](https://example.com/generated-image.png)" + }, + "finish_reason": "stop" + } + ], + "model": "nano-banana", + "usage": { + "prompt_tokens": 15, + "completion_tokens": 100, + "total_tokens": 115 + } +} +``` + +**图片结果提取方式：** Images 接口读取 `data[0].b64_json` 或 `data[0].url`。Chat Completions 兼容入口中，图片 URL 通常嵌入在 `choices[0].message.content`，格式为 Markdown 图片语法 `![image1](url)`，可通过正则表达式 `!\[.*?\]\((.*?)\)` 提取。 + +### 2.2 完整调用示例（Python） + +```python +import base64 +import requests +import re + +BASE_URL = "http://192.129.209.36:3001/v1" +API_KEY = "your-api-key-here" + +headers = { + "Authorization": f"Bearer {API_KEY}", + "Content-Type": "application/json" +} + +def generate_image(prompt, model="gemini_3.1_flash_image_preview"): + response = requests.post( + f"{BASE_URL}/images/generations", + headers=headers, + json={ + "model": model, + "prompt": prompt, + "n": 1, + "size": "1024x1024", + } + ) + + result = response.json() + + if "error" in result: + print(f"生成失败: {result['error']}") + return None + + if result.get("data"): + first = result["data"][0] + if first.get("b64_json"): + image_bytes = base64.b64decode(first["b64_json"]) + return {"type": "b64_json", "bytes": image_bytes} + if first.get("url"): + return {"type": "url", "url": first["url"]} + + # 兼容少数下游按 Chat Completions 格式返回 Markdown 图片 URL 的情况 + content = result.get("choices", [{}])[0].get("message", {}).get("content", "") + + urls = re.findall(r'!\[.*?\]\((.*?)\)', content) + if urls: + return {"type": "url", "url": urls[0]} + + url_pattern = r'(https?://[^\s\)]+\.(png|jpg|jpeg|webp))' + urls = re.findall(url_pattern, content) + if urls: + return {"type": "url", "url": urls[0][0]} + + print(f"未找到图片结果，原始响应: {str(result)[:300]}") + return None + +image_result = generate_image("A sunset over snow-capped mountains, oil painting style") +if image_result: + if image_result["type"] == "b64_json": + print(f"图片 base64 已解码，字节数: {len(image_result['bytes'])}") + else: + print(f"图片 URL: {image_result['url']}") +``` + +--- + +## 三、文本对话 + +文本对话使用标准 OpenAI Chat Completions 接口。 + +``` +POST {Base URL}/chat/completions +Content-Type: application/json +Authorization: Bearer +``` + +**请求体：** + +```json +{ + "model": "gemini-2.5-flash", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "你好，请介绍一下你自己"} + ], + "max_tokens": 100, + "temperature": 0.7 +} +``` + +**可用文本模型：** + +| 模型名 | 说明 | +|--------|------| +| gemini-2.5-flash | Gemini 2.5 Flash，快速文本对话 | + +响应格式与 OpenAI Chat Completions 完全一致。 + +--- + +## 四、列出可用模型 + +``` +GET {Base URL}/models +Authorization: Bearer +``` + +返回当前 API Key 可访问的所有模型列表，格式同 OpenAI Models API。 + +--- + +## 五、错误处理 + +### 5.1 常见错误码 + +| HTTP 状态码 | 错误信息 | 原因 | 解决方案 | +|-------------|----------|------|----------| +| 401 | Invalid authentication | API Key 无效 | 检查 Authorization Header | +| 403 | No available channel | 无可用渠道 | 检查模型名是否正确 | +| 429 | Rate limit exceeded | 请求频率过高 | 降低请求频率 | +| 500 | Internal server error | 服务器内部错误 | 稍后重试 | + +### 5.2 视频生成特殊错误 + +| 场景 | 原因 | 解决方案 | +|------|------|----------| +| 提交后 task_id 为空 | 上游中转站不可用 | 稍后重试或换模型 | +| 状态一直 QUEUED | 上游排队中 | 耐心等待，veo3.1-pro 可能排队较久 | +| 状态 FAILURE，`fail_reason=upstream returned unrecognized message` | 上游返回的状态字符串未在 `statusToTaskStatus` 中映射（旧版漏映射 `IN_PROGRESS` 已修复） | 检查 `relay/channel/task/openaivideo/provider.go:statusToTaskStatus` 是否覆盖了上游所有状态值 | +| 状态 FAILURE | 内容违规或上游错误 | 修改 prompt 或重试 | +| 图生视频 images 数量超限 | 不同模型对图片数量限制不同 | veo3.1 系列最多 2 张，components 最多 3 张，veo3-pro-frames 最多 1 张 | +| `veo_3_1-* / sora_2 model_not_found` | bltcy/xgapi 上游 distributor 在查找前会把 `.` 替换为 `_`，但注册表里没有对应条目 | 不要把 `veo3.1*` / `sora-2` 走 bltcy 主路径；通过 `model_mapping` 或 fallback 改走 Apexer | + +--- + +## 六、价格与上游采购价参考 + +### 对外定价 + +| 模型 | 类型 | 单次价格 | 说明 | +|------|------|----------|------| +| veo2 | 视频 | $0.3 | Veo2 基础版 | +| veo2-fast | 视频 | $0.3 | Veo2 快速版 | +| veo2-pro | 视频 | $0.6 | Veo2 高质量版 | +| veo2-fast-frames | 视频 | $0.3 | Veo2 首尾帧 | +| veo2-fast-components | 视频 | $0.3 | Veo2 多图参考 | +| veo3 | 视频 | $0.4 | Veo3 基础版，支持音频 | +| veo3-fast | 视频 | $0.3 | Veo3 快速版 | +| veo3-pro | 视频 | $1.5 | Veo3 高质量版 | +| veo3-pro-frames | 视频 | $1.5 | Veo3 图生视频 | +| veo3-fast-frames | 视频 | $0.3 | Veo3 快速图生视频 | +| veo3.1-fast | 视频 | $0.3 | 快速生成，性价比最高 | +| veo3.1 | 视频 | $0.4 | 标准质量，支持首尾帧 | +| veo3.1-pro | 视频 | $1.5 | 高质量，支持首尾帧 | +| veo3.1-pro-4k | 视频 | $15 | 4K 最高质量 | +| veo3.1-components | 视频 | $0.4 | 多图参考模式（1-3张） | +| veo3.1-fast-components | 视频 | $0.3 | 快速多图参考 | +| veo3.1-lite | 视频 | $0.6 | 暂不推荐；2026-05-24 创建失败 | +| veo3.1-lite-4k | 视频 | $0.65 | 暂不推荐；未完成真实生成验证 | +| veo3.1-fast-4k | 视频 | $1.5 | 快速 4K | +| veo3.1-4k | 视频 | $1.5 | 标准 4K | +| veo3.1-components-4k | 视频 | $1.5 | 多图参考 4K | +| veo3.1-fast-components-4k | 视频 | $1.5 | 快速多图参考 4K | +| nano-banana | 图片 | $0.18 | 快速生图 | +| nano-banana-hd | 图片 | $0.22 | 高清生图 | +| nano-banana-pro | 图片 | $0.3 | 专业生图 | +| gemini-2.5-flash-image-preview | 图片 | $0.14 | 最便宜 | +| gemini-2.5-flash-image | 图片 | $0.14 | 最便宜 | +| gemini-3-pro-image-preview | 图片 | $0.3 | 最高质量 | +| gemini_3.0_pro_image_preview | 图片 | $0.3 | Apexer Pro | +| gemini_3.0_pro_image_preview_4K | 图片 | $0.35 | Apexer Pro 4K | +| gemini_3.1_flash_image_preview | 图片 | $0.25 | Apexer Flash | +| gemini_3.1_flash_image_preview_4K | 图片 | $0.3 | Apexer Flash 4K | +| gpt-image-2 | 图片 | $0.5 | OpenAI 图片模型，xgapi 主路径 + 漫小白兜底已验证；参考图走漫小白 edits | +| gpt-image-2-1k | 图片 | $0.55 | 漫小白 1K 档位，已验证 | +| gpt-image-2-2k | 图片 | $0.6 | 漫小白 2K 档位 | +| gpt-image-2-4k | 图片 | $0.65 | 漫小白 4K 档位 | +| grok-imagine-video | 视频 | $0.25 | 漫小白 Grok 1.0（10s），已验证 | +| grok-imagine-video-1.5-preview | 视频 | $0.45 | 漫小白 Grok 1.5，必须参考图，预上传协议待验证 | +| baidu/ERNIE-Image-Turbo | 图片 | 按后台配置 | SiliconFlow 快速通用生图 | +| Qwen/Qwen-Image | 图片 | 按后台配置 | SiliconFlow Qwen 生图 | +| Tongyi-MAI/Z-Image | 图片 | 按后台配置 | SiliconFlow 通义 Z-Image | +| Qwen/Qwen-Image-Edit-2509 | 图片 | 按后台配置 | SiliconFlow Qwen 图像编辑 | +| gemini-2.5-flash | 文本 | 按 token 计费 | 快速对话 | + +### 上游采购价（内部参考） + +| 模型 | 上游价格 | 上游来源 | +|------|----------|----------| +| veo2 | ≈$0.2 | Apexer | +| veo2-fast | ≈$0.2 | Apexer | +| veo2-pro | ≈$0.5 | Apexer | +| veo3 | ≈$0.3 | Apexer | +| veo3-fast | ≈$0.2 | Apexer | +| veo3-pro | ≈$1 | Apexer | +| veo3.1 | ≈$0.3 | Apexer | +| veo3.1-pro | ≈$1 | Apexer | +| veo3.1-fast | $0.2 | bltcy.ai | +| veo3.1 | $0.3 | bltcy.ai | +| veo3.1-pro | $1 | bltcy.ai | +| veo3.1-pro-4k | $13 | bltcy.ai | +| veo3.1-components | $0.3 | bltcy.ai | +| veo3.1-fast-components | $0.2 | bltcy.ai | +| veo3.1-lite | $0.5 | xgapi.top（当前创建失败，不建议采购/推荐） | +| veo3.1-fast-4k | $1.5 | bltcy.ai | +| veo3.1-4k | $1.5 | bltcy.ai | +| veo3.1-components-4k | $1.5 | bltcy.ai | +| veo3-pro-frames | ≈$1 | bltcy.ai | +| veo3-fast-frames | ≈$0.2 | bltcy.ai | +| veo2-fast-frames | ≈$0.2 | bltcy.ai | +| nano-banana | $0.08 | bltcy.ai | +| nano-banana-hd | $0.12 | bltcy.ai | +| nano-banana-pro | $0.2 | bltcy.ai | +| gemini-2.5-flash-image | $0.04 | bltcy.ai | +| gemini-3-pro-image-preview | $0.2 | bltcy.ai | +| gemini_3.0_pro_image_preview | $0.18 | Apexer | +| gemini_3.0_pro_image_preview_4K | $0.25 | Apexer | +| gemini_3.1_flash_image_preview | $0.15 | Apexer | +| gemini_3.1_flash_image_preview_4K | $0.2 | Apexer | +| baidu/ERNIE-Image-Turbo | 按 SiliconFlow 账号计费 | SiliconFlow | +| Qwen/Qwen-Image | 按 SiliconFlow 账号计费 | SiliconFlow | +| Tongyi-MAI/Z-Image | 按 SiliconFlow 账号计费 | SiliconFlow | +| Qwen/Qwen-Image-Edit-2509 | 按 SiliconFlow 账号计费 | SiliconFlow | +| gpt-image-2（漫小白线路） | $0.03 | 漫小白 manxiaobai | +| gpt-image-2-1k / -2k / -4k | $0.05 / $0.06 / $0.07 | 漫小白 manxiaobai | +| gemini-3-pro-image-preview（漫小白线路） | $0.125 | 漫小白 manxiaobai | +| gemini-3.1-flash-image-preview（漫小白线路） | $0.1 | 漫小白 manxiaobai | +| grok-imagine-video | $0.2 | 漫小白 manxiaobai | +| grok-imagine-video-1.5-preview | $0.35 | 漫小白 manxiaobai | + +### 已对接平台 + +| 平台 | Base URL 关键词 | 优先级 | 支持模型 | 特点 | +|------|----------------|--------|----------|------| +| bltcy.ai / ablai.top | 默认（无匹配时） | 100（最高） | MiniMax-Hailuo-02/2.3*, doubao-seedance-*, wan*, 生图模型（注：veo3.1*/sora-2 受上游 BUG 影响不可用） | 统一格式接口，支持首尾帧、多图参考 | +| www.937qq.cn | 937qq / qilin | 80（Grok 专用） | grok-imagine-1.0-video, grok-imagine-1.0-video-20s, grok-imagine-1.0-video-30s | 麒麟 API，xAI Grok 视频专用；已验证 JSON 直传、多参数、1 张参考图、2 张首尾帧；新版插件支持 7 张参考图和 20/30 秒长时长模型 | +| open.hongniaoai.com | xb-sora2 / hongniao | 90（Sora2 主路径） | 推荐 `xb-sora2`；其他线路模型需单独验证 | Hongniao AI 视频平台，使用 `X-API-Key`、`/videos/generate`、`/videos/{task_id}`；2026-05-24 真实验证 `xb-sora2` 完成并可下载 | +| api.marswave.ai / ListenHub | listenhub | 140（图片主路径） | `gemini-3-pro-image-preview`、`gemini-3.1-flash-image-preview`、`gpt-image-2` | ListenHub 图片平台，2026-06-07 专项复测和优先级切换验证均通过；`gpt-image-2` 当前直接生图优先命中该渠道，返回 `data[0].b64_json` | +| xgapi.top | xgapi-images | 130（图片兜底） | `gpt-image-2`、`gpt-image-2(线路XF)`、`gr-image-2`、`nano-banana-pro` | xgapi 图片平台，作为 `gpt-image-2` 无参考图兜底以及历史别名主路径；直接生图会把比例自动补到上游 prompt | +| api.siliconflow.cn / SiliconFlow | siliconflow-images | 0（默认） | `baidu/ERNIE-Image-Turbo`、`Qwen/Qwen-Image`、`Tongyi-MAI/Z-Image`、`Qwen/Qwen-Image-Edit-2509` | SiliconFlow 图片平台，2026-06-06 已通过本服务 `/v1/images/generations` 和 `/v1/images/edits` 验证，返回 `data[0].url` | +| api.lk888.ai | lk888 / AI聚合站 | 35（Grok 线路） | 推荐 `grok-video-3` | AI 聚合站媒体生成平台，使用 Bearer Token、`/v1/media/generate`、`/v1/skills/task-status`；2026-05-24 真实验证 `grok-video-3` 完成并可下载 | +| www.aiapexers.com | apexer | 50（第二） | 视频：veo3.1_*；图片：gemini_3.*_image_preview | Apexer new-api 实例，视频和图片均已按统一入口适配 | +| xgapi.top | xgapi | 10（兜底） | `veo3.1-lite`, `sora-2` | 当前不可作为主路径；2026-05-24 `veo3.1-lite` 创建失败 | +| runway-api | runway | 暂不启用 | seedance/gen4/wan/kling/happyhorse 系列 | 当前暂不可用，不推荐给上游 | + +> **路由实务（2026-05-28 验证）**: +> - `veo3.1-fast` 请求 → Apexer/Veo 链路真实生成完成，`/content` 返回 `200 video/mp4` ✅ +> - `xb-sora2` 请求 → Hongniao（90）真实生成完成，`/content` 返回 `200 video/mp4` ✅ +> - `ss-sora-2` 请求 → Hongniao（90）真实生成完成，`/content` 返回 `200 video/mp4` ✅ +> - `veo3.1-4k` 请求 → Apexer/Veo 4K 链路真实生成完成，`/content` 返回 `200 video/mp4` ✅ +> - `grok-imagine-1.0-video` → 937qq / Qilin（80）真实生成完成，注意仅支持 `720x1280`/`1280x720`/`1024x1024`/`1024x1792`/`1792x1024` ✅ +> - `grok-video-3` → AI 聚合站 / LK888（35）今天上游返回"参数验证失败"，2026-05-24 曾可用，疑似上游临时问题 ⚠️ +> - `je-grok` → Hongniao 今天上游返回 429（限流），路由正常但高峰期不可用 ⚠️ +> - `openai-sora-2` 当前不要推荐给上游：真实创建失败，兼容层 duration 映射仍需修复 ⚠️ +> - `sora-2(线路BF)` / `grok-video-3(线路W)` / `全能视频2.0` 虽出现在模型列表，但真实创建失败 ⚠️ +> - xgapi 与 Runway 暂不在主路径上，不推荐给上游 + +### 渠道优先级与自动故障转移 + +系统内置了渠道优先级和自动故障转移机制，上游调用方无需感知下游中转站的差异或故障： + +**优先级规则：** +1. 请求首先路由到优先级最高的可用渠道（如 bltcy, priority=100） +2. 如果该渠道请求失败（5xx、429 等可重试错误），自动降级到下一优先级渠道（如 Apexer, priority=50） +3. 如果所有渠道都失败，返回错误 + +**自动故障转移配置：** + +| 配置项 | 当前值 | 说明 | +|--------|--------|------| +| RetryTimes | 2 | 失败后最多重试 2 次（覆盖 3 个优先级层级） | +| AutomaticDisableChannelEnabled | true | 渠道持续失败时自动禁用 | +| AutomaticEnableChannelEnabled | true | 被禁用的渠道恢复后自动启用 | + +**故障转移覆盖模型：** + +以下模型在多个渠道注册，支持自动故障转移： + +| 模型 | 主渠道（优先级 100） | 备用渠道（优先级 50） | +|------|---------------------|---------------------| +| veo3.1-fast | bltcy-veo | apexer-veo | +| veo3.1 | bltcy-veo | apexer-veo | +| veo3.1-pro | bltcy-veo | apexer-veo | +| veo3.1-fast-4k | bltcy-veo | apexer-veo | +| veo3.1-4k | bltcy-veo | apexer-veo | +| veo3.1-pro-4k | bltcy-veo | apexer-veo | +| veo3.1-fast-components | bltcy-veo | apexer-veo | +| veo3.1-components | bltcy-veo | apexer-veo | +| veo3.1-fast-components-4k | bltcy-veo | apexer-veo | +| veo3.1-components-4k | bltcy-veo | apexer-veo | + +以下模型仅在一个渠道注册，无故障转移： + +| 模型 | 唯一渠道 | +|------|----------| +| veo3.1-lite | xgapi-veo（当前创建失败，不建议上游调用） | +| grok-imagine-1.0-video | qilin-grok-video | +| grok-imagine-1.0-video-20s | qilin-grok-video | +| grok-imagine-1.0-video-30s | qilin-grok-video | + +> **扩展提示**：要增加故障转移覆盖的模型，需要在多个渠道的模型列表中注册同一模型，并配置正确的 model_mapping（模型名映射）。 + +**模型名映射（model_mapping）：** + +不同中转站使用不同的模型命名约定。系统通过渠道的 `model_mapping` 字段自动转换： + +| 我们的模型名 | Apexer OpenAI 视频格式模型名 | +|-------------|-----------------| +| veo3.1 | veo3.1_relaxed | +| veo3.1-fast | veo3.1_fast | +| veo3.1-pro | veo3.1_pro | +| veo3.1-4k | veo3.1_relaxed_4k | +| veo3.1-fast-4k | veo3.1_fast_4k | +| veo3.1-pro-4k | veo3.1_pro_4k | +| veo3.1-components | veo3.1_relaxed + `type=3` | +| veo3.1-fast-components | veo3.1_fast + `type=3` | +| veo3.1-components-4k | veo3.1_relaxed_4k + `type=3` | +| veo3.1-fast-components-4k | veo3.1_fast_4k + `type=3` | + +bltcy 使用与系统相同的命名，无需映射。 + +### 定价策略 + +- 视频生成：在采购价基础上加 $0.1/次 +- 超过 $1 的模型：按采购价 ×1.5 定价 +- $13 以上的模型：按 $15 定价 +- 图片生成：在采购价基础上加 $0.1/次 + +--- + +## 七、视频生成架构分析 + +### 7.1 整体架构 + +视频生成采用 **Provider 模式**，将不同中转站的差异封装在 Provider 接口背后，对上游调用方完全透明： + +``` +上游调用方 + │ + ▼ +┌──────────────────────────────────────────────────┐ +│ 统一 API 入口 (POST /v1/videos 或 /v1/video/generations)│ +│ 统一查询入口 (GET /v1/videos/{id} 或 /v1/video/generations/{id})│ +└──────────────┬───────────────────────────────────┘ + │ + ▼ +┌──────────────────────────────────────────────────┐ +│ TaskAdaptor (relay/channel/task/openaivideo/) │ +│ ┌─────────────────────────────────────────────┐ │ +│ │ provider 接口 │ │ +│ │ ├─ submitURL() 提交任务 URL │ │ +│ │ ├─ queryURL() 查询任务 URL │ │ +│ │ ├─ parseSubmitResponse() 解析提交响应 │ │ +│ │ ├─ parseQueryResponse() 解析查询响应 │ │ +│ │ ├─ buildSubmitResponseBody() 构建统一响应 │ │ +│ │ ├─ needsMultipart() 是否需要 multipart │ │ +│ │ ├─ mapModelForImages() 模型名自动映射 │ │ +│ │ └─ normalizeRequest() 平台参数归一化 │ │ +│ └─────────────────────────────────────────────┘ │ +│ ┌──────┐ ┌──────────┐ ┌──────┐ ┌────────┐ │ +│ │bltcy │ │Apexer │ │xgapi │ │newapi │ │ +│ └──────┘ └──────────┘ └──────┘ └────────┘ │ +└──────────────────────────────────────────────────┘ + │ + ▼ +┌──────────────────────────────────────────────────┐ +│ 渠道选择 + 优先级 + 自动重试 + 自动禁用/恢复 │ +│ (service/channel_select.go + controller/relay.go)│ +└──────────────────────────────────────────────────┘ +``` + +### 7.2 Provider 路由机制 + +Provider 通过 `getProviderByBaseURL(baseURL)` 自动选择，匹配规则： + +| Base URL 包含关键词 | 选择的 Provider | 说明 | +|---------------------|----------------|------| +| `xgapi` | xgapiProvider | 星光站 | +| `937qq` / `qilin` | qilinProvider | 麒麟 API / Grok 视频专用 | +| `apexer` | apexerapiProvider | Apexer 站 | +| `newapi` | newapiProvider | 通用 new-api 实例 | +| 其他（默认） | bltcyProvider | 柏拉图站 | + +**设计原则**：Provider 在 `TaskAdaptor.Init()` 阶段一次性确定，后续提交、查询、解析全部使用同一个 Provider，避免自动检测带来的路由错误。 + +### 7.3 各平台能力对比 + +| 能力 | bltcy.ai | www.937qq.cn | www.aiapexers.com | xgapi.top | 通用 new-api | +|------|----------|--------------|---------------|-----------|-------------| +| 提交端点 | `/v2/videos/generations` | `/v1/videos` | `/v1/videos` | `/v1/videos` | `/v1/video/generations` | +| 查询端点 | `/v2/videos/generations/{id}` | `/v1/videos/{id}` | `/v1/videos/{id}` | `/v1/videos/{id}` | `/v1/video/generations/{id}` | +| 需要 Multipart | ❌ | ❌ | ❌ | ✅ | ✅ | +| 模型名映射 | frames 自动映射 | 原样 | 横线→下划线 + type 自动推断 | 原样 | 原样 | +| 提交响应格式 | `{task_id}` | `{id}` | `{id}` | `{id, object, ...}` | `{id, task_id, ...}` | +| 查询响应格式 | `{data: {output}}` | `{url}` / `{video_url}` | `{video_url}` | `{video_url}` | `{status, progress}` | +| 文生视频 | ✅ | ✅（Grok） | ✅ | ✅ | ✅ | +| 首帧图生视频 | ✅ | ✅（2026-05-15 验证） | ✅ | ✅ | ✅ | +| 首尾帧图生视频 | ✅ | ✅（2026-05-15 验证） | ✅（自动 `type=2`） | ❓ | ⚠️ 需验证 | +| 多图 Components | ✅ | ⚠️ 已验证 1-2 张；3 张未验证 | ✅（自动 `type=3`，pro 系列不支持） | ❓ | ❓ | +| sora-2 | ✅ | ❌ | ❓ | ✅ | ❓ | + +> ✅ 已验证支持 | ❓ 未验证 | ⚠️ 需验证 | ❌ 不支持 + +### 7.4 上游屏蔽感知机制 + +系统在多个层面屏蔽了下游中转站的差异，上游调用方只需使用统一的 API： + +**1. 统一 API 格式** +- 上游只看到 OpenAI Video 格式：`POST /v1/videos`（兼容 `POST /v1/video/generations`）+ `GET /v1/videos/{id}`（兼容 `GET /v1/video/generations/{id}`） +- 不同中转站的端点差异（`/v2/` vs `/v1/`、`/videos` vs `/video/generations`）完全透明 + +**2. 统一模型名** +- 上游使用标准模型名（如 `veo3.1-fast`），系统自动映射到各中转站的实际模型名 +- 映射分两层： + - **Provider 层**：`mapModelForImages()` 和 `normalizeRequest()` 处理 images 相关映射、下游特殊字段（如 bltcy 的 frames 映射、Apexer 的下划线转换和 `type=1/2/3` 推断） + - **Channel 层**：`model_mapping` 处理平台间命名差异（如 `veo3.1` → `veo3.1_relaxed`） + +**3. 统一响应格式** +- 提交响应统一返回 `{id, task_id}` 格式 +- 查询响应统一转换为 `TaskInfo` 结构（status, url, progress, reason） +- 上游无需关心下游是 `{data: {output}}` 还是 `{video_url}` 格式 + +**4. 统一状态码** +- 各平台的状态字符串（`SUCCESS`/`completed`/`succeed`/`NOT_START`/`queued` 等）统一映射为 4 种内部状态：QUEUED / IN_PROGRESS / SUCCESS / FAILURE + +### 7.5 自动重试机制 + +系统在两个阶段提供自动重试： + +**阶段一：任务提交时（同步重试）** + +``` +请求 → 选择最高优先级渠道 → 提交失败？ + → shouldRetryTaskRelay() 判断是否可重试 + → 选择下一优先级渠道 → 再次提交 + → 最多重试 RetryTimes 次 +``` + +可重试的条件： +- 5xx 服务器错误（超时除外） +- 429 限流 +- 307 重定向 +- 其他非 2xx/400/408 错误 + +不可重试的条件： +- 400 Bad Request（请求本身有问题） +- 408 Request Timeout（超时不重试） +- 2xx 成功 +- LocalError（本地校验错误） + +**阶段二：任务轮询时（异步容错）** + +``` +定时轮询 → FetchTask() 获取上游状态 + → 首先尝试 dto.TaskResponse[model.Task] 格式解析（new-api 标准格式） + → 失败则使用 Provider.parseQueryResponse() 解析（平台特定格式） + → 更新任务状态 +``` + +**阶段三：渠道自动禁用/恢复** + +``` +渠道连续失败 → processChannelError() → ShouldDisableChannel() + → 自动禁用该渠道（AutoBan=true 时） + → 后续请求自动跳过该渠道 + +定时检查 → AutomaticEnableChannelEnabled=true + → 被禁用渠道恢复后自动重新启用 +``` + +### 7.6 接入新平台指南 + +要接入一个新的中转站，需要以下步骤： + +**步骤 1：创建 Provider 文件** + +在 `relay/channel/task/openaivideo/` 目录下创建新文件，如 `newstation.go`： + +```go +package openaivideo + +type newstationProvider struct{} + +func (p *newstationProvider) submitURL(baseURL string) string { + return baseURL + "/v1/video/generations" +} + +func (p *newstationProvider) queryURL(baseURL, taskID string) string { + return baseURL + "/v1/videos/" + taskID +} + +func (p *newstationProvider) parseSubmitResponse(body []byte) (string, error) { + // 解析提交响应，返回上游 task ID +} + +func (p *newstationProvider) parseQueryResponse(body []byte) (*relaycommon.TaskInfo, error) { + // 解析查询响应，返回 TaskInfo +} + +func (p *newstationProvider) buildSubmitResponseBody(info *relaycommon.RelayInfo, upstreamTaskID string) any { + return map[string]any{ + "id": info.PublicTaskID, + "task_id": info.PublicTaskID, + } +} + +func (p *newstationProvider) needsMultipart() bool { return false } + +func (p *newstationProvider) mapModelForImages(model string, hasImages bool) string { + return model // 或添加平台特定的模型名映射逻辑 +} +``` + +**步骤 2：注册 Provider** + +在 `provider.go` 的 `getProviderByBaseURL()` 和 `getProvider()` 中添加关键词匹配： + +```go +case containsAny(baseURL, "newstation"): + return &newstationProvider{} +``` + +**步骤 3：配置渠道** + +在管理后台或数据库中添加渠道： +- `type` = 58 (ChannelTypeOpenAIVideo) +- `base_url` = 新平台的 API 地址 +- `priority` = 优先级数值 +- `models` = 支持的模型列表 +- `model_mapping` = 模型名映射（如需要） +- `auto_ban` = 1（启用自动禁用） + +**步骤 4：验证** + +1. 提交测试请求，确认任务提交成功 +2. 查询任务状态，确认轮询正常 +3. 模拟主渠道故障，确认自动故障转移 +4. 确认模型名映射正确 + +### 7.7 当前架构的局限与改进方向 + +**已解决的问题：** + +| 问题 | 状态 | 解决方案 | +|------|------|----------| +| parseQueryResponseAuto 字段碰撞导致路由错误 | ✅ 已修复 | 改用 Init 时确定的 Provider 直接解析 | +| getProvider 缺少 apexerapi 匹配 | ✅ 已修复 | 添加 apexer 关键词匹配 | +| getProviderByBaseURL 缺少 newapi 匹配 | ✅ 已修复 | 添加 newapi 关键词匹配 | + +**当前局限：** + +| 局限 | 影响 | 改进方向 | +|------|------|----------| +| Provider 路由依赖 baseURL 关键词匹配 | 如果两个平台 baseURL 相似可能误匹配 | 改为渠道配置字段指定 Provider 名 | +| 模型名映射分散在 Provider 和 Channel 两层 | 维护成本高，需要同时修改两处 | 统一由 Channel 的 model_mapping 处理 | +| `/v1/models` 会暴露部分下游线路模型 | 上游可能误以为 `sora-2(线路BF)`、`grok-video-3(线路W)` 等都能直接创建任务 | 模型列表按 OpenAPI 可用性过滤，或增加可用性标记 | +| Sora 兼容别名 duration 归一化异常 | `openai-sora-2` 请求 8 秒仍会被转成下游不接受的 10 秒 | 修复 `xb_sora` duration 映射；修复前上游直接使用 `xb-sora2` | +| Runway 私有适配器暂不可用 | `seedance-2`、`gen4`、`wan`、`kling` 等模型不应给上游推荐 | 从公开模型列表隐藏或禁用对应渠道 | +| 首尾帧/Components 等高级功能未在所有平台验证 | 部分平台可能不支持但未明确拒绝 | 添加平台能力声明，请求前校验 | +| 轮询阶段无重试 | 如果查询请求失败，只能等下一轮 | 添加查询失败重试机制 | +| 无主动健康检查 | 只有请求失败时才发现渠道不可用 | 添加定时健康检查探针 | + +**扩展性评估：** + +- ✅ 接入新平台：只需创建 Provider 文件 + 注册关键词 + 配置渠道，无需修改核心逻辑 +- ✅ 新增模型：在 `constants.go` 的 ModelList 添加 + 在 `model_ratio.go` 添加定价 + 在渠道中注册 +- ✅ 新增能力（如音频生成）：参考视频生成的 Provider 模式，创建新的 ChannelType 和 Adaptor +- ⚠️ 跨平台能力差异：当前没有平台能力声明机制，无法在请求前判断某平台是否支持特定功能 diff --git a/docs/channel-balance-query.md b/docs/channel-balance-query.md new file mode 100644 index 000000000000..149f7434ebe1 --- /dev/null +++ b/docs/channel-balance-query.md @@ -0,0 +1,201 @@ +# 下游平台余额查询 — 调研结论与统一查询设计 + +> 调研日期：2026-06-13。目标：把"每个下游平台还剩多少余额"变成网关里一处可看、可定时刷新、可告警的能力。 +> 本文基于对全部真实上游的**逐个实测**（用生产渠道的真实 key 调用，非猜测）。 + +## 一、关键背景：余额查询按"上游"路由，不是按渠道类型 + +现有 `controller/channel-billing.go` 的余额查询用 `switch channel.Type` 分发。但本网关真实对接的下游是一批**中转站**，它们在渠道表里几乎都是同几种类型（type 1 OpenAI / 58 OpenAI Video / 59 ListenHub / 24 Gemini / 40 SiliconFlow）。`channel.Type` 无法区分 bltcy / xgapi / manxiaobai（都是 type 1/58）。 + +**因此余额查询必须按 `base_url` 或 `channel.Other` hint 路由**，与视频 provider 模式（`relay/channel/task/openaivideo/provider.go` 的 `getProviderByHint`）完全一致。 + +去重：同一上游账号常挂多个渠道（bltcy 3 个、apexer 3 个、manxiaobai 3 个共用同一 key），按 `(base_url, key)` 查一次即可，结果映射回各渠道。 + +## 二、实测结论：三档可查性 + +### A 档 — 纯 API key 可查"真实剩余余额"（用户真正想要的） + +| 上游 | 渠道 | 接口 | 认证 | 余额字段 | 实测值 | +|------|------|------|------|----------|--------| +| **lk888**（api.lk888.ai/api） | 11 | `GET {base}/v1/skills/balance` | `Bearer ` | `balance`（单位 `算力`）+ `api_key_quota.used` | 剩 8.27 算力 | +| **siliconflow** | 13 | `GET /v1/user/info` | `Bearer ` | `data.totalBalance` | 已实现 | +| **listenhub/marswave**（api.marswave.ai/openapi） | 12 | `GET /openapi/v1/user/subscription` | `Bearer ` | `data.totalAvailableCredits`（credits）+ 月度/永久/限时分项 + `subscriptionExpiresAt` | 剩 5 credits | + +> siliconflow 已在 `updateChannelSiliconFlowBalance()` 实现；lk888、listenhub 需新增。 +> ListenHub 文档：https://listenhub.ai/docs/en/openapi/api-reference/subscription + +### B 档 — new-api 套壳站：key 是无限额度 token，**只能查累计消费，查不到钱包余额** + +涉及：**bltcy**（ch1/2/3）、**apexer**（ch4/6/7）、**xgapi**（ch5/14）、**qilin/937qq**（ch8）、**manxiaobai**（ch15/16/17）。 + +这些站发给我们的渠道 key 都是 `unlimited_quota: true` 的令牌，导致： + +| 接口 | 返回 | 能用吗 | +|------|------|--------| +| `GET /v1/dashboard/billing/usage` | `{"total_usage": N}`，N 单位 0.01 USD（÷100=USD）= **累计消费额** | ✅ 唯一有效信息 | +| `GET /v1/dashboard/billing/subscription` | `soft/hard_limit_usd = 100000000`（无限额度哨兵，假值） | ❌ 无意义 | +| `GET /api/usage/token/` | `total_granted:0`、`total_available` 为负的已用量、`unlimited_quota:true` | ❌ 无钱包余额 | + +实测累计消费：bltcy ≈ $9.29、apexer ≈ $6.02、xgapi ≈ $9.34、qilin ≈ $40.90、manxiaobai 已用 $9.58。 + +**要拿真实钱包余额，必须走登录态**（已用 manxiaobai 账密验证通）： + +``` +POST {base}/api/user/login {"username","password"} → 返回 user.id + 写入 session cookie +GET {base}/api/user/self 带 cookie + 头 New-Api-User: → data.quota（÷500000 = USD）= 真实余额 +``` + +> 仅 manxiaobai 在 `deployment.local.md` 里存了账密。bltcy/apexer/xgapi/qilin 目前只有 API key，没有控制台账密；要查它们的真实余额需要补登录凭据（账密，或在各站"个人设置"里生成的"系统访问令牌"）。 + +### C 档 — 无 key 接口，只能登录 web 控制台 + +| 上游 | 渠道 | 说明 | +|------|------|------| +| **hongniao/红鸟**（open.hongniaoai.com） | 9 | Express 风格中转，`/api/*` 余额端点全 404，`/user/info` `/billing` 直接返回前端 SPA；余额只在 web 控制台可见 | +| **runway** | 10 | `127.0.0.1:8787` 本机适配器，非付费上游，N/A | + +## 三、统一查询设计 + +### 3.1 余额 provider 注册表（替换 type switch） + +新增 `controller/channel-billing.go` 之外的 `relay/channel/balance/`（或在 billing 文件内）一个 provider 注册表，**与视频 provider 同构**： + +```go +type BalanceResult struct { + Kind string // "balance" 真实余额 | "spend_only" 仅累计消费 | "console_only" 仅控制台 + Remaining float64 // 剩余（Kind=balance 时有效） + Used float64 // 累计消费（可选，所有档都尽量填） + Unit string // "USD" | "CNY" | "算力" | "credits" + ExpiresAt int64 // 订阅/到期（listenhub 有） + Raw string // 原始响应留档，便于排查 +} + +type BalanceQuerier interface { + Name() string + Match(ch *model.Channel) bool // base_url / channel.Other hint + Query(c *gin.Context, ch *model.Channel, cli *http.Client) (*BalanceResult, error) +} +``` + +注册的 provider： +1. `newapiSpendProvider` — bltcy/apexer/xgapi/qilin/manxiaobai 的 key 模式：`/v1/dashboard/billing/usage` → `Used`，`Kind=spend_only`。 +2. `newapiConsoleProvider` —（可选，需账密/系统令牌）登录 → `/api/user/self` → `Remaining`，`Kind=balance`。命中条件：渠道 `setting.balance` 里配置了登录凭据。 +3. `lk888Provider` — `/v1/skills/balance` → `Remaining`（算力）+ `Used`。 +4. `listenhubProvider` — `/openapi/v1/user/subscription` → `Remaining`（credits）+ `ExpiresAt`。 +5. `siliconflowProvider` — 复用现有实现。 +6. hongniao → 不注册 provider，前端显示"仅控制台可查"；runway → 跳过。 + +分发器：先按 provider `Match` 命中走专用逻辑，未命中再退回现有 OpenAI billing 默认流程。 + +### 3.2 凭据存储 + +new-api 套壳站要查真实余额需要 API key 之外的登录凭据。利用渠道已有的 `setting`（JSON）字段，新增约定段： + +```json +{ "balance": { "mode": "newapi_console", + "login": { "username": "...", "password": "..." } } } +``` + +或 `{"balance":{"mode":"system_token","token":"<用户级系统访问令牌>"}}`（不存明文密码，更安全；令牌在各站个人设置页生成）。 +没有该配置的套壳站默认走 `spend_only`。 + +### 3.3 数据模型 + +渠道表已有 `Balance` + `BalanceUpdatedTime`。建议把 `Balance` 语义扩展为： +- A 档/登录态：存真实剩余余额； +- B 档纯 key：`Balance` 存累计消费（带 `Kind=spend_only` 标记，前端用不同颜色/文案区分"已消费"而非"剩余"）。 + +可在 `setting.balance` 里额外存 `recharged`（用户手填的累计充值额），前端据此估算 `剩余 ≈ 充值 − 累计消费`，弥补 B 档拿不到钱包余额的缺口。 + +### 3.4 触发与刷新 + +- 复用现有 `UpdateChannelBalance` / `UpdateAllChannelsBalance`（`/api/channel/update_balance[/:id]`）+ 定时任务 `AutomaticallyUpdateChannels`（`CHANNEL_UPDATE_FREQUENCY`）。 +- 批量刷新前按 `(base_url, key)` 去重，避免同一账号被查多次。 + +### 3.5 展示 + +- **后台渠道列表**：每行余额单元格显示 `剩余 + 单位`（A 档/登录态）或 `已消费 + 单位 + ⚠仅消费` 徽标（B 档）或 `仅控制台`（C 档）+ 更新时间。 +- **新增"下游余额总览"面板**：按上游平台聚合（去重后一平台一行），列：平台 / 余额或累计消费 / 单位 / 到期 / 状态 / 更新时间；余额低于阈值标红。 + +### 3.6 告警（与现有机制打通） + +- 余额低于阈值，或上游返回 402/quota（已被 `isRetryableUpstreamQuotaError` 捕获并触发 `model/channel_cooldown.go` 冷却）时推送通知——**quota 冷却本身就是"该上游没钱了"的最强信号**，把冷却事件接入余额告警即可零成本拿到"余额耗尽"提醒。 + +## 四、落地状态（2026-06-13） + +后端 provider 注册表已实现并对全部真实上游实网验证通过： + +| 优先级 | 事项 | 状态 | 实测 | +|--------|------|------|------| +| 1 | provider 注册表（按 base_url/Other 路由，先于 type switch） + lk888 / listenhub 两个 A 档 provider | ✅ 已实现 | lk888 剩 8.27 算力；listenhub 剩 5 credits | +| 2 | newapi spend provider（哨兵检测→累计消费） | ✅ 已实现 | bltcy 已消费 $9.29；qilin $40.90；hongniao 正确判为"仅控制台" | +| 3 | newapi console provider（账密登录 → /api/user/self 真实钱包余额） + `setting.balance_query` 凭据 | ✅ 已实现 | manxiaobai 登录态查通（备用账号 quota=0） | +| 4a | 前端余额列按三档（剩余/⚠仅消费/仅控制台）区分展示 + 单位 | ✅ 已实现 | tsc/eslint 通过 | +| 4b | EditChannel 加"下游余额查询"配置（模式/账密/累计充值） | ✅ 已实现 | tsc/eslint 通过 | +| 4c | 余额总览（后端聚合接口 `GET /api/channel/balance_overview`，按上游去重，一次查全） | ✅ 已实现 | 集成测试实查 10 个上游全通过 | +| 4d | 前端总览面板（可选，复用上面接口）+ 低余额告警接入 quota 冷却 | ⬜ 待做（可选） | — | + +> 前端构建依赖完整 `node_modules`（需 `bun install`）；本环境缺 `@fontsource-variable/lora` 字体包导致 `rsbuild build` 失败，与本改动无关（`tsc -b` 与 eslint 均通过）。 + +### 实现位置 + +| 文件 | 说明 | +|------|------| +| `controller/channel_balance_provider.go` | 余额 provider 注册表、三档语义、lk888/listenhub/newapi 三个 provider、登录态查询、结果落 OtherInfo | +| `controller/channel-billing.go` | `updateChannelBalance` 开头优先走 provider；`UpdateChannelBalance` 响应增加 kind/unit/used/remaining/provider | +| `dto/channel_settings.go` | `ChannelSettings.BalanceQuery`（mode/账密/系统令牌/已充值额） | +| `controller/channel_balance_provider_test.go` | 实网验证测试（`LIVE_BALANCE_TEST=1` 开启，凭据从环境变量读取） | +| `web/default/.../lib/channel-utils.ts` | 前端 `parseBalanceMeta` / `formatBalanceWithUnit` 三档解析与单位格式化 | +| `web/default/.../components/channels-columns.tsx` | 余额列按三档展示（剩余/⚠仅消费/仅控制台） | +| `web/default/.../lib/channel-form.ts` + `components/drawers/channel-mutate-drawer.tsx` | EditChannel "下游余额查询"配置表单（模式/账密/累计充值） | +| `web/default/.../types.ts` + `i18n/locales/zh.json` | `ChannelSettings.balance_query` 类型 + 中文翻译 | + +### 结果落库字段（渠道 OtherInfo） + +`balance_kind`（balance/spend_only/console_only）、`balance_unit`、`balance_used`、`balance_remaining`、`balance_provider`、`balance_expires_at`、`balance_checked_time`。 +渠道 `Balance` 列：A 档/登录态存真实剩余；spend_only 档存"已充值−累计消费"估算值（未填充值则存累计消费占位，保持为正以免被余额≤0 自动禁用误伤）。 + +### 给套壳站配登录凭据拿真实余额（可选） + +在渠道 `setting`（JSON）里加： + +```json +{ "balance_query": { "mode": "newapi_console", + "username": "<下游站账号>", "password": "<密码>", + "recharged": 0 } } +``` + +未配置时套壳站默认走 spend_only（只显示累计消费）。`recharged` 选填，用于让 spend_only 档估算"剩余 ≈ 充值 − 已消费"。 + +### 用法 + +- 单渠道：`GET /api/channel/update_balance/:id` → 返回 `{balance, kind, unit, used, remaining, provider}`。 +- 全部刷新：`GET /api/channel/update_balance`；定时刷新沿用 `CHANNEL_UPDATE_FREQUENCY`。 +- **一次查出所有下游余额（推荐，后端聚合）**：`GET /api/channel/balance_overview`（AdminAuth）。 + - 按 `(base_url, key)` 去重——同账号多渠道（bltcy 3 个 / manxiaobai 3 个 / apexer 3 个 / xgapi 2 个）只查一次。 + - `?cached=true` 只读已存储余额（不发上游请求，秒回）；`?include_disabled=true` 连同禁用渠道一起查。 + - 返回 `data[]`，每项含 `base_url / channel_ids / channel_names / provider / kind / remaining / used / unit / expires_at / recharged / est_remaining / checked_time / error`。 + - `kind`：`balance`（真实余额）/`spend_only`（仅累计消费）/`console_only`（仅控制台）/`unknown`。 + +实测返回示例（节选）： + +```json +{"success":true,"data":[ + {"base_url":"https://api.lk888.ai/api","channel_ids":[11],"provider":"lk888","kind":"balance","remaining":8.27,"used":2.03,"unit":"算力"}, + {"base_url":"https://api.marswave.ai/openapi","channel_ids":[12],"provider":"listenhub","kind":"balance","remaining":5,"unit":"credits","expires_at":1795799704}, + {"base_url":"https://api.bltcy.ai","channel_ids":[1,2,3],"provider":"newapi","kind":"spend_only","used":9.286388,"unit":"USD"}, + {"base_url":"http://www.937qq.cn","channel_ids":[8],"provider":"newapi","kind":"spend_only","used":40.899,"unit":"USD"}, + {"base_url":"https://open.hongniaoai.com/v1","channel_ids":[9],"kind":"console_only","error":"该上游不支持 API 余额查询，仅 web 控制台可查"} +]} +``` + +curl： + +```bash +curl -s "http://<网关>/api/channel/balance_overview" \ + -H "Authorization: Bearer <管理员令牌>" | jq . +``` + +## 五、一句话结论 + +10 个真实上游里：**3 个（lk888 / siliconflow / listenhub）纯 key 就能查真实余额**；**5 个 new-api 套壳站（bltcy/apexer/xgapi/qilin/manxiaobai）的 key 只能查累计消费，真实钱包余额必须补登录凭据走登录态**；**hongniao 只能登录控制台、runway 无关**。统一方案 = 按 base_url 路由的余额 provider 注册表（与视频 provider 同构）+ 三档语义（真实余额/累计消费/仅控制台）+ 复用现有刷新与 quota 冷却做告警。 diff --git a/docs/channel-failover-review.md b/docs/channel-failover-review.md new file mode 100644 index 000000000000..edd400926331 --- /dev/null +++ b/docs/channel-failover-review.md @@ -0,0 +1,163 @@ +# 多渠道图片/视频生成故障转移 — 实现评审与接入指南 + +> 评审日期：2026-06-10。目标：多渠道互为 backup 生成图片/视频，自动切换或降级，并支撑后续接入更多下游平台。 +> 本文评审当前分支（feature/openai-video-failover）+ 生产部署（192.129.209.36:3001）的实际形态。 + +## 一、当前架构总览 + +一次生成请求经过 4 层，每层都有故障转移参与： + +``` +请求 /v1/images/generations 或 /v1/videos + │ + ├─ 1. 分发层（middleware/distributor.go） + │ 按 group+model 从渠道池选渠道；端点类型过滤 + │ （common/endpoint_type.go）；参考图请求排除 xgapi 直出渠道 + │ + ├─ 2. 选路层（model/channel_cache.go / model/ability.go） + │ priority 桶降序：retry=0 取最高优先级桶，重试一次降一桶； + │ 同桶内按 weight 随机 + │ + ├─ 3. 重试层（controller/relay.go） + │ 同步请求：shouldRetry —— 渠道错误、可配置状态码、 + │ quota 关键词（isRetryableUpstreamQuotaError）→ 换渠道重试， + │ 最多 RetryTimes（生产=2，即最多 3 个优先级桶） + │ 视频任务：shouldRetryTaskRelay —— 429/5xx/307 重试，400 不重试 + │ + └─ 4. 适配层（relay/channel/） + openai/ 同步图片；task/openaivideo/ 视频 provider 模式 + （8 方法接口 + 按 channel.Other/BaseURL 自动检测，已接 8 家）； + ListenHub 独立渠道类型 59 +``` + +渠道健康反馈（与请求路径并行）： +- `processChannelError` → `ShouldDisableChannel`（service/channel.go）：渠道级错误、可配置状态码、`AutomaticDisableKeywords` 文案匹配 → 自动禁用（渠道 AutoBan 开关可豁免）。 +- `controller/channel-test.go`：定时探活（仅 Master 节点、需配置频率）成功后自动恢复 AutoDisabled 渠道。 + +## 二、评审结论：方向正确，骨架合理 + +以下设计与「多渠道 backup + 自动切换」目标契合，应保持： + +1. **priority 桶 + 桶内 weight 随机 + RetryTimes 跳桶**是业界标准做法，且经生产验证有效（2026-06-10：channel 12 余额不足 400 → 自动转移 channel 14 成功出图）。 +2. **quota 错误识别为「可重试」而非简单失败**（commit 48f8be86d）方向正确：余额不足是渠道级临时态，请求级转移让用户无感。 +3. **视频 provider 模式**（relay/channel/task/openaivideo/provider.go）是本仓库最适合扩展的部分：新平台 = 新增一个文件实现 8 个方法 + 在 3 处检测函数注册，已用 bltcy/xgapi/qilin/hongniao/runway/lk888/newapi/manxiaobai 八家验证（manxiaobai 即按本 Checklist 于 2026-06-11 接入）。 +4. **非标准上游独立渠道类型**（ListenHub type 59）避免把非 OpenAI 协议硬塞进 OpenAI 适配器。 +5. **文档驱动运维**（deployment.md 验证记录 + 每次 DB 变更前备份）在多次升级/回归中起了实际作用。 + +## 三、风险与不合理点（按影响排序） + +### P0-1 quota 错误「只重试、不反馈」，坏渠道持续吃首跳（✅ 已修复 2026-06-11，方案见第五节） + +ListenHub 余额 2026-06-07 耗尽后，3 天内每个 gpt-image-2 请求都先打它白吃一跳（+2~3s 延迟），直到 06-10 人工降优先级。重试解决了「单请求最终成功」，但没有渠道健康反馈回路。 + +系统其实已有现成机制，只是没接上： +- `AutomaticDisableKeywords`（线上可配，无需改码）默认含 OpenAI/Anthropic 余额文案，但**不含**中转站常见文案（如 ListenHub 的 `Insufficient credits for Image generation`）。 +- 自动恢复依赖**定时探活**（channel-test），生产未配置测试频率 → `AutomaticEnableChannelEnabled=true` 实际永远不会被触发。 + +**建议**： +1. 把 `insufficient credits` / `insufficient balance` / `余额不足` / `额度不足` 等加进管理后台的自动禁用关键词（与 controller/relay.go `isRetryableUpstreamQuotaError` 的关键词表对齐）——quota 错误于是变成「本请求换渠道重试 + 该渠道自动下线」。 +2. 配置定时探活频率形成恢复闭环。注意：图片/视频渠道探活会产生真实生成费用，建议低频（如 4~6 小时）或对高价渠道关闭 AutoBan 用人工恢复。 + +### P0-2 双源 priority（channels 表 vs abilities 表）漂移 + +内存缓存路径用 `channels.priority` 排序，非缓存路径/桶计算用 `abilities.priority`。通过管理后台改渠道会同步两者；直接 SQL 改 `channels` 不会（2026-06-10 实际踩坑：abilities 仍是 140/0 旧值导致选路与预期不符）。 + +**建议**：运维优先走管理后台改优先级；必须 SQL 直改时，固定执行 +`UPDATE abilities SET priority=(SELECT priority FROM channels WHERE channels.id=abilities.channel_id);` +更彻底的做法是启动时做一次一致性校验/自动同步。 + +### P1-1 厂商特例散落在通用层，新平台会持续放大 + +- `middleware/distributor.go` 用 base_url/渠道名字符串匹配识别 xgapi（`isXGAPIChannel`），排除参考图请求； +- `relay/channel/openai/adaptor.go` 内嵌 xgapi 专属的「比例写入 prompt」逻辑； +- 每接一个有怪癖的新平台，都要往通用分发/适配层加 `if 是某厂商`。 + +**建议**：把这类差异改成**渠道能力标签**（channel `setting`/`param` JSON 中声明，如 `{"supports_reference_image": false, "aspect_ratio_via_prompt": true}`），分发层和适配层只读标签不认厂商。接入新平台变成纯配置动作，也消除字符串匹配误伤（任何名字含 "xgapi" 的渠道都会被当成星光）。 + +### P1-2 图片模型分类靠硬编码列表，加模型要发版 + +`common/model.go` `ImageGenerationModels` 决定 OpenAI 型渠道是否暴露 images 端点。2026-06-10 实际踩坑：横线命名 gemini-3.x 不在列表 → 只剩 ListenHub 一个候选 → ListenHub 断供即整个模型不可用（即文档旧「已知问题 3」的根因）。每新增一个图片模型名都要改代码+重新构建部署。 + +**建议**：模型→端点的归类下放到配置（渠道配置或 abilities 增加端点维度），代码列表只作兜底。短期至少把该列表挪进 operation_setting 做成线上可配。 + +### P1-3 quota 错误识别靠关键词，新平台文案漏判（✅ 已可线上配置 2026-06-11） + +`isRetryableUpstreamQuotaError` 的关键词硬编码在 controller/relay.go。下游平台文案五花八门（英文变体/结构化 error.code/其他语言），漏判即退化为「不重试直接对外失败」。 + +**建议**：关键词表挪到 operation_setting（与自动禁用关键词同级、线上可配）；有 error.code 的上游优先按 code 匹配。 + +### P1-4 参考图路径是单点，未达成 backup 目标（✅ 已解决 2026-06-11） + +`gpt-image-2` 带参考图原本仅 ListenHub 支持（xgapi 被能力排除），ListenHub 断供时该路径完全不可用。2026-06-11 接入漫小白渠道 15（支持 `/v1/images/edits` 参考图，已真实验证约 43 秒出图）后，参考图路径恢复双渠道：漫小白首选（120），ListenHub 兜底（40）。 + +### P2-1 视频任务对 quota 错误不转移（✅ 已修复 2026-06-11） + +`shouldRetryTaskRelay` 把 400 一律视为不可重试，且不含 quota 关键词判断。视频上游若以 400 返回余额不足（中转站常见），不会故障转移。 + +**建议**：把 `isRetryableUpstreamQuotaError` 同样接入 shouldRetryTaskRelay。 + +### P2-2 RetryTimes 全局唯一 + +`RetryTimes=2` 意味着最多覆盖 3 个优先级桶。图片模型当前正好 3 桶（130/100/40 类），再加平台分层后会出现「桶比重试次数多」→ 最低优先级桶永远轮不到。 + +**建议**：渠道分层控制在 RetryTimes+1 桶内；或后续支持按端点/模型差异化重试次数。 + +## 四、新平台接入 Checklist + +### 接入图片平台（同步 /v1/images/*） + +1. **协议判断**：OpenAI Images 兼容 → 直接用 type 1（OpenAI）渠道；非兼容 → 参考 ListenHub（type 59）新建渠道类型 + relay 适配器。 +2. **模型分类**：所有新模型名确认命中 `common.ImageGenerationModels`（含别名/带后缀名），否则 OpenAI 型渠道不会暴露 images 端点（P1-2 的坑）。 +3. **能力确认**：是否支持参考图（`image`/`images`）、是否支持 `size`/`aspect_ratio`、流式（upstream merge 后已支持 images 流式中继）；不支持的能力确认分发层不会把这类请求路由过去。 +4. **StreamOptions**：按 CLAUDE.md Rule 4 确认是否加入 `streamSupportedChannels`。 +5. **配置渠道**：priority 放进现有分层（首选 130 / 主力 100 / 兜底 ≤40），开 AutoBan；改完用管理后台或同步 abilities（P0-2）。 +6. **quota 文案**：拿到该平台「余额不足」的真实报错文案，确认命中 `isRetryableUpstreamQuotaError` 关键词表，并加进自动禁用关键词（P0-1/P1-3）。 +7. **真实验证**：经公网入口直出 + 参考图（如支持）各打一发，确认 HTTP 200、`data[0].url/b64_json` 有效、日志命中预期 channel；故意打一发会失败的请求验证转移路径。 +8. **登记文档**：deployment.md 渠道表 + 验证记录 + api-usage.md 模型表。 + +### 接入视频平台（异步 /v1/videos） + +1. 在 `relay/channel/task/openaivideo/` 新建 `.go`，实现 provider 接口 8 方法（submitURL/queryURL/parseSubmitResponse/parseQueryResponse/buildSubmitResponseBody/needsMultipart/mapModelForImages 等）。 +2. 在 provider.go 的 `getProviderByHint` / BaseURL 检测 / `getProviderForRelayInfo` 三处注册识别特征（建议用渠道 `other` 字段显式 hint，少用 URL 推断）。 +3. 统一参数收敛：把调用方的 `images`/`image`/`input_reference`、`seconds`/`duration`、`aspect_ratio`/`size` 映射成平台原生参数（参考 qilin.go/lk888.go 的处理与比例映射表）。 +4. 渠道 type 58（OpenAI Video），priority 放层级，注意 `channels.models` 与 `abilities` 同时补齐（2026-06-07 LK888 的坑：只有 models 没有 abilities → /v1/models 不暴露）。 +5. 真实验证：提交→轮询→`/content` 下载 200 video/mp4；记录 task_id 进文档。 + +## 五、改进路线图（建议顺序） + +| 优先级 | 事项 | 改动面 | 效果 | 状态 | +|--------|------|--------|------|------| +| 1 | quota 错误渠道冷却（替代探活闭环） | 小 | 坏渠道自动跳过/到期自动恢复，零探活成本 | ✅ 2026-06-11 已实现并部署 | +| 2 | abilities/channels priority 一致性 | SOP | 消除选路与配置不一致 | ✅ SOP 见下文 | +| 3 | quota 关键词表挪 operation_setting；接入视频任务重试 | 小 | 新平台文案可配，视频也能转移 | ✅ 2026-06-11 已实现并部署 | +| 4 | 渠道能力标签化（参考图/比例注入等），替换厂商字符串匹配 | 中 | 接新平台零通用层改动 | 待做（下批平台接入前） | +| 5 | ImageGenerationModels 配置化 | 中 | 加模型不发版 | 待做 | +| 6 | 补第二家参考图渠道 | 配置+验证 | 消除参考图单点 | ✅ 2026-06-11 漫小白渠道 15 已充值激活并真实验证（/v1/images/edits 参考图 43.2s 出图，语义生效），参考图单点消除 | +| 7 | 下游余额可见性（统一查询 + 总览接口） | 中 | 一处看全下游余额、提前发现耗尽、辅助充值决策 | ✅ 2026-06-13 已实现（详见 [channel-balance-query.md](./channel-balance-query.md)）：余额 provider 注册表（三档语义）+ 聚合接口 `GET /api/channel/balance_overview` | + +### 已实现机制说明（2026-06-11） + +**quota 冷却**（P0-1 的最终方案，未采用「关键词自动禁用 + 定时探活」，因为定时探活对图片/视频渠道会发真实生成请求持续产生费用）： + +- 上游返回 400/402/403 且文案命中 quota 关键词 → 该渠道进入冷却（`QUOTA_ERROR_COOLDOWN_SECONDS` 环境变量控制，默认 600 秒），本次请求照常换渠道重试。 +- 冷却期内选路把该渠道整体移出候选（优先级桶随之重排）；该模型全部候选都在冷却时放行兜底，保证不出现「全员冷却无渠道可用」。 +- 冷却到期自动恢复；期间第一个放行/到期请求天然充当被动探活。人工启用渠道也会立即解除冷却。 +- 实现：`model/channel_cooldown.go`（注册表）、`model/channel_cache.go`（选路过滤）、`controller/relay.go` `processChannelError`（触发）。 +- 限制：冷却过滤只作用于内存缓存选路路径（`MEMORY_CACHE_ENABLED=true`，生产默认）；DB 直查路径未接入。 +- 关联：余额耗尽是触发冷却的主要原因之一。要在耗尽**之前**主动发现，用余额总览接口 `GET /api/channel/balance_overview`（详见 [channel-balance-query.md](./channel-balance-query.md)）定期查各下游余额。 + +**quota 关键词线上可配**：运营设置项 `UpstreamQuotaErrorKeywords`（按行分隔，默认含 insufficient credits/balance、quota exceeded、余额不足、额度不足等）。同步/视频两条重试路径与冷却触发共用这份关键词。接入新平台时把其真实余额错误文案补进该设置即可，无需发版。 + +**视频任务 quota 转移**：`shouldRetryTaskRelay` 现在对 400/402/403 + quota 文案的任务错误执行换渠道重试（此前 400 一律不重试）。 + +**priority 双源一致性 SOP**：渠道优先级一律通过管理后台修改（自动同步 abilities）；必须 SQL 直改时，改完执行： + +```sql +UPDATE abilities SET priority=(SELECT priority FROM channels WHERE channels.id=abilities.channel_id); +``` + +或调用管理后台「修复数据库一致性」（`model.FixAbility`，会全量重建 abilities 并刷新缓存）。 + +## 六、一句话结论 + +骨架（priority 桶选路 + 跳桶重试 + provider 适配模式）是合理且已被生产验证的，可以放心在其上加平台。反馈回路已于 2026-06-11 通过 quota 冷却机制接通（坏渠道自动跳过、到期自动恢复），参考图单点也已由漫小白渠道补齐（路线图 6 ✅）；剩余主要欠账是**厂商特例没有配置化**（路线图 4/5），它决定接入第 10 家平台时的边际成本，建议在下一批平台接入前完成。 diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 000000000000..ddc40de75015 --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,636 @@ +# 服务器部署与更新文档 + +## 当前生产部署（Coolify 管理） + +2026-05-26 已将服务迁移到新服务器，并接入 Coolify 管理。当前对上游暴露的生产入口为： + +| 项目 | 值 | +|------|-----| +| Base URL | `http://192.129.209.36:3001/v1` | +| 管理面板 | `http://192.129.209.36:3001` | +| Coolify 面板 | `http://192.129.209.36:8000` | +| Coolify 资源名 | `new-api-video-gateway` | +| Coolify UUID | `jssc8c4sc4gk80oo84ks480w` | +| Coolify 项目/环境 | `My first project / production` | +| 容器名 | `new-api-jssc8c4sc4gk80oo84ks480w` | +| 镜像 | `new-api-local:coolify` | +| 运行状态 | `running:healthy` | + +### 新服务器信息 + +| 项目 | 值 | +|------|-----| +| 公网 IP | `192.129.209.36` | +| SSH 端口 | `22` | +| SSH 账号 | `root` | +| 系统 | Ubuntu 24.04 / Linux 6.8 | +| 磁盘 | 约 96G，部署时剩余约 76G | +| 内存 | 约 5.8GiB + 9GiB swap（含 `/swapfile-build` 6GiB 构建辅助 swap） | +| 服务端口 | `3001 -> 3000` | + +SSH 连接： + +```bash +ssh root@192.129.209.36 +``` + +本地私有连接备忘录存放在 `docs/deployment.local.md`，该文件已通过 `.gitignore` 排除，不提交到远端。 + +### Coolify 管理的运行架构 + +``` +服务器 (192.129.209.36) +├── Coolify / Traefik +│ ├── coolify # 面板，映射 8000 +│ ├── coolify-proxy # 占用 80/443/8080 +│ ├── coolify-db / coolify-redis +│ └── /data/coolify/services/jssc8c4sc4gk80oo84ks480w/ +│ ├── docker-compose.yml # Coolify 生成的 Compose +│ └── .env +├── /opt/new-api-src/ # 当前源码和 Dockerfile +├── /opt/new-api-data/ +│ └── one-api.db # 从旧服务器迁移的 SQLite 数据库 +├── /opt/new-api-logs/ # new-api 日志目录 +└── new-api-jssc8c4sc4gk80oo84ks480w + └── 0.0.0.0:3001 -> 3000/tcp +``` + +Coolify 使用的 Compose 由 Coolify 资源生成，核心配置如下： + +| 配置 | 值 | +|------|-----| +| `build.context` | `/opt/new-api-src` | +| `build.dockerfile` | `Dockerfile` | +| `container_name` | `new-api-jssc8c4sc4gk80oo84ks480w` | +| `SQLITE_PATH` | `/data/one-api.db?_busy_timeout=30000` | +| 数据卷 | `/opt/new-api-data:/data` | +| 日志卷 | `/opt/new-api-logs:/data/logs` | +| `NODE_NAME` | `new-api-racknerd-coolify` | + +### 新服务器验证记录 + +| 时间 | 项目 | 结果 | +|------|------|------| +| 2026-05-26 | HTTP 健康检查 | `GET http://192.129.209.36:3001/api/status` 返回 `success=true` | +| 2026-05-26 | Coolify 管理状态 | `service_applications.status=running:healthy`，容器 label `coolify.managed=true` | +| 2026-05-26 | 真实视频生成 | `grok-video-3` 任务 `task_kTOu1dhTCYZvSYtynESiQQz0rEqlHOjO` 完成 | +| 2026-05-26 | 视频下载 | `/v1/videos/task_kTOu1dhTCYZvSYtynESiQQz0rEqlHOjO/content` 返回 `200 video/mp4`，大小 `868422` 字节 | +| 2026-05-28 | upstream 合并 | 合并 origin/main 78 commits（channel 重构、gjson 优化、Claude fcIdx 修复、前端依赖升级等），仅 package.json 有冲突 | +| 2026-05-28 | Docker 重建 | `docker build -t new-api-local:coolify .` 成功，前端 (bun) + 后端 (go) 均编译通过 | +| 2026-05-28 | 容器重启 | Coolify service `running:healthy`，API status `success=true` | +| 2026-05-28 | 全模型回归 | 5 个视频模型真实验证完成（veo3.1-fast, xb-sora2, grok-imagine-1.0-video, ss-sora-2, veo3.1-4k），详见 [api-usage.md](./api-usage.md) | +| 2026-05-28 | Runway 状态 | 确认 Runway 渠道当前未配置，kling-3.0/o3 等新模型已注册在 constants.go 但未暴露给上游 | +| 2026-06-06 | SiliconFlow 图片渠道 | 已部署硅基流动渠道 `siliconflow-images`（channel id `13`，type `40`，base URL `https://api.siliconflow.cn`，group `default`）。模型 `baidu/ERNIE-Image-Turbo`、`Qwen/Qwen-Image`、`Tongyi-MAI/Z-Image`、`Qwen/Qwen-Image-Edit-2509` 已经通过远端 `/v1/images/generations` 或 `/v1/images/edits` 真实请求验证，均返回 HTTP 200 且 `data` 长度为 1。 | +| 2026-06-06 | SiliconFlow 部署备份 | 新增渠道前备份 SQLite 到 `/opt/new-api-data/one-api.db.before-siliconflow-20260606-064135`；API key 仅保存在远端数据库渠道配置中，不写入仓库文档。 | +| 2026-06-06 | upstream 同步部署 | 当前功能分支已 merge `origin/main` 最新提交 `adc390c5f`，解决 4 个冲突后远端 Docker/Coolify 构建成功，镜像 `new-api-local:coolify` 重建完成，容器状态 `healthy`。 | +| 2026-06-06 | 合并后 SiliconFlow 回归 | 远端公网入口复测 4 个 SiliconFlow 模型均返回 HTTP 200、`data` 长度 1 且包含 `url`：`baidu/ERNIE-Image-Turbo` 20.89s、`Qwen/Qwen-Image` 18.76s、`Tongyi-MAI/Z-Image` 12.20s、`Qwen/Qwen-Image-Edit-2509` 24.60s。 | +| 2026-06-07 | 图片路由修复与 xgapi 兜底 | 部署 URL 拼接去重和图片模型分类修复；新增 `xgapi-images`（channel id `14`，type `1`，priority `130`），使用 xgapi `gpt-image-2` 兜底 `gpt-image-2`、`gpt-image-2(线路XF)`、`gr-image-2`、`nano-banana-pro`，并从 Hongniao 视频渠道移除不支持标准 Images 的 `gr-image-2` / `gpt-image-2(线路XF)`。数据库变更前备份到 `/opt/new-api-data/one-api.db.before-image-fallback-20260606-190024`。 | +| 2026-06-07 | 图片模型稳定性回归 | 公网统一入口连续 2 轮验证 `gpt-image-2`、`gpt-image-2(线路XF)`、`gr-image-2`、`nano-banana`、`nano-banana-hd`、`nano-banana-pro`，共 12 次全部 HTTP 200，均返回标准 `data[0].url`。 | +| 2026-06-07 | xgapi 图片比例与参考图兼容 | 部署 xgapi 图片兼容逻辑：直接生图按 `size` / `aspect_ratio` 自动把比例补到上游 prompt；带 `image` / `images` / `referenceImages` 的请求避开 xgapi 直接生图线路并回退 ListenHub。远端复测 `gpt-image-2`：无比例 prompt + `size=1792x1024` 命中 channel 14，HTTP 200，46.33s，PNG `1659x948`；参考图请求命中 channel 12，HTTP 200，45.08s，PNG `2048x2048`。 | +| 2026-06-07 | 近期使用记录巡检 | 远端 `logs` 近 24h：成功消费 77 条，其中 `/v1/images/generations` 73 条、`/v1/images/edits` 4 条；部署后近 30 分钟错误日志 0 条。远端 `tasks` 近 30 天视频任务 121 条：103 成功、18 失败；失败集中在 2026-05-28 及以前的上游 token/429/旧尺寸格式问题，当前未发现新视频失败。 | +| 2026-06-07 | Apexer gpt-image-2 配置修剪 | 生产错误日志显示 `gpt-image-2` 在 Apexer channel 6/7 会触发上游 distributor 503 或 `only imagen models are supported`，已从 `apexer-images-openai` / `apexer-images-gemini` 移除，仅保留 xgapi 直接生图与 ListenHub 参考图 fallback。变更前备份 SQLite 到 `/opt/new-api-data/one-api.db.before-apexer-gpt-image2-prune-20260606-194956`。 | +| 2026-06-07 | 图片/视频生产自测 | 重新验证当前路径：`gpt-image-2` 直接生图 channel 14，46.51s，PNG `1659x948`；`gpt-image-2` 参考图 channel 12，93.63s，PNG `2048x2048`；`Qwen/Qwen-Image-Edit-2509` 编辑 channel 13，29.97s，PNG；`grok-video-3` channel 11 轮询到 `completed`，`/content` 返回 `200 video/mp4`。 | +| 2026-06-11 | 漫小白充值激活与全链路回归 | 用户自行注册充值并提供新 key，渠道 15/16/17 已切换（旧 vipergw2026 账号保留备用）。公网入口真实验证全部通过：`gpt-image-2-1k` 命中 ch15（137.6s，PNG 1824×1024）；`gemini-3.1-flash-image-preview` 经 Gemini 原生入口命中 ch16（22.1s，JPEG 1376×768，16:9 生效）；`grok-imagine-video` 命中 ch17 全流程（提交→轮询 completed→`/content` 下载 200 video/mp4 18.5MB）；`gpt-image-2` 参考图 `/v1/images/edits` 命中 ch15（43.2s，抽检确认红底保留、白圆按 prompt 变笑脸太阳）——**参考图单点正式消除**。过程中修复两个问题：(1) 漫小白 `/v1/videos` JSON 解析丢 model 字段，provider 改 multipart 表单提交；(2) 无直链上游的任务 ResultURL 会被填成本服务自指代理地址，content 回源改用上游鉴权 content 端点。 | +| 2026-06-11 | 漫小白渠道接入 | 注册 `api.manxiaobai.online` 账号并创建渠道 15/16/17（图片 OpenAI / 图片 Gemini / 视频，详见下方专节）。新增 `manxiaobaiProvider`（标准 /v1/videos JSON 协议）。轮换兜底真实验证：临时把 channel 15 调到 140，`gpt-image-2` 请求命中 15 → 403「当前模型暂时不可用」→ 冷却触发 → 直接转移 channel 14 成功出图（47.0s）；冷却期第二发直通 14（45.3s）；`grok-imagine-video` 经网关路由 channel 17，上游接受请求形态并返回 quota 预检 403，task 路径 quota 重试触发。验证后 channel 15 恢复 priority 120。**渠道已就绪但上游账号余额为 0，充值后即自动生效**。 | +| 2026-06-11 | quota 识别两处加固 | （1）上游 OpenAI 风格错误体透传时 errorCode 是上游 code（如 `insufficient_user_quota`）而非 `bad_response_status_code`，原严格检查导致冷却不触发——已改为「排除 SkipRetry + 消息与 code 一起关键词匹配」，并把 `insufficient_user_quota` 加入默认关键词；（2）冷却渠道从候选整体移除会导致优先级桶序左移、重试跳桶（实测 attempt2 跳过 130 桶落到 40 桶）——已改为桶序基于全量渠道稳定、冷却只在桶内过滤、滤空顺延下一桶。两处均已部署并真实验证。 | +| 2026-06-11 | quota 冷却机制部署 | 部署渠道 quota 冷却 + 可配置 quota 关键词（`UpstreamQuotaErrorKeywords`）+ 视频任务 quota 转移。生产验证：临时把 channel 12 调回 priority 140，请求 1 命中 12 → 400 → 日志 `channel #12 entered quota cooldown for 10m0s` → 转移 channel 14 成功（46.9s）；请求 2 在冷却期内直接命中 channel 14（53.4s），未再打 12。验证后 channel 12 恢复 priority 40（channels + abilities 双表）。冷却时长由 `QUOTA_ERROR_COOLDOWN_SECONDS` 控制（默认 600 秒）。详见 [channel-failover-review.md](./channel-failover-review.md) 第五节。 | +| 2026-06-10 | upstream 同步部署 | 合并 `origin/main` 最新提交 `59a93cf5c`（27 commits：images API 流式中继、image edit 支持、kimi k2.6 温度归一、模型定价编辑器重构等），仅 `relay/helper/stream_scanner.go` 冲突（保留本分支 `DefaultStreamingTimeout`，采纳上游 128MB SSE buffer）。远端 Docker/Coolify 重建成功，容器 `healthy`。 | +| 2026-06-10 | channel_info 扫描修复 | 合并部署后发现 `model/channel.go` 的 `ChannelInfo.Scan` 对空值（channel 11 `NULL`、channel 14 空字符串）报 `unexpected end of JSON input`，导致 channel 缓存每分钟刷新失败。已修复 Scan 容忍空值并兼容 string 类型；同时把存量空 `channel_info` 归一为 `{}`，变更前备份到 `/opt/new-api-data/one-api.db.before-channel-info-fix-*`。重新部署后观察 4 分钟无该错误。 | +| 2026-06-10 | 合并后图片生成回归 | 公网入口真实验证：`gpt-image-2` `1792x1024` 命中 channel 14，51.3s，PNG `1659x948`；`Qwen/Qwen-Image` 命中 channel 13，20.5s，返回 `url`；修复部署后复测 `gpt-image-2` `1024x1024`，88.7s，PNG `1254x1254`，下载抽检画面与 prompt 一致。生产日志确认 channel 12 余额不足（400）时自动故障转移到 channel 14 成功。 | +| 2026-06-10 | ListenHub 渠道降级 | 渠道 12（listenhub-images）上游 Marswave 余额自 2026-06-07 10:27 起耗尽，此后 3 天该渠道全部请求返回 `400 Insufficient credits`（最后一次成功 2026-06-07 10:17），且实际优先级为 140（高于 xgapi 的 130，文档原记录 120 已过时），导致 gpt-image-2 直接生图每次先空跑一跳。已将优先级降到 40 作兜底，变更前备份到 `/opt/new-api-data/one-api.db.before-listenhub-priority-drop-*`。注意：带参考图的 gpt-image-2 请求会避开 xgapi，ListenHub 是唯一候选——在上游充值前该路径会失败；充值恢复后建议把优先级调回 120。 | +| 2026-06-10 | 横线 gemini-3.x 图片模型修复 | 排查「已知问题 3」根因：`common.ImageGenerationModels` 缺少横线命名 `gemini-3-pro-image-preview` / `gemini-3.1-flash-image-preview`，导致 OpenAI 类型渠道不暴露 images 端点，该两模型实际只有 ListenHub 一个候选，ListenHub 断供即全挂。已（1）把两个横线名加入分类列表；（2）实测确认 bltcy 不支持该两模型（images/chat 均拒绝），从 channel 2/3 移除挂载；（3）channel 6（Apexer）通过 model_mapping 把横线名映射到 `gemini_3.0_pro_image_preview` / `gemini_3.1_flash_image_preview` 承接；（4）同步 `abilities.priority` 与 `channels.priority`（此前 abilities 仍是旧值 140/0，直改 channels 不生效）。变更前备份 `/opt/new-api-data/one-api.db.before-gemini-dash-remap-*`。 | +| 2026-06-10 | 降级与改造后图片回归 | 公网入口实测：`gpt-image-2` 直出直接命中 channel 14（50.9s，无 ListenHub 空跳）；`gemini-3-pro-image-preview` 命中 channel 6 Apexer 映射（48.7s，PNG 1024×1024，抽检画面符合 prompt）；`gemini-3.1-flash-image-preview` 命中 channel 6（41.8s，b64 约 1.97MB）。 | +| 2026-06-07 | LK888 模型列表能力补齐 | 发现 channel 11 的 `channels.models` 已有 `sora-2,grok-video-3`，但 `abilities` 缺失导致 `/v1/models` 不暴露裸 `grok-video-3`。已补齐 channel 11 的 `sora-2` / `grok-video-3` abilities，变更前备份到 `/opt/new-api-data/one-api.db.before-lk888-abilities-20260606-200036`；同步后 `/v1/models` 已确认包含 `grok-video-3`。 | +| 2026-06-13 | 下游余额查询调研与实现 | 逐个实测全部 10 个真实上游的余额接口（详见 [channel-balance-query.md](./channel-balance-query.md)）：lk888（`/v1/skills/balance` 剩 8.27 算力）、listenhub（`/openapi/v1/user/subscription` 剩 5 credits）、siliconflow 三家纯 key 可查真实余额；bltcy/apexer/xgapi/qilin/manxiaobai 五家是 new-api 套壳站、key 为无限额度令牌，只能查累计消费（`/v1/dashboard/billing/usage`），真实钱包余额需配控制台账密走登录态；hongniao 仅控制台、runway 本机适配器。已实现按 base_url/Other 路由的余额 provider 注册表（三档语义）+ 聚合接口 `GET /api/channel/balance_overview`（按 (base_url,key) 去重，一次查全）。`go build ./...` 通过；用生产库副本跑集成测试实查 10 个上游均通过（HTTP 200，16.78s）。 | + +### 新服务器常用命令 + +```bash +# 查看 Coolify 管理的 new-api 容器 +docker ps --filter name=new-api --format "table {{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}" + +# 查看 Coolify 生成的 Compose +sed -n '1,220p' /data/coolify/services/jssc8c4sc4gk80oo84ks480w/docker-compose.yml + +# 查看容器日志 +docker logs -f new-api-jssc8c4sc4gk80oo84ks480w + +# 查看服务健康 +curl -sS http://127.0.0.1:3001/api/status + +# 查看数据库渠道 +sqlite3 /opt/new-api-data/one-api.db "SELECT id, name, type, status, base_url FROM channels ORDER BY id;" + +# 一次查出所有下游平台余额（按 base_url+key 去重，需管理员令牌） +curl -s "http://127.0.0.1:3001/api/channel/balance_overview" \ + -H "Authorization: Bearer <管理员令牌>" | jq . +# 只读已存储余额、不发上游请求：追加 ?cached=true +``` + +> 注意：Coolify API 默认保持关闭。2026-05-26 为自动创建 `new-api-video-gateway` 曾临时开启 API 并创建临时 token；接入完成后已删除 token，并将 `instance_settings.is_api_enabled` 恢复为 `false`。 + +### 同步 upstream 最新代码流程 + +本仓库的业务改动集中在当前功能分支，`fork/main` 只作为 fork 的主分支镜像使用。同步官方 `origin/main` 时优先使用 merge，不使用 rebase，避免改写已部署和已推送的历史。 + +推荐流程： + +```bash +# 1. 确认工作区干净 +git status -sb + +# 2. 拉取官方仓库和 fork 最新引用 +git fetch origin --prune +git fetch fork --prune + +# 3. 记录合并前备份点 +git branch ccj/pre-origin-main-merge-$(date +%Y%m%d-%H%M%S) HEAD + +# 4. 合并官方 main +git merge --no-ff origin/main + +# 5. 解决冲突后做静态检查 +git diff --check +rg -n "^(<<<<<<<|=======|>>>>>>>)" -g '!web/**/node_modules/**' -g '!tmp/**' -g '!logs/**' . || true + +# 6. 提交并推送当前功能分支 +git commit +git push fork feature/openai-video-failover +``` + +合并后不要在本地跑项目 test/build；如需验证，按本文档的新服务器 Coolify 流程远端构建部署，再用真实 API 请求验证关键自定义通道。 + +--- + +## 旧服务器信息（历史/备用） + +| 项目 | 值 | +|------|-----| +| 服务商 | 衡天云 | +| 区域 | 日本/东京一区 | +| 配置 | 2C4G + 50G SSD | +| 系统 | Debian 12 | +| 公网 IP | 206.119.182.61 | +| SSH 端口 | 38554 | +| SSH 账号 | root | +| 服务端口 | 80 (HTTP) | + +--- + +## 本地代码改动清单 + +以下文件是相对于上游 new-api 项目的自定义改动： + +### 新增文件 + +| 文件 | 说明 | +|------|------| +| `relay/channel/task/openaivideo/adaptor.go` | OpenAI Video 任务适配器主文件 | +| `relay/channel/task/openaivideo/provider.go` | Provider 接口定义 + 自动检测逻辑 | +| `relay/channel/task/openaivideo/bltcy.go` | bltcy.ai / ablai.top 中转站适配 | +| `relay/channel/task/openaivideo/xgapi.go` | xgapi.top 中转站适配 | +| `relay/channel/task/openaivideo/qilin.go` | 937qq / 麒麟 API Grok 视频适配 | +| `relay/channel/task/openaivideo/newapi.go` | 通用 new-api 实例适配 | +| `relay/channel/task/openaivideo/manxiaobai.go` | 漫小白（api.manxiaobai.online）视频适配（multipart 提交、seconds 字符串化、尺寸归一） | +| `relay/channel/task/openaivideo/constants.go` | 模型列表常量 | +| `model/channel_cooldown.go` | 渠道 quota 冷却注册表（坏渠道临时移出选路，到期自动恢复） | +| `setting/operation_setting/quota_error.go` | 可配置的上游余额/额度错误关键词（运营设置 `UpstreamQuotaErrorKeywords`） | +| `controller/channel_balance_provider.go` | 下游余额查询 provider 注册表（按 base_url/Other 路由的三档余额查询 + 登录态 + 聚合接口 `GetChannelBalanceOverview`） | +| `controller/channel_balance_provider_test.go` | 余额 provider 实网测试（`LIVE_BALANCE_TEST=1` 开启，凭据走环境变量） | +| `docs/api-usage.md` | API 调用文档 | +| `docs/deployment.md` | 本文档 | +| `docs/channel-failover-review.md` | 多渠道故障转移评审与新平台接入 Checklist | +| `docs/channel-balance-query.md` | 下游平台余额查询调研结论与统一查询设计 | + +### 修改文件 + +| 文件 | 改动说明 | +|------|----------| +| `constant/channel.go` | 新增 `ChannelTypeOpenAIVideo = 58` | +| `relay/relay_adaptor.go` | 注册 openaivideo 任务适配器 | +| `setting/ratio_setting/model_ratio.go` | 新增 Veo/图片/漫小白模型定价（含上游采购价注释） | +| `controller/relay.go` | quota 错误识别（可重试 + 触发渠道冷却）、视频任务 quota 转移 | +| `controller/video_proxy.go` | OpenAI Video 类型无直链/自指 ResultURL 时回退上游鉴权 content 端点 | +| `model/channel_cache.go` | 选路接入 quota 冷却过滤（桶序稳定，桶空顺延） | +| `model/option.go` | 注册 `UpstreamQuotaErrorKeywords` 运营设置项 | +| `common/model.go` | 图片模型分类补横线命名 gemini-3.x | +| `middleware/distributor.go` | 图片端点过滤、参考图请求避开 xgapi 直出渠道 | +| `web/default/src/features/channels/constants.ts` | 前端新增渠道类型 58 | +| `web/default/src/features/channels/lib/channel-type-config.ts` | 前端渠道配置提示信息 | +| `controller/channel-billing.go` | `updateChannelBalance` 接入余额 provider（先于 type switch）；单渠道查询响应增加 kind/unit/used/remaining/provider | +| `dto/channel_settings.go` | `ChannelSettings.BalanceQuery`（mode/账密/累计充值），用于套壳站登录态查真实余额 | +| `router/api-router.go` | 注册 `GET /api/channel/balance_overview` 余额总览聚合接口 | +| `web/default/.../channels-columns.tsx` + `lib/channel-utils.ts` | 余额列按三档（剩余/⚠仅消费/仅控制台）展示 + 单位 | +| `web/default/.../drawers/channel-mutate-drawer.tsx` + `lib/channel-form.ts` + `types.ts` + `i18n/locales/zh.json` | EditChannel "下游余额查询"配置表单 + 类型 + 翻译 | + +--- + +## 服务器运行架构 + +``` +服务器 (206.119.182.61) +├── /root/new-api/ # 项目源码目录 +│ ├── new-api # 编译好的二进制文件 +│ ├── web/default/dist/ # 前端构建产物 +│ └── web/classic/dist/ # 经典前端构建产物 +├── /data/ # 数据目录 +│ ├── one-api.db # SQLite 数据库 +│ └── logs/ # 日志目录 +├── /swapfile # 4G Swap 分区 +└── systemd service # new-api.service (开机自启) +``` + +**运行方式：** 直接运行 Go 二进制文件（非 Docker），通过 systemd 管理。 + +**环境变量（在 /etc/systemd/system/new-api.service 中配置）：** + +| 变量 | 值 | 说明 | +|------|-----|------| +| PORT | 80 | HTTP 端口 | +| TZ | Asia/Shanghai | 时区 | +| MEMORY_CACHE_ENABLED | true | 启用内存缓存 | +| BATCH_UPDATE_ENABLED | true | 批量更新 | +| NODE_NAME | new-api-tokyo | 节点名 | + +--- + +## 同步远端代码 & 更新部署 + +### 方法一：rsync 同步代码 + 服务器编译（推荐） + +```bash +# 1. 从本地同步代码到服务器（排除不需要的文件） +rsync -avz \ + -e "sshpass -p 'm0HLTSun1xE4' ssh -o StrictHostKeyChecking=no -p 38554" \ + --exclude='.git' \ + --exclude='node_modules' \ + --exclude='data/' \ + --exclude='logs/' \ + --exclude='.DS_Store' \ + /Users/bytedance/go/src/github.com/new-api/ \ + root@206.119.182.61:/root/new-api/ + +# 2. SSH 到服务器编译前端 +sshpass -p 'm0HLTSun1xE4' ssh -p 38554 root@206.119.182.61 +cd /root/new-api +export PATH=$PATH:/usr/local/go/bin:/root/.bun/bin + +# 编译 default 前端 +cd web/default && bun install && DISABLE_ESLINT_PLUGIN=true bun run build && cd ../.. + +# 编译 classic 前端 +cd web/classic && bun install && VITE_REACT_APP_VERSION=$(cat ../../VERSION) bun run build && cd ../.. + +# 编译 Go 后端 +CGO_ENABLED=0 GOEXPERIMENT=greenteagc go build -ldflags "-s -w" -o new-api . + +# 重启服务 +systemctl restart new-api +``` + +### 方法二：一键更新脚本 + +在本地创建快捷脚本： + +```bash +#!/bin/bash +# deploy.sh - 一键部署脚本 + +SSH_HOST="root@206.119.182.61" +SSH_PORT="38554" +SSH_PASS="m0HLTSun1xE4" +LOCAL_DIR="/Users/bytedance/go/src/github.com/new-api/" +REMOTE_DIR="/root/new-api/" + +echo "=== 1. 同步代码 ===" +sshpass -p "$SSH_PASS" rsync -avz \ + -e "ssh -o StrictHostKeyChecking=no -p $SSH_PORT" \ + --exclude='.git' --exclude='node_modules' \ + --exclude='data/' --exclude='logs/' --exclude='.DS_Store' \ + "$LOCAL_DIR" "$SSH_HOST:$REMOTE_DIR" + +echo "=== 2. 编译 & 重启 ===" +sshpass -p "$SSH_PASS" ssh -o StrictHostKeyChecking=no -p $SSH_PORT "$SSH_HOST" << 'REMOTE_SCRIPT' +export PATH=$PATH:/usr/local/go/bin:/root/.bun/bin +cd /root/new-api + +# 前端 +cd web/default && bun install && DISABLE_ESLINT_PLUGIN=true bun run build && cd ../.. +cd web/classic && bun install && VITE_REACT_APP_VERSION=$(cat ../../VERSION 2>/dev/null || echo "dev") bun run build && cd ../.. + +# 后端 +CGO_ENABLED=0 GOEXPERIMENT=greenteagc go build -ldflags "-s -w" -o new-api . + +# 重启 +systemctl restart new-api +sleep 3 +systemctl status new-api --no-pager | head -10 +REMOTE_SCRIPT + +echo "=== 部署完成 ===" +``` + +--- + +## 服务器管理命令 + +```bash +# SSH 连接 +sshpass -p 'm0HLTSun1xE4' ssh -o StrictHostKeyChecking=no -p 38554 root@206.119.182.61 + +# 查看服务状态 +systemctl status new-api + +# 重启服务 +systemctl restart new-api + +# 停止服务 +systemctl stop new-api + +# 查看实时日志 +journalctl -u new-api -f + +# 查看最近 50 行日志 +journalctl -u new-api -n 50 --no-pager + +# 查看文件日志 +tail -f /data/logs/*.log + +# 查看数据库 +sqlite3 /data/one-api.db "SELECT id, name, status FROM channels;" + +# 查看内存使用 +free -h + +# 查看磁盘使用 +df -h / +``` + +--- + +## 管理面板 + +| 项目 | 值 | +|------|-----| +| 地址 | http://206.119.182.61/ | +| 用户名 | admin | +| 密码 | Admin@2026 | + +--- + +## 已配置的渠道 + +| 渠道 ID | 名称 | 类型 | 优先级 | AutoBan | 上游 | 主要模型 | +|---------|------|------|--------|---------|------|----------| +| 1 | bltcy-veo | 58 (OpenAI Video) | 100 | 开启 | https://api.bltcy.ai | veo2/veo3/veo3.1 视频模型 + MiniMax-Hailuo-02/2.3 | +| 2 | bltcy-openai-v2 | 1 (OpenAI) | 100 | 关闭 | https://api.bltcy.ai | 文本/兼容模型备用 | +| 3 | bltcy-images | 1 (OpenAI) | 100 | 关闭 | https://api.bltcy.ai | nano-banana, gemini-2.5 image 系列（2026-06-10 移除横线 gemini-3.x：bltcy 实测不支持） | +| 4 | apexer-veo | 58 (OpenAI Video) | 50 | 开启 | https://www.aiapexers.com | veo3.1/fast/pro、4K、components | +| 5 | xgapi-veo | 58 (OpenAI Video) | 10 | 开启 | https://xgapi.top | veo3.1-lite, sora-2（账号资源受限，详见已知问题） | +| 6 | apexer-images-openai | 1 (OpenAI) | 60 | 开启 | https://www.aiapexers.com | gemini_3.*_image_preview（OpenAI 图片/对话格式）；2026-06-10 起通过 model_mapping 承接横线名 `gemini-3-pro-image-preview` / `gemini-3.1-flash-image-preview` | +| 7 | apexer-images-gemini | 24 (Gemini) | 50 | 开启 | https://www.aiapexers.com | gemini_3.*_image_preview（Gemini 原生格式） | +| 8 | qilin-grok-video | 58 (OpenAI Video) | 80 | 开启 | http://www.937qq.cn | grok-imagine-1.0-video, grok-imagine-1.0-video-20s, grok-imagine-1.0-video-30s | +| 9 | xb-sora2 | 58 (OpenAI Video) | 90 | 关闭 | https://open.hongniaoai.com/v1 | xb-sora2, ss-sora-2, sora-2(线路BF), sora-2-pro(线路BF), je-grok, grok-video-3(线路W), 全能视频2.0 等 | +| 10 | runway-explore | 58 (OpenAI Video) | 110 | 关闭 | http://127.0.0.1:8787 | seedance-2, gen4-turbo, wan-2.6-flash, kling-2.5-turbo-standard, gen4.5, happyhorse-1, wan-2.6, kling-2.5-turbo-pro, kling-2.6, wan-2.2-animate | +| 11 | ai-juhe-lk888 | 58 (OpenAI Video) | 35 | 关闭 | https://api.lk888.ai/api | sora-2, grok-video-3 | +| 12 | listenhub-images | 59 (ListenHub) | 40 | 开启 | https://api.marswave.ai/openapi | gemini-3-pro-image-preview, gemini-3.1-flash-image-preview, gpt-image-2（参考图 fallback；2026-06-10 因上游余额耗尽从 140 降到 40，充值后建议恢复 120） | +| 13 | siliconflow-images | 40 (SiliconFlow) | 0 | 开启 | https://api.siliconflow.cn | baidu/ERNIE-Image-Turbo, Qwen/Qwen-Image, Tongyi-MAI/Z-Image, Qwen/Qwen-Image-Edit-2509 | +| 14 | xgapi-images | 1 (OpenAI) | 130 | 关闭 | https://xgapi.top | gpt-image-2, gpt-image-2(线路XF), gr-image-2, nano-banana-pro（直接生图；后三者映射到上游 gpt-image-2） | +| 15 | manxiaobai-images | 1 (OpenAI) | 120 | 关闭 | https://api.manxiaobai.online | gpt-image-2（兜底）, gpt-image-2-1k/2k/4k（独家档位）；支持 /v1/images/edits 参考图（xgapi 被排除时本渠道为参考图首选） | +| 16 | manxiaobai-images-gemini | 24 (Gemini) | 55 | 关闭 | https://api.manxiaobai.online | gemini-3-pro-image-preview, gemini-3.1-flash-image-preview（Apexer 之后第二兜底） | +| 17 | manxiaobai-video | 58 (OpenAI Video) | 30 | 关闭 | https://api.manxiaobai.online | grok-imagine-video, grok-imagine-video-1.5-preview（独家）, grok-imagine-1.0-video（映射到 grok-imagine-video，作 Qilin 兜底）；`other=manxiaobai` | + +### AI 聚合站 / LK888 验证记录 + +2026-05-24 已接入为 OpenAI Video 类型 58 的 `lk888Provider`。该平台实际能力发现返回 38 个视频模型，但当前仅注册 `sora-2` 与 `grok-video-3` 两个模型；Seedance、Veo、Kling、Vidu、Wan、PixVerse、HappyHorse、Hailuo 等其他模型先不接入。完整调用和参数映射见 [AI 聚合站 / LK888 视频渠道接入文档](./lk888-video-api.md)。 + +| 能力 | 结果 | 说明 | +|------|------|------| +| 模型发现 | ✅ | `GET https://api.lk888.ai/api/v1/skills/models?type=video` 返回 38 个视频模型 | +| 余额查询 | ✅ | `GET /v1/skills/balance` 可用，测试 key 当前余额为 10.3 算力 | +| Sora 生成 | ✅ | 临时将渠道优先级调到 95 后，通过本项目 `/v1/videos` 提交 `model=sora-2`，任务 `task_Iqqit0P2UcMJAYNbzyrqcK5OrSKSNSXW` 命中渠道 11 并完成；测试后优先级已恢复 35 | +| Sora 下载 | ✅ | `GET /v1/videos/task_Iqqit0P2UcMJAYNbzyrqcK5OrSKSNSXW/content` 返回 `200 OK`，`Content-Type: video/mp4` | +| Grok 生成 | ✅ | 通过本项目 `/v1/videos` 提交 `model=grok-video-3`，任务 `task_mtkqjxwQRWoherMjTEJx0qfyyCPKSeep` 完成 | +| Grok 下载 | ✅ | `GET /v1/videos/task_mtkqjxwQRWoherMjTEJx0qfyyCPKSeep/content` 返回 `200 OK`，`Content-Type: video/mp4` | + +LK888 的媒体生成协议与 OpenAI Video 不同：创建任务走 `POST /v1/media/generate`，模型特定参数必须放入 `params` 对象；状态轮询走 `GET /v1/skills/task-status?task_id=...`，以 `is_final` / `state` 判断终态。当前 Provider 会把调用方常见的 `duration` / `seconds`、`orientation` / `aspect_ratio`、`images` / `image` / `input_reference` 转为 LK888 所需的 `params` 格式。 + +### Runway API 适配器验证记录 + +> **2026-05-28 状态**：Runway 渠道当前未在 Coolify 新服务器上配置。`constants.go` 中已注册 Kling 3.0/O3 系列模型（`kling-3.0-pro`、`kling-3.0-standard`、`kling-3.0-4k`、`kling-o3-pro`、`kling-o3-standard`、`kling-o3-4k`、`kling-2.6-motion-control`、`qilin-video-storyboard-pro`）但 Runway 适配器未就绪，模型列表未暴露给上游。 + +2026-05-19 已部署 `runway-api` 为同机私有 systemd 服务： + +| 项目 | 结果 | 说明 | +|------|------|------| +| 服务状态 | ✅ | `runway-api` 运行在 `127.0.0.1:8787`，不对公网暴露 | +| Token 状态 | ✅ | 使用 `/root/runway-api/.runwayml-token`，当前 token 到期时间为 2026-06-18 09:44:21 +08:00 | +| 可用性探测 | ✅ | `POST /can-start {"kind":"video","model":"seedance-2"}` 返回 `canStartNewTask=true` | +| new-api 接入 | ✅ | 新增 `runway-explore` 渠道，`other=runway`，通过 `X-API-Key` 调用本机适配器 | +| 模型暴露 | ✅ | `/v1/models` 已返回 Runway 渠道配置的 10 个视频模型 | + +Runway 生成任务必须按异步任务使用。new-api 提交到适配器的上游任务 ID 是 runway-api 的本地 `jobId`，轮询走 `GET /jobs/{jobId}`；适配器返回的 `/files/...` 结果会由 new-api 视频代理转发，不直接暴露 `127.0.0.1:8787` 给外部用户。 + +### Hongniao AI / xb-sora2 验证记录 + +2026-05-18 已通过远端服务 `http://206.119.182.61/v1/videos` 验证： + +| 能力 | 结果 | 说明 | +|------|------|------| +| 模型发现 | ✅ | `GET https://open.hongniaoai.com/v1/models` 返回 11 个真实模型；远端渠道注册 11 个真实模型 + 3 个文档兼容别名 | +| 文生视频 | ✅ | 通过本项目 `/v1/videos` 提交 `model=xb-sora2`，任务 `task_woE206uzgDCVrYTOkPhyyTtVP14GldbP` 完成，`progress=100`，返回视频 URL | +| 参考图字段 | ✅ | Provider 已把统一字段 `images` / `image` / `input_reference` / `image_url` 收敛为 Hongniao 下游 `images` 数组；带 1 张 `images` 参考图的任务 `task_A80f7CbmU4xxDSCn7Xi6fCLGJREpPW0C` 已完成，`progress=100`，返回视频 URL；Hongniao 文档说明最多 5 张 | + +Hongniao 的真实服务地址是 `https://open.hongniaoai.com/v1`，不是文档顶部示例里的 `https://localhost:3000/v1`。认证使用 `X-API-Key`。本项目对外仍保持 OpenAI Video 风格的 `/v1/videos` 和 `/v1/videos/{task_id}`，内部 Provider 转发到 Hongniao 的 `/videos/generate` 与 `/videos/{task_id}`。 + +### 937qq / 麒麟 Grok 视频验证记录 + +2026-05-15 已通过远端服务 `http://206.119.182.61/v1/videos` 验证： + +| 能力 | 结果 | 说明 | +|------|------|------| +| 多参数文生视频 | ✅ | JSON 直传 `seconds=6`、`size=1792x1024`、`quality=standard`，任务 `task_Hf8pVXgoCMlICXLRqIEHPPYbJQ4hIfZ3` 完成 | +| 单参考图 | ✅ | JSON `images` 数组 1 张，使用 base64 红底白圆参考图，任务 `task_RVKBuqOx4q9gWxPg2GWYSPMJv6UcoRyG` 完成；抽帧确认画面保留红底白圆 | +| 首尾帧 | ✅ | JSON `images` 数组 2 张，使用 base64 红圆首帧 + 蓝方块尾帧，任务 `task_9yHZfodDd4tVh6RWScooHkGUY59M6E9W` 完成；抽帧确认从红圆过渡到蓝方块 | + +注意：937qq 的 `/v1/videos` 支持 JSON 请求体，`images` 必须保持数组格式直传；不要转换成 multipart 表单字符串。2026-05-16 对照下载目录里的 `video_plugin_麒麟API_v1.1.9` 后确认，麒麟插件实际会把参考图转成 grok2api 风格的 `image_reference`。当前 Qilin Provider 已在内部把上游的 `images` / `image` / `image_urls` / `reference_images` / `reference_image_urls` / `image_url` / `file_paths` 自动补成 `image_reference`，上游仍统一传 `images`。 + +2026-05-15 追加验证复杂医生参考图请求： + +| 请求 | 结果 | 说明 | +|------|------|------| +| 仅传 `aspect_ratio=9:16` | ⚠️ | 任务 `task_TdDRc6FZeTOBvSWqWZS38zJ63OcKrBPp` 完成，但输出 688×464 横图，且人物身份未按参考图锁定 | +| 追加 `size=1024x1792` | ⚠️ | 任务 `task_iXjVIOUQei52gllPz8CwpJ49tok2YSMR` 完成，输出 464×688 竖图，但不是严格 9:16，人物身份仍未锁定 | + +处理策略：Qilin Provider 会把调用方的 `aspect_ratio=9:16` 自动补成 `size=720x1280`，`aspect_ratio=16:9` 自动补成 `size=1280x720`，`aspect_ratio=1:1` 自动补成 `size=1024x1024`。`ratio` 字段按同一规则兼容。2026-05-20 对照 `video_plugin_麒麟API_v1.1.12` 后，`4:3`、`3:4`、`21:9` 也按插件映射补成 `1152x864`、`864x1152`、`1680x720`，但这三个比例还没有完成生产视频抽检。这能提升方向命中率，但 937qq/Grok 对人物身份一致性和严格画幅比例仍是软约束。 + +部署后回归：任务 `task_e50dCra8ZpNeaOTN21rFw36ArEAZSG06` 仅传 `aspect_ratio=9:16` 未显式传 `size`，输出 464×688 竖图，确认内部映射生效。 + +2026-05-15 追加比例矩阵测试： + +| 请求参数 | 任务 | 结果 | +|----------|------|------| +| `aspect_ratio=9:16`（内部补 `size=1024x1792` 的旧映射） | `task_AwJVEUNCWCPhFgHhltKL4GgqWm7bhfk6` | 输出 464×688，约 2:3 | +| `size=1024x1792` | `task_e4Fa20yw4fviFLGGIwsxwy7eOzBUP8fI` | 输出 464×688，约 2:3 | +| `size=720x1280` | `task_VUWyg8YNg8B1v0dzSeaUCVziqfjLlSNw` | 输出 416×752，接近 9:16 | +| `size=1280x720` | `task_6xUfdSHpbEep7xk9ixLx2jsA0U1KY8dV` | 输出 752×416，接近 16:9 | +| `size=1080x1920` | - | 400 拒绝 | +| `size=576x1024` | - | 400 拒绝 | + +基于矩阵测试，内部映射已从 `1024x1792` / `1792x1024` 改为 `720x1280` / `1280x720`。 + +结合 xAI 官方文档和新版麒麟插件，Grok Imagine 视频原生参数是 `duration`、`aspect_ratio`、`resolution`。Qilin Provider 现在会让 `seconds` 和 `duration` 互补，默认补 `resolution=720p`，并按 `resolution` 补 937qq/Grok 原生 `quality` 字段。`grok-imagine-1.0-video` 传 20 / 30 秒时会自动改用下游传输模型 `grok-imagine-1.0-video-20s` / `grok-imagine-1.0-video-30s`；直接请求这两个模型时会锁定对应时长。 + +同一复杂医生参考图 query 的多比例复测（2026-05-15）： + +| 请求比例 | 任务 | 输出尺寸 | 结论 | +|----------|------|----------|------| +| `9:16` | `task_YQuoXWC21lI0fcP4zY9Ot4rpsfzO4YrM` | 416×752 | 接近 9:16，竖屏有效 | +| `16:9` | `task_6865jHA2i9bCQxaXJFG2OC3dLEkAB1fU` | 752×416 | 接近 16:9，横屏有效 | +| `1:1` | `task_PI8UQctgkRTUBmwx5814KAuH2ZFzkmSh` | 688×464 | 未按 1:1，落到默认横屏 | +| `2:3` | `task_NIQcraAEhz7gC5QpomT1bo6SzfXWJv41` | 688×464 | 未按 2:3，落到默认横屏 | +| `3:2` | `task_3weOBgVkdI02WLENxzpL13SWN2tQNL3N` | 688×464 | 未按 3:2，落到默认横屏 | +| `3:4` | `task_WsNdQrTNhz7VHG0ter4uVU5jyNHXuOno` | 688×464 | 未按 3:4，落到默认横屏 | +| `4:3` | `task_KnOMhWcX8GA3RsjxFlBWNNWY3QCgi2jU` | 688×464 | 未按 4:3，落到默认横屏 | + +结论：937qq/Grok 当前在本服务路径下只验证出 `9:16` 和 `16:9` 两个方向有效；官方其他比例桶没有被下游兼容层正确映射。 + +2026-05-16 部署后参考图转换回归： + +| 请求 | 任务 | 输出尺寸 | 结论 | +|------|------|----------|------| +| 只传上游统一字段 `images`，由 Qilin Provider 自动补 `image_reference` | `task_QFcwttd20S49mJUdM9Y7wTDNM5XhBdtM` | 720×1280 | 抽帧确认参考图身份、黑色服装、诊室场景和指膝腿动作生效 | +| `aspect_ratio=1:1`，由 Qilin Provider 自动补 `size=1024x1024` | `task_EocEzfLxfQGPZ04Y7nYKgga7l0hYnpZ6` | 960×960 | 方形比例生效，抽帧确认居中绿球画面正常 | +| 用户真实医生讲解 query 参考图优先改造 | `task_k6Id9R1pS3LbK22GHLLnDbUHFVPfsF5x` | 720×1280 | 抽帧确认灰发老年女性、黑色中式服装、诊室环境和指背/指脸/指膝腿动作保留较好 | + +比例当前结论：调用方优先传结构化参数，不要只写在 prompt 中。竖屏传 `aspect_ratio=9:16` 或 `size=720x1280`；横屏传 `aspect_ratio=16:9` 或 `size=1280x720`；方形传 `aspect_ratio=1:1` 或 `size=1024x1024`。`4:3`、`3:4`、`21:9` 已按新版插件映射透传，但暂不做生产视频效果承诺。 + +### Apexer 图片接口策略 + +Apexer 图片生成同时支持 Google 原生格式和 OpenAI 兼容格式，本服务拆成两个渠道以保持上游入口统一： + +| 上游入口 | 渠道 | 说明 | +|----------|------|------| +| `/v1beta/models/{model}:generateContent` | `apexer-images-gemini` | 透传 Gemini 原生 `contents` / `generationConfig.imageConfig` | +| `/v1/chat/completions` | `apexer-images-openai` | OpenAI 对话格式，支持多张 `image_url` 参考图 | +| `/v1/images/generations` | `apexer-images-openai` | OpenAI 图片格式，支持单张 `image` 参考图 | + +`/v1/images/generations` 的 `extra_body.google.image_config` 已在后端保留并透传，用于 Apexer 的 `aspect_ratio` / `image_size` 参数控制。 + +### ListenHub 图片渠道 + +ListenHub 已新增为独立渠道类型，适合接入 `https://api.marswave.ai/openapi/v1/images/generation` 这类非标准 OpenAI Images 上游。后台新增渠道时配置： + +| 配置项 | 值 | +|--------|----| +| 渠道 ID | 12 | +| 名称 | `listenhub-images` | +| 类型 | 59 (ListenHub) | +| 状态 | 手动禁用，待部署 `type=59` 代码后启用 | +| 优先级 | `120`，部署并启用后优先于现有图片渠道 | +| Base URL | `https://api.marswave.ai/openapi` | +| 支持模型 | `gemini-3-pro-image-preview`, `gemini-3.1-flash-image-preview`, `gpt-image-2` | +| 对外入口 | `/v1/images/generations` | +| 返回格式 | OpenAI Images 兼容，图片在 `data[].b64_json` | + +字段映射：`prompt` 原样透传；`gpt-image-2` 自动使用 `provider=openai`，其他模型默认 `provider=google`；`size` 会映射为 `imageConfig.aspectRatio`；`quality=1K/2K/4K` 会映射为 `imageConfig.imageSize`；`image` / `images` / `referenceImages` 会转换为 ListenHub 的 `referenceImages`。 + +当前部署检查：截至 2026-06-01，已创建线上渠道 `listenhub-images`（ID 12，`priority=120`），但保持手动禁用，因为当前线上运行版本尚未包含 `type=59` 适配代码。已使用 ListenHub Key 直连 Marswave 上游验证 3 个模型均成功返回 PNG base64；部署本次代码后启用渠道并执行 channel test。 + +### 渠道优先级策略 + +当前视频生成的优先级策略： + +1. **第一优先级：bltcy / 柏拉图**，`priority=100` +2. **Grok 专用：937qq / 麒麟 API**，建议 `priority=80`，注册 `grok-imagine-1.0-video`、`grok-imagine-1.0-video-20s`、`grok-imagine-1.0-video-30s` +3. **第二优先级：Apexer**，`priority=50` +4. **其他平台：兜底**，建议 `priority=10` + +系统配置： + +| 配置项 | 当前值 | 说明 | +|--------|--------|------| +| RetryTimes | 2 | 单次提交失败后最多重试 2 次，可覆盖 3 个优先级层级 | +| AutomaticDisableChannelEnabled | true | 渠道连续失败时自动禁用 | +| AutomaticEnableChannelEnabled | true | 渠道恢复后自动启用 | + +Apexer 的 OpenAI 视频格式模型名与本服务对外模型名不同，当前通过渠道 `model_mapping` 做转换： + +```json +{ + "veo3.1": "veo3.1_relaxed", + "veo3.1-fast": "veo3.1_fast", + "veo3.1-pro": "veo3.1_pro", + "veo3.1-4k": "veo3.1_relaxed_4k", + "veo3.1-fast-4k": "veo3.1_fast_4k", + "veo3.1-pro-4k": "veo3.1_pro_4k", + "veo3.1-components": "veo3.1_relaxed", + "veo3.1-fast-components": "veo3.1_fast", + "veo3.1-components-4k": "veo3.1_relaxed_4k", + "veo3.1-fast-components-4k": "veo3.1_fast_4k" +} +``` + +Apexer 新版 `/v1/videos` 需要 `type` 参数；本服务在 Provider 层自动推断并补齐： + +| 调用方请求 | Apexer type | +|-----------|-------------| +| 不传 `images` | 1（文生视频） | +| 传 1-2 张 `images` | 2（首帧/首尾帧） | +| components 模型或 3 张 `images` | 3（垫图参考） | + +调用方不需要显式传 Apexer 的 `type`，也不需要知道下游模型名使用下划线。 + +自动故障转移已验证（2026-05-13）： + +1. `veo3.1-fast` 在 bltcy 侧因上游 BUG 返回 503 → 自动重试落到 Apexer #4，生成 1280×720 视频成功（82s，扣 1,200,000 额度）。 +2. `MiniMax-Hailuo-02` 直接走 bltcy #1，108s 生成 1366×768 视频成功（扣 1,600,000 额度）。 +3. Apexer 余额已于 2026-05-12 充值，恢复可用。 + +--- + +## 已知问题 + +1. **bltcy 上游 `.→_` 路由 BUG（高优先）**：bltcy 内部 distributor 在模型名查找前会把 `.` 替换为 `_`，但其注册表里的同名条目并未做对应处理，导致 `veo3.1*` / `sora-2` 等含点号或形如 `sora-2` 的模型在 bltcy 上游均返回 `model_not_found: veo_3_1-* / sora_2`。当前对策： + - **无点号视频模型走 bltcy**：`MiniMax-Hailuo-02`、`MiniMax-Hailuo-2.3*`、`doubao-seedance-*` 等可直接生成。 + - **`veo3.1*` 走 Apexer**：通过 RetryTimes=2 自动从 bltcy → Apexer 故障转移；Apexer 模型名映射在渠道 `model_mapping` 内（`veo3.1`→`veo3.1_relaxed` 等）。 +2. **xgapi 账号资源受限**：xgapi #5 已配置但当前账号： + - `veo3.1-lite` 在 xgapi 内部也存在 `.→_` 距离形变（`veo_3_1-lite`），返回 `无可用渠道（distributor）`。 + - `sora-2` 模型可达后端但其 sora 反代会话获取失败（`无法获取有效的 SessionID 尝试了 6 次`）。 + - 暂作占位/兜底，不建议放主路径。 +3. **gemini-3.1-flash-image-preview 渠道选择异常（已修复 2026-06-10）**：根因是 `common.ImageGenerationModels` 缺少横线命名的 gemini-3.x，OpenAI 类型渠道对 `/v1/images/generations` 被端点过滤排除，实际只剩 ListenHub 一个候选。现已加入分类列表，并由 channel 6（Apexer）通过 model_mapping 承接横线名。 +4. **web/classic dist 为空（上游兼容性）**：`web/classic/src/index.jsx:23` 显式导入 `@douyinfe/semi-ui/dist/css/semi.css`，但当前 `@douyinfe/semi-ui@2.97.0` 的 `package.json` `exports` 字段不再暴露该深路径，Vite 构建失败 → `dist/index.html` 为 0 字节。生产二进制嵌入的就是这个空文件。由于主使用 default 主题，不影响业务，但访问 `/?theme=classic` 会拿到空 HTML。修复路径：删除该冗余 import（`vitePluginSemi({cssLayer:true})` 本来就会接管 semi 样式）。 +5. **4G 内存构建风险**：服务器只有 4G 内存，前端构建时可能 OOM。已添加 4G Swap 作为保护，但构建速度会变慢。**当前流程已固定为本地 cross-compile + rsync 二进制**，避开服务器构建。 + +--- + +### 漫小白 / manxiaobai 渠道接入（2026-06-11） + +上游 `https://api.manxiaobai.online`（new-api 同构站，OpenAI 兼容 + Gemini 原生双入口）。**2026-06-11 已激活**：渠道 15/16/17 使用用户自注册充值账号的 key（凭据在 `docs/deployment.local.md`；早期 `vipergw2026` 自注册账号保留备用），图片直出/参考图编辑/Gemini 原生/视频全流程均已真实验证。 + +| 能力 | 模型 | 上游采购价 | 接入方式 | +|------|------|-----------|----------| +| 图片直出/编辑 | gpt-image-2 / -1k / -2k / -4k | $0.03 / 0.05 / 0.06 / 0.07 | channel 15（type 1），标准 `/v1/images/generations` + `/v1/images/edits` | +| Gemini 图片 | gemini-3-pro-image-preview, gemini-3.1-flash-image-preview | $0.125 / $0.1 | channel 16（type 24），Gemini 原生 `/v1beta/...:generateContent` | +| Grok 视频 | grok-imagine-video（10s，文生+参考图）, grok-imagine-video-1.5-preview（必须参考图，10/15s） | $0.2 / $0.35 | channel 17（type 58），`manxiaobaiProvider`（JSON `/v1/videos`，seconds 必须字符串） | + +接入要点： + +- quota 报错文案与众不同：images 端点 403 + `insufficient_user_quota` +「当前模型暂时不可用，请稍后重试或联系管理员」；videos/gemini 端点 403 +「用户额度不足」。已把 `insufficient_user_quota` 和「当前模型暂时不可用」补进运营设置 `UpstreamQuotaErrorKeywords`。 +- 视频协议细节：`seconds` 传数字会被 400 拒绝（必须字符串）；尺寸仅支持 `1792x1024` / `1024x1792`；任务流程 POST `/v1/videos` → GET `/v1/videos/{id}` → GET `/v1/videos/{id}/content`。`manxiaobaiProvider` 已做 seconds 字符串化与尺寸归一。 +- `grok-imagine-video-1.5-preview` 必须带参考图，上游另有 `/v1/video-reference-images` 预上传接口（base64 带 `data:image/png;base64,` 前缀）——充值后实测确认 provider 是否需要补预上传逻辑。 +- 图片/视频结果 URL（`/generated/...`）只保留约 2 小时，下游需及时转存。 +- 2026-06-11 充值激活后已完成渠道 15/16/17 真实验证（详见验证记录表）。可选优化：把 channel 15 提到 130 与 xgapi 同桶做权重轮换（当前为 120 第一兜底）。 +- 遗留待办：`grok-imagine-video-1.5-preview` 必须带参考图且需走 `/v1/video-reference-images` 预上传接口，provider 尚未实现预上传逻辑，使用前需实测补齐。 + +## 多渠道故障转移评审 + +多渠道互备/自动切换的实现评审、风险清单和新平台接入 Checklist 见 [channel-failover-review.md](./channel-failover-review.md)（2026-06-10）。接入新图片/视频平台前请先过一遍其中的 Checklist。 + +## 后续优化建议 + +1. **绑定域名 + HTTPS**：使用 Let's Encrypt 免费证书 +2. **安装 Redis**：提升缓存性能，`apt install redis && systemctl enable redis` +3. **切换 PostgreSQL**：流量增长后可升级数据库 +4. **添加更多中转站**：在 Provider 模式下，只需在 `relay/channel/task/openaivideo/` 下新增一个 provider 文件即可 +5. **下游余额监控告警**：余额总览接口 `GET /api/channel/balance_overview` 已可一次查全下游余额（详见 [channel-balance-query.md](./channel-balance-query.md)）；可接定时任务，在余额低于阈值或返回 `console_only`/`spend_only` 估算剩余偏低时主动推送告警，提前充值。给 bltcy/apexer/xgapi/qilin 套壳站配 `setting.balance_query.mode=newapi_console`（账密）即可显示真实钱包余额而非仅累计消费。 diff --git a/docs/grok-video-api.md b/docs/grok-video-api.md new file mode 100644 index 000000000000..fdeddef46135 --- /dev/null +++ b/docs/grok-video-api.md @@ -0,0 +1,408 @@ +# Grok 视频生成 API 调用文档 + +最后更新：2026-06-06 + +> **2026-05-28 回归验证**：`grok-imagine-1.0-video` 重新通过真实验证（task `task_0N4mwgTkQS8mlV8iYiTa1D385u2o2CRf`，产出 3.8MB MP4）。注意该模型**仅支持以下尺寸**：`720x1280`、`1280x720`、`1024x1024`、`1024x1792`、`1792x1024`，传入 `1920x1080` 会报错。`grok-imagine-1.0-video-20s` 今天返回 `model_not_found`（渠道未注册该模型）。 + +本文档主要描述通过本服务调用 937qq / Qilin 的 Grok 视频模型。调用方使用统一的 OpenAI Video 兼容入口，不需要知道 937qq / Qilin 的令牌、接口地址或私有字段。 + +## 快速结论 + +生产调用时按下面规则传参： + +| 目标 | 推荐写法 | +|------|----------| +| 创建视频 | `POST /v1/videos`，JSON 请求体 | +| 参考图 | 传 `images: ["https://..."]`，保持数组 | +| 本地参考图 | 优先上传成公网 PNG URL；小图可传 `data:image/...;base64,...` | +| 时长 | 支持 `6`、`10`、`15`、`20`、`30` 秒；20/30 秒会自动映射到长时长传输模型 | +| 竖屏 | `aspect_ratio: "9:16"` 或 `size: "720x1280"` | +| 横屏 | `aspect_ratio: "16:9"` 或 `size: "1280x720"` | +| 方形 | `aspect_ratio: "1:1"` 或 `size: "1024x1024"` | +| 人物参考图 | Prompt 必须显式复述人物视觉特征，并排除错误身份 | +| 医生参考图 | 不要用 `him` / `his` 描述医生；如果参考图是女性，要写 `same elderly woman doctor` | + +本服务会自动把上游的 `images` 转成 Qilin/Grok 下游更偏好的 `image_reference`。调用方不要直接依赖 `image_reference`，除非是在做供应商级排查。 + +AI 聚合站 / LK888 也已接入 `grok-video-3`，该线路使用 `/v1/media/generate` 和 `/v1/skills/task-status`，与 Qilin 的 `grok-imagine-*` 不是同一个下游协议。LK888 Grok 的调用、参数映射和已验证任务见 [AI 聚合站 / LK888 视频渠道接入文档](./lk888-video-api.md)。 + +## 连接信息 + +| 项目 | 值 | +|------|-----| +| Base URL | `http://192.129.209.36:3001/v1` | +| 模型 | `grok-imagine-1.0-video`、`grok-imagine-1.0-video-20s`、`grok-imagine-1.0-video-30s` | +| 认证方式 | HTTP Header `Authorization: Bearer ` | +| 内部测试 API Key | `EW93ybOP6Zr1axAPYNEu8VpehQzdTkZBTATszAGYEDiwpCmJ` | + +## 接口 + +| 接口 | 方法 | 说明 | +|------|------|------| +| `/v1/videos` | POST | 创建视频任务 | +| `/v1/videos/{task_id}` | GET | 查询任务状态和结果 | +| `/v1/videos/{task_id}/content` | GET | 下载视频文件 | + +旧入口 `/v1/video/generations` 仍兼容，新接入方统一使用 `/v1/videos`。 + +## 生产推荐 Payload + +### 单参考图生成视频 + +```json +{ + "model": "grok-imagine-1.0-video", + "prompt": "Use the provided reference image as the exact character identity: an elderly Chinese woman doctor with gray hair, black traditional Chinese medical clothing, in an indoor clinic. Keep her face, age, gray hair, black outfit, and clinic room consistent. No subtitles, no on-screen text.", + "images": [ + "https://example.com/reference.png" + ], + "mode": "r2v", + "strength": 0.9, + "aspect_ratio": "9:16", + "size": "720x1280", + "seconds": "6", + "duration": 6, + "resolution": "720p", + "quality": "high" +} +``` + +### 真实医生讲解 Query 推荐写法 + +这是对真实医生参考图 query 的当前推荐版本，重点是保留参考图身份。 + +```json +{ + "model": "grok-imagine-1.0-video", + "prompt": "Use the provided reference image as the exact doctor identity. The doctor must remain the same elderly Chinese woman from the reference image throughout the whole video: gray hair, elderly Chinese female face, black traditional Chinese medical clothing, calm indoor clinic room. Do not change her into a man. Do not change her into a white-coat western doctor. Create a vertical 9:16 medical explainer video with five fast distinct editorial shots. Use real hard cuts between shots, not camera tilt, pan, push, pull, reframing, or vertical movement pretending to be cuts. In every shot, the same elderly woman doctor stands beside one adult patient, speaks very fast but naturally in Mandarin Chinese directly to camera, and clearly points to one body part on the patient. No subtitles, no captions, no on-screen text, no written Chinese characters. Shot 1: the same woman doctor points at the patient's neck. Hard cut. Shot 2: the same woman doctor points at the patient's back. Hard cut. Shot 3: the same woman doctor points at the patient's head. Hard cut. Shot 4: the same woman doctor points at the patient's face. Hard cut. Shot 5: the same woman doctor points at the patient's knee. Keep her gray hair, face, age, black outfit, clinic room, and calm traditional Chinese medicine style consistent in all five shots. Spoken Mandarin line, delivered very fast and naturally: 日常养生贵在规律，三餐定时清淡饮食，少重油重盐与甜食。每日保证七至八小时睡眠，避免长期熬夜损伤脏腑。坚持适度运动，快走、慢跑均可增强体质。遇事放平心态，少生气少焦虑，情绪平和更益身心。多喝温水，远离久坐，养成良好习惯，才能长久守护身体健康。", + "images": [ + "https://cdn.vdgen.shop/qy-tests/scene_01_524155a2.png" + ], + "mode": "r2v", + "strength": 0.9, + "aspect_ratio": "9:16", + "size": "720x1280", + "seconds": "10", + "duration": 10, + "resolution": "720p", + "quality": "high" +} +``` + +实测任务 `task_k6Id9R1pS3LbK22GHLLnDbUHFVPfsF5x`：输出 `720×1280`、约 10 秒。抽帧确认参考图保留较好，医生保持为灰发老年中国女性、黑色中式服装、室内诊室环境，并出现指背、指脸、指膝腿等动作。 + +## 关键参数 + +| 参数 | 类型 | 必填 | 推荐/说明 | +|------|------|------|-----------| +| `model` | string | 是 | 推荐 `grok-imagine-1.0-video`。也可直接传 `grok-imagine-1.0-video-20s` / `grok-imagine-1.0-video-30s` | +| `prompt` | string | 是 | 建议英文描述；中文台词可放在 prompt 内 | +| `images` | array[string] | 否 | 推荐参考图字段。支持公网 URL 或 `data:image/...;base64,...`；新版插件上限为 7 张 | +| `mode` | string | 否 | 参考图任务建议传 `r2v` | +| `strength` | number | 否 | 参考图任务建议传 `0.9`；这是软约束 | +| `aspect_ratio` | string | 否 | 推荐 `9:16`、`16:9`、`1:1`；新版插件还映射 `4:3`、`3:4`、`21:9` | +| `size` | string | 否 | 推荐 `720x1280`、`1280x720`、`1024x1024`；宽高比字段会自动映射 | +| `seconds` | string | 否 | 支持 `6`、`10`、`15`、`20`、`30` | +| `duration` | integer | 否 | 可与 `seconds` 同传；不传时服务会从 `seconds` 自动补。基础模型传 20/30 秒时会自动转下游长时长模型 | +| `resolution` | string | 否 | 推荐 `720p`；不传时服务默认补 `720p` | +| `quality` | string | 否 | 推荐 `high`；不传时服务按 `resolution` 自动补 | + +兼容但不推荐作为业务主路径的字段：`image`、`image_urls`、`reference_images`、`reference_image_urls`、`image_url`。这些字段会被服务端尽量转换成下游参考图结构，但新调用方统一使用 `images`。 + +不要这样传： + +| 写法 | 原因 | +|------|------| +| `image: {"url": "..."}` | 937qq/Qilin 要求 `image` 是字符串，不接受对象 | +| `size: "9:16"` | `size` 只接受像素尺寸，不接受比例字符串 | +| prompt 里写 `@Image1` | 937qq/Qilin 这条链路会报 reference placeholder 错误 | +| 只把比例写在 prompt 里 | 实测不控制真实视频编码尺寸 | +| 人物参考图只写 `preserve identity` | 太弱，容易漂移 | +| 女性参考图里使用 `him` / `his` | 会把角色拉向男性 | + +## 服务端自动转换 + +为了让调用方不感知下游差异，本服务会自动做这些处理： + +| 调用方传入 | 服务端处理 | +|------------|------------| +| `images` / `image` / `image_urls` 等参考图字段 | 补 Qilin/Grok 原生 `image_reference` | +| `seconds: "10"` 且未传 `duration` | 补 `duration: 10` | +| `duration: 20` 且模型为 `grok-imagine-1.0-video` | 下游模型改为 `grok-imagine-1.0-video-20s`，并锁定 20 秒 | +| `duration: 30` 且模型为 `grok-imagine-1.0-video` | 下游模型改为 `grok-imagine-1.0-video-30s`，并锁定 30 秒 | +| 直接传 `grok-imagine-1.0-video-20s` / `30s` | 分别锁定 `duration` 和 `seconds` 为 20 / 30 | +| 未传 `resolution` | 补 `resolution: "720p"` | +| 未传 `quality` 且分辨率是 HD 档 | 补 `quality: "high"` | +| `aspect_ratio: "9:16"` 且未传 `size` | 补 `size: "720x1280"` | +| `aspect_ratio: "16:9"` 且未传 `size` | 补 `size: "1280x720"` | +| `aspect_ratio: "1:1"` 且未传 `size` | 补 `size: "1024x1024"` | +| `aspect_ratio: "4:3"` / `"3:4"` / `"21:9"` 且未传 `size` | 分别补 `1152x864` / `864x1152` / `1680x720` | +| `ratio` | 按 `aspect_ratio` 同样规则兼容 | + +下游实际使用的参考图结构类似： + +```json +{ + "image_reference": [ + { + "type": "image_url", + "image_url": { + "url": "https://example.com/reference.png" + } + } + ] +} +``` + +这是内部实现细节，调用方继续传 `images`。 + +## 参考图写作规范 + +参考图能否生效，主要取决于两件事：图片是否被下游接收，以及 prompt 是否把参考图中的关键视觉锚点写清楚。 + +### 推荐写法 + +在 prompt 前半段固定写： + +```text +Use the provided reference image as the exact character identity. +The person must remain the same [age/gender/ethnicity] from the reference image: +[hair], [face/age], [clothing], [room/environment]. +Do not change [him/her] into [common wrong identity]. +Keep [face], [hair], [clothing], and [environment] consistent. +``` + +医生参考图示例： + +```text +Use the provided reference image as the exact doctor identity. +The doctor must remain the same elderly Chinese woman from the reference image: +gray hair, elderly Chinese female face, black traditional Chinese medical clothing, +calm indoor clinic room. +Do not change her into a man. +Do not change her into a white-coat western doctor. +``` + +### 不推荐写法 + +```text +Use the provided reference image as the doctor identity and preserve the same doctor. +``` + +这句话太泛，模型容易生成白大褂医生、男性医生或完全重写人物。 + +## 参考图输入建议 + +| 输入方式 | 建议 | +|----------|------| +| 公网 URL | 推荐。确保 937qq/Grok 下游可直接访问 | +| `data:image/png;base64,...` | 可用于小图或快速验证 | +| 大 base64 | 不推荐，容易触发网关 body size 限制 | +| 本地 JPEG | 建议先转 PNG，再上传公网 URL | +| 真实人物图片 | 建议 PNG，长边控制到 1280 左右，文件控制到约 1.5MB | + +下载目录里的麒麟插件会把本地图片转 PNG、压缩到约 1.5MB，再上传 OSS 得到 URL。这说明对真实人物参考图，PNG 公网 URL 是更稳的生产路径。 + +## 比例与尺寸 + +| 目标比例 | 推荐参数 | 已验证结果 | +|----------|----------|------------| +| 竖屏 | `aspect_ratio: "9:16"` 或 `size: "720x1280"` | 输出过 `720×1280`、`416×752` | +| 横屏 | `aspect_ratio: "16:9"` 或 `size: "1280x720"` | 输出过 `752×416` | +| 方形 | `aspect_ratio: "1:1"` 或 `size: "1024x1024"` | 输出过 `960×960` | +| 4:3 | `aspect_ratio: "4:3"` 或 `size: "1152x864"` | 按新版插件映射透传，未做生产实测承诺 | +| 3:4 | `aspect_ratio: "3:4"` 或 `size: "864x1152"` | 按新版插件映射透传，未做生产实测承诺 | +| 21:9 | `aspect_ratio: "21:9"` 或 `size: "1680x720"` | 按新版插件映射透传，未做生产实测承诺 | + +注意： + +- `size` 必须是像素尺寸，不要传 `"9:16"`。 +- 只在 prompt 写 “vertical 9:16” 不可靠。 +- `4:3`、`3:4`、`21:9` 已按新版麒麟插件映射透传，但还没有像 9:16 / 16:9 / 1:1 一样完成生产视频抽检。 +- 下游会按自身编码规格缩放，不能保证像素级等于传入尺寸。 + +## 创建任务 + +```bash +curl -s "http://192.129.209.36:3001/v1/videos" \ + -H "Authorization: Bearer $API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "grok-imagine-1.0-video", + "prompt": "Use the provided reference image as the exact character identity: an elderly Chinese woman doctor with gray hair, black traditional Chinese medical clothing, in an indoor clinic. Keep her face, age, gray hair, black outfit, and clinic room consistent. No subtitles, no on-screen text.", + "images": [ + "https://example.com/reference.png" + ], + "mode": "r2v", + "strength": 0.9, + "aspect_ratio": "9:16", + "size": "720x1280", + "seconds": "6", + "duration": 6, + "resolution": "720p", + "quality": "high" + }' +``` + +创建响应： + +```json +{ + "id": "task_xxx", + "task_id": "task_xxx", + "object": "video", + "model": "grok-imagine-1.0-video", + "status": "queued", + "progress": 0, + "created_at": 0 +} +``` + +## 查询任务 + +```bash +curl -s "http://192.129.209.36:3001/v1/videos/" \ + -H "Authorization: Bearer $API_KEY" +``` + +完成响应： + +```json +{ + "id": "task_xxx", + "object": "video", + "model": "grok-imagine-1.0-video", + "status": "completed", + "progress": 100, + "video_url": "https://example.com/video.mp4", + "created_at": 1778855922, + "completed_at": 1778855936 +} +``` + +失败响应： + +```json +{ + "id": "task_xxx", + "object": "video", + "model": "grok-imagine-1.0-video", + "status": "failed", + "progress": 0, + "error": { + "message": "generation failed", + "code": "generation_error" + } +} +``` + +## 下载视频 + +```bash +curl -L "http://192.129.209.36:3001/v1/videos//content" \ + -H "Authorization: Bearer $API_KEY" \ + -o output.mp4 +``` + +也可以直接下载查询响应中的 `video_url`。 + +## Python 示例 + +```python +import time +import requests + +BASE_URL = "http://192.129.209.36:3001/v1" +API_KEY = "YOUR_API_KEY" + +headers = { + "Authorization": f"Bearer {API_KEY}", + "Content-Type": "application/json", +} + +payload = { + "model": "grok-imagine-1.0-video", + "prompt": ( + "Use the provided reference image as the exact character identity: " + "an elderly Chinese woman doctor with gray hair, black traditional " + "Chinese medical clothing, in an indoor clinic. Keep her face, age, " + "gray hair, black outfit, and clinic room consistent. No subtitles, " + "no on-screen text." + ), + "images": ["https://example.com/reference.png"], + "mode": "r2v", + "strength": 0.9, + "aspect_ratio": "9:16", + "size": "720x1280", + "seconds": "6", + "duration": 6, + "resolution": "720p", + "quality": "high", +} + +submit = requests.post(f"{BASE_URL}/videos", headers=headers, json=payload).json() +task_id = submit["task_id"] + +while True: + time.sleep(10) + result = requests.get(f"{BASE_URL}/videos/{task_id}", headers=headers).json() + print(result["status"], result.get("progress", 0)) + + if result["status"] == "completed": + print(result["video_url"]) + break + if result["status"] == "failed": + raise RuntimeError(result.get("error", {}).get("message", "generation failed")) +``` + +## 能力边界 + +| 能力 | 当前结论 | +|------|----------| +| 文生视频 | 支持 | +| 单参考图 | 支持，生产推荐路径 | +| 首尾帧 | 支持，但 Grok 对“精确首尾帧”不如专门视频模型稳定 | +| 多张参考图 | 下游接受多图结构，但生产建议先用单张主参考图 | +| 人物身份一致性 | 可明显提升，但仍是软约束 | +| 多段硬切镜头 | 不稳定，可能生成连续动作 | +| 中文口播逐字准确 | 不稳定 | +| 禁止字幕/文字 | 不稳定，仍可能生成文字 | + +如果业务必须严格五个镜头、严格硬切、严格台词，建议拆成多个短视频任务分别生成，再在业务侧拼接。不要指望单条 10 秒 prompt 同时稳定满足人物锁定、五段硬切、多人互动、长中文口播。 + +## 已验证任务 + +| 用例 | task_id | 结果 | +|------|---------|------| +| 真实医生 query 参考图优先改造 | `task_k6Id9R1pS3LbK22GHLLnDbUHFVPfsF5x` | 输出 720×1280；抽帧确认灰发老年女性、黑色中式服装、诊室环境和指背/指脸/指膝腿动作保留较好 | +| 部署后 `images` 自动转 `image_reference` | `task_QFcwttd20S49mJUdM9Y7wTDNM5XhBdtM` | 输出 720×1280；抽帧确认参考图身份、服装、场景和指膝腿动作生效 | +| 麒麟插件 `image_reference` schema | `task_x1uthTcUUSc2K2KtwUEBUpWEB36Wb0Al` | 输出竖屏；抽看视频，参考图身份和指腿动作保留明显 | +| `aspect_ratio=1:1` 自动映射 | `task_EocEzfLxfQGPZ04Y7nYKgga7l0hYnpZ6` | 输出 960×960；方形比例生效 | +| base64 单参考图 | `task_RVKBuqOx4q9gWxPg2GWYSPMJv6UcoRyG` | 抽帧确认参考图生效 | +| base64 首尾帧 | `task_9yHZfodDd4tVh6RWScooHkGUY59M6E9W` | 抽帧确认首尾帧生效 | + +## 排障 + +### 参考图人物不像 + +检查： + +- 是否传了 `images` 数组。 +- 图片 URL 是否公网可访问。 +- prompt 是否明确写了年龄、性别、发型、服装、环境。 +- 是否错误使用了 `him` / `his` 等男性代词。 +- 是否明确排除了常见错误身份，例如 `man`、`white-coat western doctor`。 +- 是否把任务写得过复杂。复杂分镜、长口播、多人互动都会稀释参考图约束。 + +### 比例不对 + +检查： + +- 竖屏传 `aspect_ratio: "9:16"` 或 `size: "720x1280"`。 +- 横屏传 `aspect_ratio: "16:9"` 或 `size: "1280x720"`。 +- 方形传 `aspect_ratio: "1:1"` 或 `size: "1024x1024"`。 +- 不要只在 prompt 中写比例。 + +### 本地图片怎么传 + +生产建议：本地图片先转 PNG，长边约 1280，文件约 1.5MB 内，上传公网 URL，再放入 `images`。小图可以转成 `data:image/png;base64,...` 直接放入 `images`。 diff --git a/docs/lk888-video-api.md b/docs/lk888-video-api.md new file mode 100644 index 000000000000..9a4f6c26afe0 --- /dev/null +++ b/docs/lk888-video-api.md @@ -0,0 +1,167 @@ +# AI 聚合站 / LK888 视频渠道接入文档 + +最后更新：2026-05-28 + +> **2026-05-28 状态**：`grok-video-3` 今天上游 LK888 返回"参数验证失败"，2026-05-24 曾验证可用，疑似上游临时问题。建议上游当前改用 `grok-imagine-1.0-video`（937qq/Qilin 链路，已验证可用）。详情见 [api-usage.md](./api-usage.md)。 + +本文档描述通过本项目统一 OpenAI Video 入口调用 AI 聚合站（LK888）的视频模型。当前只接入并暴露 Sora 与 Grok 两个模型，其他视频模型已完成能力发现，但暂不注册到生产渠道。 + +## 快速结论 + +| 项目 | 值 | +|------|-----| +| 本项目入口 | `POST /v1/videos`、`GET /v1/videos/{task_id}` | +| 渠道类型 | `58` / OpenAI Video | +| 渠道名 | `ai-juhe-lk888` | +| 下游 Base URL | `https://api.lk888.ai/api` | +| 下游认证 | `Authorization: Bearer ` | +| 下游创建任务 | `POST /v1/media/generate` | +| 下游查询任务 | `GET /v1/skills/task-status?task_id={task_id}` | +| 当前暴露模型 | `sora-2`、`grok-video-3` | + +## 已启用模型 + +| 模型 | 能力 | 关键参数 | 当前用途 | +|------|------|----------|----------| +| `sora-2` | 文生视频、图生视频 | `duration=4/8/12`，`orientation=portrait/landscape`，`input_reference` | Sora 备用/验证线路 | +| `grok-video-3` | 文生视频、图生视频、首帧参考 | `duration=6/10`，`aspect_ratio=2:3/3:2/1:1`，`size=720P/1080P`，`images` | Grok 视频线路 | + +LK888 返回的视频模型总数为 38 个，包含 Seedance、Veo、Kling、Vidu、Wan、PixVerse、HappyHorse、Hailuo 等。当前不注册这些模型，后续需要时再按模型逐个补参数映射、计费和回归记录。 + +## 请求映射 + +调用方继续使用本项目 OpenAI Video 风格请求： + +```json +{ + "model": "grok-video-3", + "prompt": "A small red cube rotates slowly on a clean white studio background.", + "duration": 6, + "orientation": "landscape" +} +``` + +LK888 下游要求媒体生成参数放入 `params` 对象。Provider 会自动转换为： + +```json +{ + "model": "grok-video-3", + "prompt": "A small red cube rotates slowly on a clean white studio background.", + "params": { + "duration": "6", + "aspect_ratio": "3:2" + } +} +``` + +通用转换规则： + +| 调用方字段 | LK888 字段 | +|------------|------------| +| `duration` / `seconds` | `params.duration`，字符串 | +| `orientation=landscape` | Sora: `params.orientation=landscape`；Grok: `params.aspect_ratio=3:2` | +| `orientation=portrait` | Sora: `params.orientation=portrait`；Grok: `params.aspect_ratio=2:3` | +| `aspect_ratio=16:9` / `size=1280x720` | Grok: `params.aspect_ratio=3:2` | +| `aspect_ratio=9:16` / `size=720x1280` | Grok: `params.aspect_ratio=2:3` | +| `images` | `params.images` | +| `image` / `input_reference` / `image_url` | `params.images` | +| `params` | 原样合并，优先级高于顶层兼容字段 | + +## Sora 示例 + +```bash +curl -s "http://192.129.209.36:3001/v1/videos" \ + -H "Authorization: Bearer $API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "sora-2", + "prompt": "A calm sunrise over a mountain lake, cinematic, slow camera movement, no text.", + "duration": 4, + "orientation": "landscape" + }' +``` + +已验证任务： + +| 项目 | 值 | +|------|-----| +| 任务 ID | `task_Iqqit0P2UcMJAYNbzyrqcK5OrSKSNSXW` | +| 状态 | `completed` | +| 命中渠道 | `ai-juhe-lk888`，上游任务 ID `26081355` | +| 下载 | `GET /v1/videos/task_Iqqit0P2UcMJAYNbzyrqcK5OrSKSNSXW/content` 返回 `200 OK`，`Content-Type: video/mp4` | + +## Grok 示例 + +```bash +curl -s "http://192.129.209.36:3001/v1/videos" \ + -H "Authorization: Bearer $API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "grok-video-3", + "prompt": "A small red cube rotates slowly on a clean white studio background, simple product demo, no text.", + "duration": 6, + "orientation": "landscape" + }' +``` + +已验证任务： + +| 项目 | 值 | +|------|-----| +| 任务 ID | `task_mtkqjxwQRWoherMjTEJx0qfyyCPKSeep` | +| 状态 | `completed` | +| 下载 | `GET /v1/videos/task_mtkqjxwQRWoherMjTEJx0qfyyCPKSeep/content` 返回 `200 OK`，`Content-Type: video/mp4` | + +## Sora 路由说明 + +`sora-2` 与现有 Hongniao 渠道重名。当前远端将 LK888 渠道设置为低优先级备用；生产默认仍优先走现有 Sora 主路径。测试 LK888 Sora 时临时把 LK888 渠道优先级从 `35` 调到 `95`，任务完成后已恢复为 `35`。 + +## 任务查询与下载 + +查询： + +```bash +curl -s "http://192.129.209.36:3001/v1/videos/$TASK_ID" \ + -H "Authorization: Bearer $API_KEY" +``` + +完成响应会返回本项目代理地址： + +```json +{ + "id": "task_xxx", + "object": "video", + "model": "grok-video-3", + "status": "completed", + "progress": 100, + "video_url": "http://192.129.209.36:3001/v1/videos/task_xxx/content" +} +``` + +下载： + +```bash +curl -L "http://192.129.209.36:3001/v1/videos/$TASK_ID/content" \ + -H "Authorization: Bearer $API_KEY" \ + -o output.mp4 +``` + +Provider 会保存 LK888 的真实 `result_url`，对外展示和下载统一走本项目 `/content` 代理，避免调用方直接依赖下游 CDN 地址。 + +## 实现位置 + +| 文件 | 说明 | +|------|------| +| `relay/channel/task/openaivideo/lk888.go` | LK888 submit/query/参数归一化 | +| `relay/channel/task/openaivideo/provider.go` | `other=lk888` 或 `api.lk888.ai` 选择 LK888 provider | +| `relay/channel/task/openaivideo/adaptor.go` | LK888 结果 URL 对外转成本项目 `/content` 代理 | +| `relay/channel/task/openaivideo/constants.go` | 当前只新增 `grok-video-3`；`sora-2` 已存在 | + +## 注意事项 + +- 下游能力发现接口：`GET https://api.lk888.ai/api/v1/skills/models?type=video`。 +- 下游模型详情接口：`GET /v1/skills/models/{model_name}`，新增模型必须先看参数定义。 +- 下游价格接口：`GET /v1/skills/models/{model_name}/pricing?status=active`。 +- 付费接口调用前可查余额：`GET /v1/skills/balance`（返回 `balance` 算力 + `api_key_quota.used`）。本项目已把该接口接入统一余额查询，可经 `GET /api/channel/balance_overview` 一并查出，详见 [channel-balance-query.md](./channel-balance-query.md)。 +- LK888 媒体接口要求模型特定参数放在 `params` 内；不要把未知顶层字段直接透传到下游。 +- 上传类参数必须是公网 URL；平台不提供文件上传托管。 diff --git a/docs/openapi/relay.json b/docs/openapi/relay.json index 62a0b65b082d..d80d08871b11 100644 --- a/docs/openapi/relay.json +++ b/docs/openapi/relay.json @@ -555,6 +555,83 @@ "parameters": [], "requestBody": { "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "model": { + "description": "模型名称", + "example": "grok-imagine-1.0-video", + "type": "string" + }, + "prompt": { + "description": "提示词", + "example": "A doctor speaking to camera in a clean clinic", + "type": "string" + }, + "images": { + "description": "参考图片 URL 或 data:image/...;base64,... 数组。Grok/Qilin 渠道会在内部自动转换为 image_reference；新版 Grok 插件上限为 7 张。", + "type": "array", + "items": { + "type": "string" + } + }, + "aspect_ratio": { + "description": "画幅比例。Grok/Qilin 渠道支持 9:16、16:9、1:1、4:3、3:4、21:9，并会自动映射到 size。", + "example": "9:16", + "type": "string" + }, + "ratio": { + "description": "兼容麒麟插件的画幅比例字段；未传 size 时按 aspect_ratio 同样规则映射。", + "example": "9:16", + "type": "string" + }, + "size": { + "description": "视频尺寸。Grok/Qilin 推荐 720x1280、1280x720、1024x1024；新版插件还映射 1152x864、864x1152、1680x720。", + "example": "720x1280", + "type": "string" + }, + "seconds": { + "description": "生成秒数。Grok 支持 6、10、15、20、30；20/30 秒会自动转长时长传输模型。", + "example": "6", + "type": "string" + }, + "duration": { + "description": "生成秒数，Grok/Qilin 会和 seconds 互补；20/30 秒会自动转长时长传输模型。", + "example": 6, + "type": "integer" + }, + "resolution": { + "description": "分辨率档位。Grok/Qilin 未传时默认 720p。", + "example": "720p", + "type": "string" + }, + "quality": { + "description": "Grok/Qilin 原生画质字段，未传时按 resolution 自动补 high 或 standard。", + "example": "high", + "type": "string" + } + }, + "required": [ + "model", + "prompt" + ] + }, + "examples": { + "grokReference": { + "summary": "Grok 参考图视频", + "value": { + "model": "grok-imagine-1.0-video", + "prompt": "Use the provided reference image as the doctor identity. No subtitles.", + "images": [ + "https://example.com/reference.png" + ], + "aspect_ratio": "9:16", + "seconds": "6" + } + } + } + }, "multipart/form-data": { "schema": { "type": "object", diff --git a/docs/sora-video-api.md b/docs/sora-video-api.md new file mode 100644 index 000000000000..37bea309f39b --- /dev/null +++ b/docs/sora-video-api.md @@ -0,0 +1,205 @@ +# Sora 视频生成渠道调用文档 + +最后更新：2026-05-28 + +> **2026-05-28 回归验证**：`xb-sora2` 重新通过真实验证（task `task_AnRb9zA2TNPKnUl3WjK0ep2yvbBgdaoD`，约 3.5 分钟完成，产出 6.4MB MP4）。`ss-sora-2` 首次通过真实验证（task `task_s4H8Mwn0LwsUMviZTWviEH2GVBHvC7V4`，约 3 分钟完成，产出 7.8MB MP4）。`openai-sora-2` 仍因 seconds 归一化问题不可用。 + +本文档描述通过本服务调用 Sora/Hongniao 视频生成渠道。调用方只需要使用本项目统一的 OpenAI Video 兼容入口，不需要感知 Hongniao 的下游 API Key、接口路径和响应包装。 + +## 快速结论 + +生产调用时按下面规则传参： + +| 目标 | 推荐写法 | +|------|----------| +| 创建视频 | `POST /v1/videos`，JSON 请求体 | +| 查询任务 | `GET /v1/videos/{task_id}` | +| 推荐模型 | `xb-sora2` | +| 参考图 | 传 `images: ["https://..."]`，保持数组 | +| 本地参考图 | 先上传成公网 URL；小图可传 `data:image/...;base64,...` | +| 横屏 | `orientation: "landscape"` 或 `aspect_ratio: "16:9"` | +| 竖屏 | `orientation: "portrait"` 或 `aspect_ratio: "9:16"` | +| 时长 | `duration: 8` 或 `duration: 12`；不传默认按模型补齐 | + +`xb-sora2` 是当前主路径，已经通过生产网关验证文生视频和单参考图视频。AI 聚合站 / LK888 也已接入 `sora-2`，当前作为低优先级备用/验证线路，详情见 [AI 聚合站 / LK888 视频渠道接入文档](./lk888-video-api.md)。 + +## 连接信息 + +调用方连接本项目，不直接连接 Hongniao： + +| 项目 | 值 | +|------|-----| +| Base URL | `http://192.129.209.36:3001/v1` | +| 认证方式 | HTTP Header `Authorization: Bearer ` | +| 创建任务 | `POST /v1/videos` | +| 查询任务 | `GET /v1/videos/{task_id}` | +| 下载内容 | `GET /v1/videos/{task_id}/content` | + +下游 Hongniao 渠道配置： + +| 项目 | 值 | +|------|-----| +| 渠道类型 | `58` / OpenAI Video | +| 渠道名 | `xb-sora2` | +| 下游 Base URL | `https://open.hongniaoai.com/v1` | +| 下游认证 | `X-API-Key` | +| 下游创建任务 | `POST /videos/generate` | +| 下游查询任务 | `GET /videos/{task_id}` | +| 下游模型发现 | `GET /models` | + +## 模型列表 + +远端已从 Hongniao `/models` 拉取并配置以下真实模型。建议业务侧优先使用 `xb-sora2`，其他模型用于明确指定线路或做排障对比。 + +| 模型名 | 说明 | 推荐时长 | +|--------|------|----------| +| `xb-sora2` | Sora-2 线路 XB，当前稳定主路径 | 8 / 12 | +| `ss-sora-2` | Sora-2 线路 S | 4 / 8 / 12 | +| `sora-2(线路BF)` | Sora-2 线路 BF | 4 / 8 / 12 | +| `sora-2-pro(线路BF)` | Sora-2 Pro 线路 BF | 4 / 8 / 12 | +| `je-grok` | Grok 视频线路 JE | 6 / 10 | +| `grok-video-3(线路W)` | Grok 视频线路 W | 6 / 10 | +| `全能视频2.0` | Hongniao 全能视频 | 4 / 5 / 8 / 10 / 15 | +| `香蕉2(线路V)` | Hongniao 返回的香蕉视频模型 | 按下游模型能力 | +| `香蕉pro(线路G)` | Hongniao 返回的香蕉 Pro 模型 | 按下游模型能力 | +| `gr-image-2` | Hongniao 返回的 gpt-image-2 相关模型 | 按下游模型能力 | +| `gpt-image-2(线路XF)` | Hongniao 返回的 gpt-image-2 线路 XF | 按下游模型能力 | + +本项目还保留了文档兼容别名： + +| 调用方模型 | 实际下游模型 | +|------------|--------------| +| `openai-sora-2` | `xb-sora2` | +| `sora-2-image-to-video` | `xb-sora2` | +| `sora-2-pro-text-to-video` | `sora-2-pro(线路BF)` | +| `sora-2` | `xb-sora2` | +| `sora-2-pro` | `sora-2-pro(线路BF)` | + +## 参数说明 + +| 参数 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `model` | string | 是 | 推荐 `xb-sora2` | +| `prompt` | string | 是 | 视频内容描述；建议写清楚主体、动作、镜头、风格 | +| `duration` | number | 否 | 视频时长。`xb-sora2` 建议 `8` 或 `12` | +| `seconds` | string/number | 否 | 兼容字段，会转换为 `duration` | +| `orientation` | string | 否 | `landscape` 或 `portrait` | +| `aspect_ratio` | string | 否 | `16:9` 会转 `landscape`，`9:16` 会转 `portrait` | +| `ratio` | string | 否 | 兼容比例字段，按 `aspect_ratio` 同规则处理 | +| `size` | string | 否 | 兼容字段，如 `1280x720` / `720x1280`，会转换为方向 | +| `images` | array[string] | 否 | 参考图片 URL 或 `data:image/...;base64,...`，Hongniao 文档说明最多 5 张 | +| `image` | string/array | 否 | 兼容字段，会收敛到 `images` | +| `input_reference` | string/array | 否 | 兼容字段，会收敛到 `images` | +| `image_url` | string/array | 否 | 兼容字段，会收敛到 `images` | + +Provider 会删除 Hongniao 不需要的 OpenAI 兼容字段，例如 `n`、`seed`、`response_format`、`width`、`height`、`fps`、`user`。 + +## 文生视频 + +```bash +curl -s "http://192.129.209.36:3001/v1/videos" \ + -H "Authorization: Bearer $API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "xb-sora2", + "prompt": "A calm sunrise over a mountain lake, cinematic, slow camera movement", + "orientation": "landscape", + "duration": 8 + }' +``` + +成功响应： + +```json +{ + "id": "task_woE206uzgDCVrYTOkPhyyTtVP14GldbP", + "task_id": "task_woE206uzgDCVrYTOkPhyyTtVP14GldbP", + "object": "video", + "model": "xb-sora2", + "status": "queued", + "progress": 0 +} +``` + +## 参考图视频 + +```bash +curl -s "http://192.129.209.36:3001/v1/videos" \ + -H "Authorization: Bearer $API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "xb-sora2", + "prompt": "Use the reference image as visual inspiration. A gentle cinematic camera move, warm daylight, high quality.", + "orientation": "landscape", + "duration": 8, + "images": [ + "https://example.com/reference.png" + ] + }' +``` + +推荐只用 `images` 数组作为业务主字段。兼容字段 `image`、`input_reference`、`image_url` 可以用于旧调用方迁移，但新接入不要依赖这些字段。 + +## 查询任务 + +```bash +curl -s "http://192.129.209.36:3001/v1/videos/$TASK_ID" \ + -H "Authorization: Bearer $API_KEY" +``` + +处理中响应： + +```json +{ + "id": "task_xxx", + "object": "video", + "model": "xb-sora2", + "status": "in_progress", + "progress": 30 +} +``` + +完成响应： + +```json +{ + "id": "task_xxx", + "object": "video", + "model": "xb-sora2", + "status": "completed", + "progress": 100, + "video_url": "https://..." +} +``` + +## 内部转换规则 + +本项目对外保持统一 OpenAI Video 风格，内部由 `xbSoraProvider` 适配 Hongniao 协议。 + +| 调用方输入 | 下游处理 | +|------------|----------| +| `Authorization: Bearer <用户 token>` | 使用渠道密钥设置 `X-API-Key` | +| `/v1/videos` | 转发到 `POST https://open.hongniaoai.com/v1/videos/generate` | +| `/v1/videos/{task_id}` | 转发到 `GET https://open.hongniaoai.com/v1/videos/{upstream_task_id}` | +| `openai-sora-2` / `sora-2-image-to-video` | 映射为 `xb-sora2` | +| `sora-2-pro-text-to-video` | 映射为 `sora-2-pro(线路BF)` | +| `seconds` | 转为 `duration` | +| `aspect_ratio` / `ratio` / `size` | 转为 `orientation` | +| `images` / `image` / `input_reference` / `image_url` | 收敛为下游 `images` 数组 | +| Hongniao 外层 `code:"0000"` 响应 | 解包为本项目任务状态 | + +## 已验证能力 + +| 能力 | 结果 | 任务 | +|------|------|------| +| 模型发现 | 通过 | `GET https://open.hongniaoai.com/v1/models` 返回 11 个真实模型 | +| 文生视频 | 通过 | `task_woE206uzgDCVrYTOkPhyyTtVP14GldbP`，`completed`，`progress=100`，返回视频 URL | +| 单参考图视频 | 通过 | `task_A80f7CbmU4xxDSCn7Xi6fCLGJREpPW0C`，`completed`，`progress=100`，返回视频 URL | + +## 注意事项 + +- 生产推荐模型是 `xb-sora2`，不要默认使用文档里的 `openai-sora-2`。`openai-sora-2` 只是兼容别名，会映射到 `xb-sora2`。 +- `sora-2` 还有 LK888 备用线路。由于当前生产主路径仍是 Hongniao，如需专门验证 LK888 Sora，需要临时调整渠道优先级；验证完成后保持 LK888 渠道低优先级。 +- Hongniao 文档顶部曾出现 `https://localhost:3000/v1`，实际生产地址已确认为 `https://open.hongniaoai.com/v1`。 +- `images` 已验证能被下游接受并完成任务，但人物身份一致性、首尾帧严格程度仍取决于 Hongniao 下游模型，不是网关能完全保证的能力。 +- 下游返回的视频 URL 是签名 URL，可能有过期时间。长期保存请在生成完成后尽快下载或转存。 diff --git a/dto/channel_settings.go b/dto/channel_settings.go index b6a1ab9f7138..bf7f7cfb642c 100644 --- a/dto/channel_settings.go +++ b/dto/channel_settings.go @@ -1,12 +1,32 @@ package dto type ChannelSettings struct { - ForceFormat bool `json:"force_format,omitempty"` - ThinkingToContent bool `json:"thinking_to_content,omitempty"` - Proxy string `json:"proxy"` - PassThroughBodyEnabled bool `json:"pass_through_body_enabled,omitempty"` - SystemPrompt string `json:"system_prompt,omitempty"` - SystemPromptOverride bool `json:"system_prompt_override,omitempty"` + ForceFormat bool `json:"force_format,omitempty"` + ThinkingToContent bool `json:"thinking_to_content,omitempty"` + Proxy string `json:"proxy"` + PassThroughBodyEnabled bool `json:"pass_through_body_enabled,omitempty"` + SystemPrompt string `json:"system_prompt,omitempty"` + SystemPromptOverride bool `json:"system_prompt_override,omitempty"` + BalanceQuery *BalanceQuerySetting `json:"balance_query,omitempty"` // 下游余额查询配置（详见 docs/channel-balance-query.md） +} + +// BalanceQuerySetting 下游平台余额查询配置。 +// 大量下游中转站发放的是「无限额度」API Key，仅凭 Key 只能查到累计消费、查不到钱包余额； +// 要拿真实钱包余额需要补充控制台登录凭据（账密或用户级系统访问令牌）。 +type BalanceQuerySetting struct { + // Mode 余额查询模式： + // ""/"auto" —— 按 base_url/Other 自动识别 provider（默认） + // "newapi_console" —— new-api 套壳站：用账密登录控制台拿真实钱包余额 + // "system_token" —— new-api 套壳站：用用户级系统访问令牌调 /api/user/self + // "disabled" —— 不查询该渠道余额 + Mode string `json:"mode,omitempty"` + // Username / Password 仅在 Mode=newapi_console 时使用 + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + // Token 仅在 Mode=system_token 时使用（下游站点「个人设置」生成的系统访问令牌） + Token string `json:"token,omitempty"` + // Recharged 用户手填的累计充值额（仅 spend_only 档用于估算剩余 ≈ Recharged - Used） + Recharged float64 `json:"recharged,omitempty"` } type VertexKeyType string diff --git a/dto/openai_image.go b/dto/openai_image.go index 547b0d18420f..3dc56e3b9194 100644 --- a/dto/openai_image.go +++ b/dto/openai_image.go @@ -18,6 +18,7 @@ type ImageRequest struct { Size string `json:"size,omitempty"` Quality string `json:"quality,omitempty"` ResponseFormat string `json:"response_format,omitempty"` + ExtraBody json.RawMessage `json:"extra_body,omitempty"` Style json.RawMessage `json:"style,omitempty"` User json.RawMessage `json:"user,omitempty"` ExtraFields json.RawMessage `json:"extra_fields,omitempty"` diff --git a/dto/openai_video.go b/dto/openai_video.go index e5cdbb0dd3a2..8439d285904c 100644 --- a/dto/openai_video.go +++ b/dto/openai_video.go @@ -20,6 +20,7 @@ type OpenAIVideo struct { Model string `json:"model"` Status string `json:"status"` // Should use VideoStatus constants: VideoStatusQueued, VideoStatusInProgress, VideoStatusCompleted, VideoStatusFailed Progress int `json:"progress"` + VideoURL string `json:"video_url,omitempty"` CreatedAt int64 `json:"created_at"` CompletedAt int64 `json:"completed_at,omitempty"` ExpiresAt int64 `json:"expires_at,omitempty"` diff --git a/middleware/distributor.go b/middleware/distributor.go index 258aebb57037..5f5b27e9e1db 100644 --- a/middleware/distributor.go +++ b/middleware/distributor.go @@ -38,6 +38,7 @@ func Distribute() func(c *gin.Context) { abortWithOpenAiMessage(c, http.StatusBadRequest, i18n.T(c, i18n.MsgDistributorInvalidRequest, map[string]any{"Error": err.Error()})) return } + requiredEndpointType := getRequiredEndpointType(c) if ok { id, err := strconv.Atoi(channelId.(string)) if err != nil { @@ -53,6 +54,10 @@ func Distribute() func(c *gin.Context) { abortWithOpenAiMessage(c, http.StatusForbidden, i18n.T(c, i18n.MsgDistributorChannelDisabled)) return } + if requiredEndpointType != "" && !channelSupportsRequest(c, channel, modelRequest.Model, requiredEndpointType) { + abortWithOpenAiMessage(c, http.StatusForbidden, i18n.T(c, i18n.MsgDistributorInvalidChannelId)) + return + } } else { // Select a channel for the user // check token model mapping @@ -110,15 +115,18 @@ func Distribute() func(c *gin.Context) { autoGroups := service.GetUserAutoGroup(userGroup) for _, g := range autoGroups { if model.IsChannelEnabledForGroupModel(g, modelRequest.Model, preferred.Id) { - selectGroup = g - common.SetContextKey(c, constant.ContextKeyAutoGroup, g) - channel = preferred - affinityUsable = true - service.MarkChannelAffinityUsed(c, g, preferred.Id) - break + if requiredEndpointType == "" || channelSupportsRequest(c, preferred, modelRequest.Model, requiredEndpointType) { + selectGroup = g + common.SetContextKey(c, constant.ContextKeyAutoGroup, g) + channel = preferred + affinityUsable = true + service.MarkChannelAffinityUsed(c, g, preferred.Id) + break + } } } - } else if model.IsChannelEnabledForGroupModel(usingGroup, modelRequest.Model, preferred.Id) { + } else if model.IsChannelEnabledForGroupModel(usingGroup, modelRequest.Model, preferred.Id) && + (requiredEndpointType == "" || channelSupportsRequest(c, preferred, modelRequest.Model, requiredEndpointType)) { channel = preferred selectGroup = usingGroup affinityUsable = true @@ -131,12 +139,7 @@ func Distribute() func(c *gin.Context) { } if channel == nil { - channel, selectGroup, err = service.CacheGetRandomSatisfiedChannel(&service.RetryParam{ - Ctx: c, - ModelName: modelRequest.Model, - TokenGroup: usingGroup, - Retry: common.GetPointer(0), - }) + channel, selectGroup, err = getRandomSatisfiedEndpointChannel(c, usingGroup, modelRequest.Model, requiredEndpointType) if err != nil { showGroup := usingGroup if usingGroup == "auto" { @@ -350,8 +353,10 @@ func getModelRequest(c *gin.Context) (*ModelRequest, bool, error) { } if strings.HasPrefix(c.Request.URL.Path, "/v1/images/generations") { modelRequest.Model = common.GetStringIfEmpty(modelRequest.Model, "dall-e") + c.Set("relay_mode", relayconstant.RelayModeImagesGenerations) } else if strings.HasPrefix(c.Request.URL.Path, "/v1/images/edits") { //modelRequest.Model = common.GetStringIfEmpty(c.PostForm("model"), "gpt-image-1") + c.Set("relay_mode", relayconstant.RelayModeImagesEdits) contentType := c.ContentType() if slices.Contains([]string{gin.MIMEPOSTForm, gin.MIMEMultipartPOSTForm}, contentType) { req, err := getModelFromRequest(c) @@ -399,6 +404,227 @@ func getModelRequest(c *gin.Context) (*ModelRequest, bool, error) { return &modelRequest, shouldSelectChannel, nil } +func getRequiredEndpointType(c *gin.Context) constant.EndpointType { + relayMode, ok := c.Get("relay_mode") + if !ok { + return "" + } + mode, ok := relayMode.(int) + if !ok { + return "" + } + switch mode { + case relayconstant.RelayModeGemini: + return constant.EndpointTypeGemini + case relayconstant.RelayModeImagesGenerations, relayconstant.RelayModeImagesEdits: + return constant.EndpointTypeImageGeneration + default: + return "" + } +} + +func channelSupportsEndpointType(channel *model.Channel, modelName string, endpointType constant.EndpointType) bool { + if endpointType == "" || channel == nil { + return true + } + endpointTypes := common.GetEndpointTypesByChannelType(channel.Type, modelName) + for _, candidate := range endpointTypes { + if candidate == endpointType { + return true + } + } + return false +} + +func channelSupportsRequest(c *gin.Context, channel *model.Channel, modelName string, endpointType constant.EndpointType) bool { + if !channelSupportsEndpointType(channel, modelName, endpointType) { + return false + } + if endpointType == constant.EndpointTypeImageGeneration && + isXGAPIChannel(channel) && + requestHasImageReference(c) { + return false + } + return true +} + +func isXGAPIChannel(channel *model.Channel) bool { + if channel == nil { + return false + } + return containsXGAPIIdentifier(channel.GetBaseURL()) || + containsXGAPIIdentifier(channel.Other) || + containsXGAPIIdentifier(channel.Name) +} + +func containsXGAPIIdentifier(value string) bool { + value = strings.ToLower(strings.TrimSpace(value)) + return strings.Contains(value, "xgapi") || + strings.Contains(value, "xingguang") || + strings.Contains(value, "星光") +} + +const contextKeyImageReferenceRequest = "_new_api_image_reference_request" + +func requestHasImageReference(c *gin.Context) bool { + if c == nil || c.Request == nil { + return false + } + if cached, ok := c.Get(contextKeyImageReferenceRequest); ok { + hasReference, _ := cached.(bool) + return hasReference + } + hasReference := parseRequestHasImageReference(c) + c.Set(contextKeyImageReferenceRequest, hasReference) + return hasReference +} + +func parseRequestHasImageReference(c *gin.Context) bool { + if strings.Contains(c.Request.Header.Get("Content-Type"), gin.MIMEMultipartPOSTForm) { + return multipartRequestHasImageReference(c) + } + var imageRequest dto.ImageRequest + if err := common.UnmarshalBodyReusable(c, &imageRequest); err != nil { + return false + } + return imageRequestHasReference(imageRequest) +} + +func multipartRequestHasImageReference(c *gin.Context) bool { + form, err := common.ParseMultipartFormReusable(c) + if err != nil || form == nil { + return false + } + for _, key := range imageReferenceFieldNames() { + if len(form.Value[key]) > 0 || len(form.File[key]) > 0 { + return true + } + } + for key, values := range form.Value { + if isImageReferenceFieldName(key) && len(values) > 0 { + return true + } + } + for key, files := range form.File { + if isImageReferenceFieldName(key) && len(files) > 0 { + return true + } + } + return false +} + +func imageRequestHasReference(request dto.ImageRequest) bool { + if rawJSONHasValue(request.Image) || rawJSONHasValue(request.Images) { + return true + } + for _, key := range imageReferenceFieldNames() { + if rawJSONHasValue(request.Extra[key]) { + return true + } + } + return rawObjectHasImageReference(request.ExtraBody) +} + +func rawObjectHasImageReference(raw []byte) bool { + if !rawJSONHasValue(raw) { + return false + } + var fields map[string]any + if err := common.Unmarshal(raw, &fields); err != nil { + return false + } + for _, key := range imageReferenceFieldNames() { + if jsonValueHasMeaning(fields[key]) { + return true + } + } + for _, key := range []string{"listenhub", "xgapi", "imageConfig", "image_config"} { + if !jsonValueHasMeaning(fields[key]) { + continue + } + nested, err := common.Marshal(fields[key]) + if err == nil && rawObjectHasImageReference(nested) { + return true + } + } + return false +} + +func rawJSONHasValue(raw []byte) bool { + value := strings.TrimSpace(string(raw)) + return value != "" && + value != "null" && + value != `""` && + value != "[]" && + value != "{}" +} + +func jsonValueHasMeaning(value any) bool { + switch typed := value.(type) { + case nil: + return false + case string: + return strings.TrimSpace(typed) != "" + case []any: + return len(typed) > 0 + case map[string]any: + return len(typed) > 0 + default: + return true + } +} + +func imageReferenceFieldNames() []string { + return []string{ + "image", + "image[]", + "images", + "referenceImages", + "reference_images", + "input_image", + "input_images", + "inputReference", + "input_reference", + } +} + +func isImageReferenceFieldName(name string) bool { + if slices.Contains(imageReferenceFieldNames(), name) { + return true + } + return strings.HasPrefix(name, "image[") || + strings.HasPrefix(name, "images[") || + strings.HasPrefix(name, "referenceImages[") || + strings.HasPrefix(name, "reference_images[") +} + +func getRandomSatisfiedEndpointChannel(c *gin.Context, usingGroup string, modelName string, endpointType constant.EndpointType) (*model.Channel, string, error) { + maxRetry := common.RetryTimes + if endpointType != "" && maxRetry < 5 { + maxRetry = 5 + } + var lastSelectGroup string + for retry := 0; retry <= maxRetry; retry++ { + channel, selectGroup, err := service.CacheGetRandomSatisfiedChannel(&service.RetryParam{ + Ctx: c, + ModelName: modelName, + TokenGroup: usingGroup, + Retry: common.GetPointer(retry), + }) + lastSelectGroup = selectGroup + if err != nil { + return nil, selectGroup, err + } + if channel == nil { + return nil, selectGroup, nil + } + if channelSupportsRequest(c, channel, modelName, endpointType) { + return channel, selectGroup, nil + } + } + return nil, lastSelectGroup, nil +} + // 修复 #4834: GET /v1/video/generations/:task_id && /v1/video/:task_id 此前不解析 model， // 当 token 启用「可用模型限制」时，下游 modelLimitEnable 校验会因 // modelRequest.Model 为空而误报 "This token has no access to model"。 @@ -432,6 +658,7 @@ func SetupContextForSelectedChannel(c *gin.Context, channel *model.Channel, mode common.SetContextKey(c, constant.ContextKeyChannelId, channel.Id) common.SetContextKey(c, constant.ContextKeyChannelName, channel.Name) common.SetContextKey(c, constant.ContextKeyChannelType, channel.Type) + common.SetContextKey(c, constant.ContextKeyChannelOther, channel.Other) common.SetContextKey(c, constant.ContextKeyChannelCreateTime, channel.CreatedTime) common.SetContextKey(c, constant.ContextKeyChannelSetting, channel.GetSetting()) common.SetContextKey(c, constant.ContextKeyChannelOtherSetting, channel.GetOtherSettings()) diff --git a/middleware/distributor_test.go b/middleware/distributor_test.go new file mode 100644 index 000000000000..d90e9e85dea6 --- /dev/null +++ b/middleware/distributor_test.go @@ -0,0 +1,71 @@ +package middleware + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/QuantumNous/new-api/constant" + "github.com/QuantumNous/new-api/model" + "github.com/gin-gonic/gin" +) + +func newImageGenerationContext(body string) *gin.Context { + gin.SetMode(gin.TestMode) + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/images/generations", strings.NewReader(body)) + c.Request.Header.Set("Content-Type", "application/json") + return c +} + +func testChannel(name string, baseURL string) *model.Channel { + return &model.Channel{ + Type: constant.ChannelTypeOpenAI, + Name: name, + BaseURL: &baseURL, + } +} + +func TestRequestHasImageReferenceFromImageField(t *testing.T) { + c := newImageGenerationContext(`{"model":"gpt-image-2","prompt":"edit this","image":"https://example.com/a.png"}`) + + if !requestHasImageReference(c) { + t.Fatal("requestHasImageReference returned false for image field") + } +} + +func TestRequestHasImageReferenceFromReferenceImagesExtraField(t *testing.T) { + c := newImageGenerationContext(`{"model":"gpt-image-2","prompt":"edit this","referenceImages":["https://example.com/a.png"]}`) + + if !requestHasImageReference(c) { + t.Fatal("requestHasImageReference returned false for referenceImages field") + } +} + +func TestChannelSupportsRequestSkipsXGAPIForReferenceImages(t *testing.T) { + c := newImageGenerationContext(`{"model":"gpt-image-2","prompt":"edit this","image":"https://example.com/a.png"}`) + xgapiChannel := testChannel("xgapi-images", "https://xgapi.top") + + if channelSupportsRequest(c, xgapiChannel, "gpt-image-2", constant.EndpointTypeImageGeneration) { + t.Fatal("xgapi image generation channel should not handle reference-image requests") + } +} + +func TestChannelSupportsRequestAllowsXGAPIWithoutReferenceImages(t *testing.T) { + c := newImageGenerationContext(`{"model":"gpt-image-2","prompt":"generate this","size":"1024x1024"}`) + xgapiChannel := testChannel("xgapi-images", "https://xgapi.top") + + if !channelSupportsRequest(c, xgapiChannel, "gpt-image-2", constant.EndpointTypeImageGeneration) { + t.Fatal("xgapi image generation channel should handle direct generation requests") + } +} + +func TestChannelSupportsRequestAllowsNonXGAPIReferenceImages(t *testing.T) { + c := newImageGenerationContext(`{"model":"gpt-image-2","prompt":"edit this","image":"https://example.com/a.png"}`) + listenHubChannel := testChannel("listenhub-images", "https://api.marswave.ai/openapi") + + if !channelSupportsRequest(c, listenHubChannel, "gpt-image-2", constant.EndpointTypeImageGeneration) { + t.Fatal("non-xgapi image channel should be eligible for reference-image requests") + } +} diff --git a/model/channel.go b/model/channel.go index 78a1477c327e..703675812eda 100644 --- a/model/channel.go +++ b/model/channel.go @@ -168,7 +168,17 @@ func (c ChannelInfo) Value() (driver.Value, error) { // Scan implements sql.Scanner interface func (c *ChannelInfo) Scan(value interface{}) error { - bytesValue, _ := value.([]byte) + var bytesValue []byte + switch v := value.(type) { + case []byte: + bytesValue = v + case string: + bytesValue = []byte(v) + } + if len(bytesValue) == 0 { + *c = ChannelInfo{} + return nil + } return common.Unmarshal(bytesValue, c) } @@ -704,6 +714,10 @@ func hasEnabledMultiKey(keys []string, statusList map[int]int) bool { } func UpdateChannelStatus(channelId int, usingKey string, status int, reason string) bool { + if status == common.ChannelStatusEnabled { + // 渠道被启用（人工或自动恢复）时同步解除 quota 冷却 + ClearChannelQuotaCooldown(channelId) + } if common.MemoryCacheEnabled { channelStatusLock.Lock() defer channelStatusLock.Unlock() diff --git a/model/channel_cache.go b/model/channel_cache.go index 03740d2cd3ab..c16749fd9a77 100644 --- a/model/channel_cache.go +++ b/model/channel_cache.go @@ -95,7 +95,6 @@ func SyncChannelCache(frequency int) { } func GetRandomSatisfiedChannel(group string, model string, retry int) (*Channel, error) { - // if memory cache is disabled, get channel directly from database if !common.MemoryCacheEnabled { return GetChannel(group, model, retry) } @@ -103,10 +102,8 @@ func GetRandomSatisfiedChannel(group string, model string, retry int) (*Channel, channelSyncLock.RLock() defer channelSyncLock.RUnlock() - // First, try to find channels with the exact model name. channels := group2model2channels[group][model] - // If no channels found, try to find channels with the normalized model name. if len(channels) == 0 { normalizedModel := ratio_setting.FormatMatchingModelName(model) channels = group2model2channels[group][normalizedModel] @@ -140,24 +137,41 @@ func GetRandomSatisfiedChannel(group string, model string, retry int) (*Channel, if retry >= len(uniquePriorities) { retry = len(uniquePriorities) - 1 } - targetPriority := int64(sortedUniquePriorities[retry]) - // get the priority for the given retry number - var sumWeight = 0 - var targetChannels []*Channel - for _, channelId := range channels { - if channel, ok := channelsIDM[channelId]; ok { - if channel.GetPriority() == targetPriority { - sumWeight += channel.GetWeight() - targetChannels = append(targetChannels, channel) + // 优先级桶序基于全量渠道保持稳定（避免冷却渠道消失导致 retry 索引漂移跳桶）。 + // quota 冷却只在桶内过滤：目标桶被滤空时顺延到更低优先级桶； + // 所有剩余桶都被滤空时，放行目标桶的冷却渠道兜底（被动探活）。 + collectBucket := func(targetPriority int64, skipCooldown bool) (int, []*Channel) { + sumWeight := 0 + var bucket []*Channel + for _, channelId := range channels { + channel, ok := channelsIDM[channelId] + if !ok || channel.GetPriority() != targetPriority { + continue } - } else { - return nil, fmt.Errorf("数据库一致性错误，渠道# %d 不存在，请联系管理员修复", channelId) + if skipCooldown && IsChannelInQuotaCooldown(channel.Id) { + continue + } + sumWeight += channel.GetWeight() + bucket = append(bucket, channel) } + return sumWeight, bucket + } + + var sumWeight int + var targetChannels []*Channel + for prioIdx := retry; prioIdx < len(sortedUniquePriorities); prioIdx++ { + sumWeight, targetChannels = collectBucket(int64(sortedUniquePriorities[prioIdx]), true) + if len(targetChannels) > 0 { + break + } + } + if len(targetChannels) == 0 { + sumWeight, targetChannels = collectBucket(int64(sortedUniquePriorities[retry]), false) } if len(targetChannels) == 0 { - return nil, errors.New(fmt.Sprintf("no channel found, group: %s, model: %s, priority: %d", group, model, targetPriority)) + return nil, errors.New(fmt.Sprintf("no channel found, group: %s, model: %s, priority: %d", group, model, sortedUniquePriorities[retry])) } // smoothing factor and adjustment diff --git a/model/channel_cooldown.go b/model/channel_cooldown.go new file mode 100644 index 000000000000..12aee9bc06a8 --- /dev/null +++ b/model/channel_cooldown.go @@ -0,0 +1,45 @@ +package model + +import ( + "fmt" + "sync" + "time" + + "github.com/QuantumNous/new-api/common" +) + +// 渠道 quota 冷却：上游返回余额/额度不足时，把渠道短暂移出选路候选， +// 避免坏渠道在恢复前持续吃首跳；冷却到期自动恢复，无需探活。 +// 冷却期间每个冷却周期最多放行一次请求（兜底路径），天然起到被动探活作用。 + +var channelQuotaCooldowns sync.Map // channelId int -> time.Time（冷却截止时间） + +var quotaCooldownDuration = time.Duration(common.GetEnvOrDefault("QUOTA_ERROR_COOLDOWN_SECONDS", 600)) * time.Second + +// SetChannelQuotaCooldown 将渠道置入 quota 冷却。 +func SetChannelQuotaCooldown(channelId int) { + if channelId <= 0 || quotaCooldownDuration <= 0 { + return + } + channelQuotaCooldowns.Store(channelId, time.Now().Add(quotaCooldownDuration)) + common.SysLog(fmt.Sprintf("channel #%d entered quota cooldown for %s", channelId, quotaCooldownDuration)) +} + +// IsChannelInQuotaCooldown 判断渠道是否处于冷却期，过期条目惰性清除。 +func IsChannelInQuotaCooldown(channelId int) bool { + value, ok := channelQuotaCooldowns.Load(channelId) + if !ok { + return false + } + until, ok := value.(time.Time) + if !ok || time.Now().After(until) { + channelQuotaCooldowns.Delete(channelId) + return false + } + return true +} + +// ClearChannelQuotaCooldown 主动解除冷却（渠道被人工启用/测试通过时调用）。 +func ClearChannelQuotaCooldown(channelId int) { + channelQuotaCooldowns.Delete(channelId) +} diff --git a/model/channel_cooldown_test.go b/model/channel_cooldown_test.go new file mode 100644 index 000000000000..6f8606b6a314 --- /dev/null +++ b/model/channel_cooldown_test.go @@ -0,0 +1,36 @@ +package model + +import ( + "testing" + "time" +) + +func TestChannelQuotaCooldownLifecycle(t *testing.T) { + const channelId = 999001 + + if IsChannelInQuotaCooldown(channelId) { + t.Fatal("channel should not be in cooldown initially") + } + + SetChannelQuotaCooldown(channelId) + if !IsChannelInQuotaCooldown(channelId) { + t.Fatal("channel should be in cooldown after SetChannelQuotaCooldown") + } + + ClearChannelQuotaCooldown(channelId) + if IsChannelInQuotaCooldown(channelId) { + t.Fatal("channel should not be in cooldown after ClearChannelQuotaCooldown") + } +} + +func TestChannelQuotaCooldownExpiry(t *testing.T) { + const channelId = 999002 + + channelQuotaCooldowns.Store(channelId, time.Now().Add(-time.Second)) + if IsChannelInQuotaCooldown(channelId) { + t.Fatal("expired cooldown should be treated as not in cooldown") + } + if _, ok := channelQuotaCooldowns.Load(channelId); ok { + t.Fatal("expired cooldown entry should be lazily deleted") + } +} diff --git a/model/option.go b/model/option.go index ed1af72ebb12..31db18d3539c 100644 --- a/model/option.go +++ b/model/option.go @@ -170,6 +170,7 @@ func InitOptionMap() { common.OptionMap["SensitiveWords"] = setting.SensitiveWordsToString() common.OptionMap["StreamCacheQueueLength"] = strconv.Itoa(setting.StreamCacheQueueLength) common.OptionMap["AutomaticDisableKeywords"] = operation_setting.AutomaticDisableKeywordsToString() + common.OptionMap["UpstreamQuotaErrorKeywords"] = operation_setting.UpstreamQuotaErrorKeywordsToString() common.OptionMap["AutomaticDisableStatusCodes"] = operation_setting.AutomaticDisableStatusCodesToString() common.OptionMap["AutomaticRetryStatusCodes"] = operation_setting.AutomaticRetryStatusCodesToString() common.OptionMap["ExposeRatioEnabled"] = strconv.FormatBool(ratio_setting.IsExposeRatioEnabled()) @@ -554,6 +555,8 @@ func updateOptionMap(key string, value string) (err error) { setting.SensitiveWordsFromString(value) case "AutomaticDisableKeywords": operation_setting.AutomaticDisableKeywordsFromString(value) + case "UpstreamQuotaErrorKeywords": + operation_setting.UpstreamQuotaErrorKeywordsFromString(value) case "AutomaticDisableStatusCodes": err = operation_setting.AutomaticDisableStatusCodesFromString(value) case "AutomaticRetryStatusCodes": diff --git a/relay/channel/claude/relay-claude.go b/relay/channel/claude/relay-claude.go index 18d7455e9f22..e065d63d296e 100644 --- a/relay/channel/claude/relay-claude.go +++ b/relay/channel/claude/relay-claude.go @@ -1,10 +1,13 @@ package claude import ( + "encoding/base64" "encoding/json" "fmt" "io" + "mime" "net/http" + "path/filepath" "strings" "github.com/QuantumNous/new-api/common" @@ -380,6 +383,45 @@ func RequestOpenAI2ClaudeMessage(c *gin.Context, textRequest dto.GeneralOpenAIRe Text: common.GetPointer[string](mediaMessage.Text), }) } + case dto.ContentTypeFile: + file := mediaMessage.GetFile() + if file == nil || file.FileData == "" { + continue + } + source := types.NewFileSourceFromData(file.FileData, mime.TypeByExtension(strings.ToLower(filepath.Ext(file.FileName)))) + base64Data, mimeType, err := service.GetBase64Data(c, source, "formatting file for Claude") + if err != nil { + return nil, fmt.Errorf("get file data failed: %s", err.Error()) + } + switch { + case strings.HasPrefix(mimeType, "application/pdf"): + claudeMediaMessages = append(claudeMediaMessages, dto.ClaudeMediaMessage{ + Type: "document", + Source: &dto.ClaudeMessageSource{ + Type: "base64", + MediaType: mimeType, + Data: base64Data, + }, + }) + case strings.HasPrefix(mimeType, "text/"): + textData, err := base64.StdEncoding.DecodeString(base64Data) + if err != nil { + return nil, fmt.Errorf("decode text file failed: %s", err.Error()) + } + claudeMediaMessages = append(claudeMediaMessages, dto.ClaudeMediaMessage{ + Type: "text", + Text: common.GetPointer[string](string(textData)), + }) + case strings.HasPrefix(mimeType, "image/"): + claudeMediaMessages = append(claudeMediaMessages, dto.ClaudeMediaMessage{ + Type: "image", + Source: &dto.ClaudeMessageSource{ + Type: "base64", + MediaType: mimeType, + Data: base64Data, + }, + }) + } default: source := mediaMessage.ToFileSource() if source == nil { diff --git a/relay/channel/gemini/constant.go b/relay/channel/gemini/constant.go index 1a2c57056795..5cbf28c4cd70 100644 --- a/relay/channel/gemini/constant.go +++ b/relay/channel/gemini/constant.go @@ -15,6 +15,9 @@ var ModelList = []string{ "gemini-3.1-pro-preview-customtools", "gemini-3.1-flash-lite-preview", "gemini-3-pro-image-preview", "nano-banana-pro-preview", "gemini-3.1-flash-image-preview", "gemini-robotics-er-1.5-preview", + "gemini_3.0_pro_image_preview", "gemini_3.0_pro_image_preview_4K", + "gemini_3.1_flash_image_preview", "gemini_3.1_flash_image_preview_4K", + "gpt-image-2", "gemini-2.5-computer-use-preview-10-2025", "deep-research-pro-preview-12-2025", "gemini-2.5-flash-native-audio-preview-09-2025", "gemini-2.5-flash-native-audio-preview-12-2025", // gemma models diff --git a/relay/channel/listenhub/adaptor.go b/relay/channel/listenhub/adaptor.go new file mode 100644 index 000000000000..5ab6148aac1a --- /dev/null +++ b/relay/channel/listenhub/adaptor.go @@ -0,0 +1,90 @@ +package listenhub + +import ( + "errors" + "fmt" + "io" + "net/http" + "strings" + + "github.com/QuantumNous/new-api/dto" + "github.com/QuantumNous/new-api/relay/channel" + relaycommon "github.com/QuantumNous/new-api/relay/common" + relayconstant "github.com/QuantumNous/new-api/relay/constant" + "github.com/QuantumNous/new-api/types" + + "github.com/gin-gonic/gin" +) + +type Adaptor struct{} + +func (a *Adaptor) Init(info *relaycommon.RelayInfo) {} + +func (a *Adaptor) ConvertGeminiRequest(*gin.Context, *relaycommon.RelayInfo, *dto.GeminiChatRequest) (any, error) { + return nil, errors.New("not implemented") +} + +func (a *Adaptor) ConvertClaudeRequest(*gin.Context, *relaycommon.RelayInfo, *dto.ClaudeRequest) (any, error) { + return nil, errors.New("not implemented") +} + +func (a *Adaptor) ConvertAudioRequest(*gin.Context, *relaycommon.RelayInfo, dto.AudioRequest) (io.Reader, error) { + return nil, errors.New("not implemented") +} + +func (a *Adaptor) ConvertEmbeddingRequest(*gin.Context, *relaycommon.RelayInfo, dto.EmbeddingRequest) (any, error) { + return nil, errors.New("not implemented") +} + +func (a *Adaptor) ConvertOpenAIRequest(*gin.Context, *relaycommon.RelayInfo, *dto.GeneralOpenAIRequest) (any, error) { + return nil, errors.New("not implemented") +} + +func (a *Adaptor) ConvertOpenAIResponsesRequest(*gin.Context, *relaycommon.RelayInfo, dto.OpenAIResponsesRequest) (any, error) { + return nil, errors.New("not implemented") +} + +func (a *Adaptor) ConvertRerankRequest(*gin.Context, int, dto.RerankRequest) (any, error) { + return nil, errors.New("not implemented") +} + +func (a *Adaptor) ConvertImageRequest(c *gin.Context, info *relaycommon.RelayInfo, request dto.ImageRequest) (any, error) { + if info.RelayMode != relayconstant.RelayModeImagesGenerations { + return nil, fmt.Errorf("unsupported image relay mode: %d", info.RelayMode) + } + return convertImageRequest(request) +} + +func (a *Adaptor) GetRequestURL(info *relaycommon.RelayInfo) (string, error) { + baseURL := strings.TrimRight(info.ChannelBaseUrl, "/") + if strings.HasSuffix(baseURL, "/v1") { + return baseURL + "/images/generation", nil + } + return baseURL + "/v1/images/generation", nil +} + +func (a *Adaptor) SetupRequestHeader(c *gin.Context, req *http.Header, info *relaycommon.RelayInfo) error { + channel.SetupApiRequestHeader(info, c, req) + req.Set("Authorization", "Bearer "+info.ApiKey) + req.Set("Content-Type", "application/json") + if req.Get("Accept") == "" { + req.Set("Accept", "application/json") + } + return nil +} + +func (a *Adaptor) DoRequest(c *gin.Context, info *relaycommon.RelayInfo, requestBody io.Reader) (any, error) { + return channel.DoApiRequest(a, c, info, requestBody) +} + +func (a *Adaptor) DoResponse(c *gin.Context, resp *http.Response, info *relaycommon.RelayInfo) (usage any, err *types.NewAPIError) { + return imageHandler(c, resp, info) +} + +func (a *Adaptor) GetModelList() []string { + return ModelList +} + +func (a *Adaptor) GetChannelName() string { + return ChannelName +} diff --git a/relay/channel/listenhub/constants.go b/relay/channel/listenhub/constants.go new file mode 100644 index 000000000000..cd9f6aa2505b --- /dev/null +++ b/relay/channel/listenhub/constants.go @@ -0,0 +1,9 @@ +package listenhub + +var ModelList = []string{ + "gemini-3-pro-image-preview", + "gemini-3.1-flash-image-preview", + "gpt-image-2", +} + +var ChannelName = "listenhub" diff --git a/relay/channel/listenhub/dto.go b/relay/channel/listenhub/dto.go new file mode 100644 index 000000000000..6b6f088e0935 --- /dev/null +++ b/relay/channel/listenhub/dto.go @@ -0,0 +1,58 @@ +package listenhub + +type ImageConfig struct { + ImageSize string `json:"imageSize,omitempty"` + AspectRatio string `json:"aspectRatio,omitempty"` +} + +type FileData struct { + FileURI string `json:"fileUri"` + MimeType string `json:"mimeType"` +} + +type InlineData struct { + Data string `json:"data"` + MimeType string `json:"mimeType"` +} + +type ReferenceImage struct { + FileData *FileData `json:"fileData,omitempty"` + InlineData *InlineData `json:"inlineData,omitempty"` +} + +type ImageRequest struct { + Provider string `json:"provider"` + Model string `json:"model,omitempty"` + Prompt string `json:"prompt"` + ReferenceImages []ReferenceImage `json:"referenceImages,omitempty"` + ImageConfig *ImageConfig `json:"imageConfig,omitempty"` +} + +type InlineDataPart struct { + MimeType string `json:"mimeType"` + Data string `json:"data"` +} + +type ContentPart struct { + InlineData *InlineDataPart `json:"inlineData,omitempty"` + Text string `json:"text,omitempty"` +} + +type CandidateContent struct { + Parts []ContentPart `json:"parts"` +} + +type Candidate struct { + Content CandidateContent `json:"content"` +} + +type ImageResponse struct { + Candidates []Candidate `json:"candidates"` + Error *ErrorBody `json:"error,omitempty"` +} + +type ErrorBody struct { + Message string `json:"message,omitempty"` + Type string `json:"type,omitempty"` + Code any `json:"code,omitempty"` +} diff --git a/relay/channel/listenhub/image.go b/relay/channel/listenhub/image.go new file mode 100644 index 000000000000..ffc59653e0ab --- /dev/null +++ b/relay/channel/listenhub/image.go @@ -0,0 +1,473 @@ +package listenhub + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "path" + "strconv" + "strings" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/dto" + relaycommon "github.com/QuantumNous/new-api/relay/common" + "github.com/QuantumNous/new-api/service" + "github.com/QuantumNous/new-api/types" + + "github.com/gin-gonic/gin" + "github.com/samber/lo" +) + +func convertImageRequest(request dto.ImageRequest) (*ImageRequest, error) { + modelName := strings.TrimSpace(request.Model) + if modelName == "" { + modelName = "gemini-3-pro-image-preview" + } + + listenHubRequest := &ImageRequest{ + Provider: providerForModel(modelName), + Model: modelName, + Prompt: request.Prompt, + ImageConfig: imageConfigFromOpenAIRequest(request), + } + + if err := applyImageExtra(listenHubRequest, request.ExtraBody); err != nil { + return nil, err + } + if err := applyImageExtraMap(listenHubRequest, request.Extra); err != nil { + return nil, err + } + if err := appendReferenceImagesFromRaw(&listenHubRequest.ReferenceImages, request.Image); err != nil { + return nil, fmt.Errorf("invalid image field: %w", err) + } + if err := appendReferenceImagesFromRaw(&listenHubRequest.ReferenceImages, request.Images); err != nil { + return nil, fmt.Errorf("invalid images field: %w", err) + } + + if listenHubRequest.Provider == "" { + listenHubRequest.Provider = providerForModel(listenHubRequest.Model) + } + if listenHubRequest.Model == "" { + listenHubRequest.Model = modelName + } + return listenHubRequest, nil +} + +func providerForModel(modelName string) string { + if strings.EqualFold(strings.TrimSpace(modelName), "gpt-image-2") { + return "openai" + } + return "google" +} + +func imageConfigFromOpenAIRequest(request dto.ImageRequest) *ImageConfig { + config := &ImageConfig{} + + if aspectRatio := aspectRatioFromImageRequest(request); aspectRatio != "" { + config.AspectRatio = aspectRatio + } + if imageSize := imageSizeFromImageRequest(request); imageSize != "" { + config.ImageSize = imageSize + } + + if config.AspectRatio == "" && config.ImageSize == "" { + return nil + } + return config +} + +func aspectRatioFromImageRequest(request dto.ImageRequest) string { + if raw, ok := request.Extra["aspect_ratio"]; ok { + var aspectRatio string + if err := common.Unmarshal(raw, &aspectRatio); err == nil && aspectRatio != "" { + return aspectRatio + } + } + + switch strings.TrimSpace(request.Size) { + case "1024x1024", "512x512", "256x256": + return "1:1" + case "1792x1024": + return "16:9" + case "1024x1792": + return "9:16" + case "1536x1024", "1248x832": + return "3:2" + case "1024x1536", "832x1248": + return "2:3" + case "1152x864": + return "4:3" + case "864x1152": + return "3:4" + case "1344x576": + return "21:9" + } + + width, height, ok := parseImageSize(request.Size) + if !ok { + return "" + } + ratio := reduceAspectRatio(width, height) + switch ratio { + case "1:1", "2:3", "3:2", "3:4", "4:3", "9:16", "16:9", "21:9", "1:4", "4:1", "1:8", "8:1": + return ratio + default: + return "" + } +} + +func imageSizeFromImageRequest(request dto.ImageRequest) string { + if raw, ok := request.Extra["image_size"]; ok { + var imageSize string + if err := common.Unmarshal(raw, &imageSize); err == nil && imageSize != "" { + return imageSize + } + } + + switch strings.ToUpper(strings.TrimSpace(request.Quality)) { + case "1K", "2K", "4K": + return strings.ToUpper(strings.TrimSpace(request.Quality)) + case "LOW", "STANDARD": + return "1K" + case "MEDIUM", "HD", "HIGH": + return "2K" + case "ULTRA", "ULTRA_HD": + return "4K" + default: + return "" + } +} + +func applyImageExtra(target *ImageRequest, raw json.RawMessage) error { + if len(raw) == 0 { + return nil + } + + var fields map[string]json.RawMessage + if err := common.Unmarshal(raw, &fields); err != nil { + return fmt.Errorf("invalid extra_body field: %w", err) + } + if nested := fields["listenhub"]; len(nested) > 0 { + if err := applyImageExtra(target, nested); err != nil { + return err + } + } + return applyImageExtraFields(target, fields) +} + +func applyImageExtraFields(target *ImageRequest, fields map[string]json.RawMessage) error { + if err := setStringField(fields, "provider", &target.Provider); err != nil { + return err + } + if err := setStringField(fields, "model", &target.Model); err != nil { + return err + } + for _, key := range []string{"imageConfig", "image_config"} { + if raw := fields[key]; len(raw) > 0 { + config, err := parseImageConfig(raw) + if err != nil { + return fmt.Errorf("invalid %s field: %w", key, err) + } + target.ImageConfig = mergeImageConfig(target.ImageConfig, config) + } + } + for _, key := range []string{"referenceImages", "reference_images"} { + if err := appendReferenceImagesFromRaw(&target.ReferenceImages, fields[key]); err != nil { + return fmt.Errorf("invalid %s field: %w", key, err) + } + } + return nil +} + +func parseImageConfig(raw json.RawMessage) (*ImageConfig, error) { + var config struct { + ImageSize string `json:"imageSize,omitempty"` + ImageSizeSnake string `json:"image_size,omitempty"` + AspectRatio string `json:"aspectRatio,omitempty"` + AspectRatioSnake string `json:"aspect_ratio,omitempty"` + } + if err := common.Unmarshal(raw, &config); err != nil { + return nil, err + } + return &ImageConfig{ + ImageSize: common.GetStringIfEmpty(config.ImageSize, config.ImageSizeSnake), + AspectRatio: common.GetStringIfEmpty(config.AspectRatio, config.AspectRatioSnake), + }, nil +} + +func setStringField(fields map[string]json.RawMessage, key string, target *string) error { + raw := fields[key] + if len(raw) == 0 { + return nil + } + var value string + if err := common.Unmarshal(raw, &value); err != nil { + return fmt.Errorf("invalid %s field: %w", key, err) + } + if value != "" { + *target = value + } + return nil +} + +func applyImageExtraMap(target *ImageRequest, extra map[string]json.RawMessage) error { + if len(extra) == 0 { + return nil + } + raw, err := common.Marshal(extra) + if err != nil { + return err + } + if err := applyImageExtra(target, raw); err != nil { + return err + } + return nil +} + +func mergeImageConfig(base *ImageConfig, override *ImageConfig) *ImageConfig { + if override == nil { + return base + } + if base == nil { + base = &ImageConfig{} + } + if override.AspectRatio != "" { + base.AspectRatio = override.AspectRatio + } + if override.ImageSize != "" { + base.ImageSize = override.ImageSize + } + return base +} + +func appendReferenceImagesFromRaw(target *[]ReferenceImage, raw json.RawMessage) error { + if len(raw) == 0 || string(raw) == "null" { + return nil + } + + var direct []ReferenceImage + if err := common.Unmarshal(raw, &direct); err == nil && len(direct) > 0 && direct[0].hasData() { + *target = append(*target, direct...) + return nil + } + + var one ReferenceImage + if err := common.Unmarshal(raw, &one); err == nil && one.hasData() { + *target = append(*target, one) + return nil + } + + var values []json.RawMessage + if err := common.Unmarshal(raw, &values); err == nil { + for _, value := range values { + if err := appendReferenceImagesFromRaw(target, value); err != nil { + return err + } + } + return nil + } + + var value string + if err := common.Unmarshal(raw, &value); err == nil { + ref, err := referenceImageFromString(value) + if err != nil { + return err + } + *target = append(*target, ref) + return nil + } + + var obj map[string]json.RawMessage + if err := common.Unmarshal(raw, &obj); err != nil { + return err + } + for _, key := range []string{"url", "fileUri", "file_uri"} { + if rawURL, ok := obj[key]; ok { + var url string + if err := common.Unmarshal(rawURL, &url); err != nil { + return err + } + ref, err := referenceImageFromString(url) + if err != nil { + return err + } + *target = append(*target, ref) + return nil + } + } + if rawImageURL, ok := obj["image_url"]; ok { + return appendImageURLReference(target, rawImageURL) + } + return nil +} + +func appendImageURLReference(target *[]ReferenceImage, raw json.RawMessage) error { + var url string + if err := common.Unmarshal(raw, &url); err == nil { + ref, err := referenceImageFromString(url) + if err != nil { + return err + } + *target = append(*target, ref) + return nil + } + + var obj struct { + URL string `json:"url"` + } + if err := common.Unmarshal(raw, &obj); err != nil { + return err + } + if obj.URL == "" { + return nil + } + ref, err := referenceImageFromString(obj.URL) + if err != nil { + return err + } + *target = append(*target, ref) + return nil +} + +func referenceImageFromString(value string) (ReferenceImage, error) { + value = strings.TrimSpace(value) + if value == "" { + return ReferenceImage{}, errors.New("empty reference image") + } + if strings.HasPrefix(value, "data:") { + mimeType, data, ok := parseDataURI(value) + if !ok { + return ReferenceImage{}, errors.New("invalid data URI reference image") + } + return ReferenceImage{InlineData: &InlineData{Data: data, MimeType: mimeType}}, nil + } + return ReferenceImage{FileData: &FileData{FileURI: value, MimeType: inferImageMimeType(value)}}, nil +} + +func parseDataURI(value string) (string, string, bool) { + commaIdx := strings.Index(value, ",") + if commaIdx < 0 { + return "", "", false + } + meta := value[len("data:"):commaIdx] + data := value[commaIdx+1:] + parts := strings.Split(meta, ";") + if len(parts) == 0 || parts[0] == "" || data == "" { + return "", "", false + } + return parts[0], data, true +} + +func inferImageMimeType(value string) string { + ext := strings.ToLower(path.Ext(strings.Split(value, "?")[0])) + switch ext { + case ".jpg", ".jpeg": + return "image/jpeg" + case ".webp": + return "image/webp" + case ".heic": + return "image/heic" + case ".heif": + return "image/heif" + default: + return "image/png" + } +} + +func (r ReferenceImage) hasData() bool { + return r.FileData != nil || r.InlineData != nil +} + +func parseImageSize(size string) (int, int, bool) { + parts := strings.Split(size, "x") + if len(parts) != 2 { + return 0, 0, false + } + width, err := strconv.Atoi(parts[0]) + if err != nil { + return 0, 0, false + } + height, err := strconv.Atoi(parts[1]) + if err != nil { + return 0, 0, false + } + if width <= 0 || height <= 0 { + return 0, 0, false + } + return width, height, true +} + +func reduceAspectRatio(width, height int) string { + divisor := gcd(width, height) + return fmt.Sprintf("%d:%d", width/divisor, height/divisor) +} + +func gcd(a, b int) int { + for b != 0 { + a, b = b, a%b + } + if a == 0 { + return 1 + } + return a +} + +func imageHandler(c *gin.Context, resp *http.Response, info *relaycommon.RelayInfo) (*dto.Usage, *types.NewAPIError) { + responseBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, types.NewOpenAIError(err, types.ErrorCodeReadResponseBodyFailed, http.StatusInternalServerError) + } + service.CloseResponseBodyGracefully(resp) + + var listenHubResponse ImageResponse + if err := common.Unmarshal(responseBody, &listenHubResponse); err != nil { + return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError) + } + if listenHubResponse.Error != nil { + return nil, types.WithOpenAIError(types.OpenAIError{ + Message: listenHubResponse.Error.Message, + Type: lo.Ternary(listenHubResponse.Error.Type == "", "listenhub_image_error", listenHubResponse.Error.Type), + Code: listenHubResponse.Error.Code, + }, resp.StatusCode) + } + + openAIResponse := dto.ImageResponse{ + Created: info.StartTime.Unix(), + } + for _, candidate := range listenHubResponse.Candidates { + for _, part := range candidate.Content.Parts { + if part.InlineData == nil || part.InlineData.Data == "" { + continue + } + if !strings.HasPrefix(strings.ToLower(part.InlineData.MimeType), "image/") { + continue + } + openAIResponse.Data = append(openAIResponse.Data, dto.ImageData{ + B64Json: part.InlineData.Data, + }) + } + } + if len(openAIResponse.Data) == 0 { + return nil, types.NewOpenAIError(errors.New("no images generated"), types.ErrorCodeBadResponseBody, http.StatusInternalServerError) + } + + jsonResponse, err := common.Marshal(openAIResponse) + if err != nil { + return nil, types.NewError(err, types.ErrorCodeBadResponseBody) + } + + c.Writer.Header().Set("Content-Type", "application/json") + c.Writer.WriteHeader(resp.StatusCode) + if _, err := c.Writer.Write(jsonResponse); err != nil { + return nil, types.NewError(err, types.ErrorCodeBadResponseBody) + } + + imageTokens := len(openAIResponse.Data) * 258 + return &dto.Usage{ + PromptTokens: imageTokens, + CompletionTokens: 0, + TotalTokens: imageTokens, + }, nil +} diff --git a/relay/channel/openai/adaptor.go b/relay/channel/openai/adaptor.go index 2c230107de37..635b4247cdeb 100644 --- a/relay/channel/openai/adaptor.go +++ b/relay/channel/openai/adaptor.go @@ -11,6 +11,7 @@ import ( "net/textproto" "net/url" "path/filepath" + "strconv" "strings" "github.com/QuantumNous/new-api/common" @@ -427,6 +428,10 @@ func (a *Adaptor) ConvertAudioRequest(c *gin.Context, info *relaycommon.RelayInf } func (a *Adaptor) ConvertImageRequest(c *gin.Context, info *relaycommon.RelayInfo, request dto.ImageRequest) (any, error) { + if info.RelayMode == relayconstant.RelayModeImagesGenerations && isXGAPIImageChannel(info) { + request.Prompt = appendXGAPIImageAspectRatioToPrompt(request.Prompt, request) + } + switch info.RelayMode { case relayconstant.RelayModeImagesEdits: if isJSONRequest(c) { @@ -567,6 +572,170 @@ func isJSONRequest(c *gin.Context) bool { return strings.HasPrefix(c.Request.Header.Get("Content-Type"), "application/json") } +func isXGAPIImageChannel(info *relaycommon.RelayInfo) bool { + if info == nil || info.ChannelMeta == nil { + return false + } + return containsXGAPIIdentifier(info.ChannelBaseUrl) || containsXGAPIIdentifier(info.ChannelOther) +} + +func containsXGAPIIdentifier(value string) bool { + value = strings.ToLower(strings.TrimSpace(value)) + return strings.Contains(value, "xgapi") || + strings.Contains(value, "xingguang") || + strings.Contains(value, "星光") +} + +func appendXGAPIImageAspectRatioToPrompt(prompt string, request dto.ImageRequest) string { + aspectRatio := xgAPIImageAspectRatioFromRequest(request) + if aspectRatio == "" || promptAlreadyMentionsAspectRatio(prompt, aspectRatio) { + return prompt + } + + prompt = strings.TrimRight(prompt, " \t\r\n") + aspectRatioInstruction := fmt.Sprintf("Required image aspect ratio: %s.", aspectRatio) + if prompt == "" { + return aspectRatioInstruction + } + return prompt + "\n\n" + aspectRatioInstruction +} + +func xgAPIImageAspectRatioFromRequest(request dto.ImageRequest) string { + if aspectRatio := aspectRatioFromImageExtraMap(request.Extra); aspectRatio != "" { + return aspectRatio + } + if aspectRatio := aspectRatioFromImageExtraBody(request.ExtraBody); aspectRatio != "" { + return aspectRatio + } + return aspectRatioFromImageSizeForPrompt(request.Size) +} + +func aspectRatioFromImageExtraMap(extra map[string]json.RawMessage) string { + for _, key := range []string{"aspect_ratio", "aspectRatio", "ratio"} { + raw := extra[key] + if len(raw) == 0 { + continue + } + var aspectRatio string + if err := common.Unmarshal(raw, &aspectRatio); err == nil { + if aspectRatio = normalizeAspectRatio(aspectRatio); aspectRatio != "" { + return aspectRatio + } + } + } + return "" +} + +func aspectRatioFromImageExtraBody(raw json.RawMessage) string { + if len(raw) == 0 { + return "" + } + var fields map[string]json.RawMessage + if err := common.Unmarshal(raw, &fields); err != nil { + return "" + } + if aspectRatio := aspectRatioFromImageExtraMap(fields); aspectRatio != "" { + return aspectRatio + } + for _, key := range []string{"xgapi", "image_config", "imageConfig"} { + if aspectRatio := aspectRatioFromImageExtraBody(fields[key]); aspectRatio != "" { + return aspectRatio + } + } + return "" +} + +func normalizeAspectRatio(value string) string { + value = strings.TrimSpace(strings.ReplaceAll(value, "：", ":")) + if value == "" || !strings.Contains(value, ":") { + return "" + } + parts := strings.Split(value, ":") + if len(parts) != 2 { + return "" + } + width, err := strconv.Atoi(strings.TrimSpace(parts[0])) + if err != nil || width <= 0 { + return "" + } + height, err := strconv.Atoi(strings.TrimSpace(parts[1])) + if err != nil || height <= 0 { + return "" + } + divisor := greatestCommonDivisor(width, height) + return fmt.Sprintf("%d:%d", width/divisor, height/divisor) +} + +func aspectRatioFromImageSizeForPrompt(size string) string { + size = strings.TrimSpace(strings.ToLower(strings.ReplaceAll(size, "×", "x"))) + if size == "" { + return "" + } + if aspectRatio := normalizeAspectRatio(size); aspectRatio != "" { + return aspectRatio + } + + switch size { + case "256x256", "512x512", "1024x1024": + return "1:1" + case "1792x1024": + return "16:9" + case "1024x1792": + return "9:16" + case "1536x1024", "1248x832": + return "3:2" + case "1024x1536", "832x1248": + return "2:3" + case "1152x864": + return "4:3" + case "864x1152": + return "3:4" + case "1344x576": + return "21:9" + } + + parts := strings.Split(size, "x") + if len(parts) != 2 { + return "" + } + width, err := strconv.Atoi(strings.TrimSpace(parts[0])) + if err != nil || width <= 0 { + return "" + } + height, err := strconv.Atoi(strings.TrimSpace(parts[1])) + if err != nil || height <= 0 { + return "" + } + divisor := greatestCommonDivisor(width, height) + return fmt.Sprintf("%d:%d", width/divisor, height/divisor) +} + +func promptAlreadyMentionsAspectRatio(prompt string, aspectRatio string) bool { + prompt = strings.ToLower(strings.ReplaceAll(prompt, "：", ":")) + if strings.Contains(prompt, strings.ToLower(aspectRatio)) { + return true + } + for _, marker := range []string{"aspect ratio", "aspect-ratio", "宽高比", "画幅比例", "图片比例", "图像比例"} { + if strings.Contains(prompt, marker) { + return true + } + } + return false +} + +func greatestCommonDivisor(a int, b int) int { + for b != 0 { + a, b = b, a%b + } + if a < 0 { + return -a + } + if a == 0 { + return 1 + } + return a +} + // detectImageMimeType determines the MIME type based on the file extension func detectImageMimeType(filename string) string { ext := strings.ToLower(filepath.Ext(filename)) diff --git a/relay/channel/openai/adaptor_test.go b/relay/channel/openai/adaptor_test.go new file mode 100644 index 000000000000..543eb17e0fa0 --- /dev/null +++ b/relay/channel/openai/adaptor_test.go @@ -0,0 +1,90 @@ +package openai + +import ( + "strings" + "testing" + + "github.com/QuantumNous/new-api/dto" + relaycommon "github.com/QuantumNous/new-api/relay/common" + relayconstant "github.com/QuantumNous/new-api/relay/constant" +) + +func TestConvertImageRequestAppendsXGAPIAspectRatioFromSize(t *testing.T) { + adaptor := &Adaptor{} + info := &relaycommon.RelayInfo{ + RelayMode: relayconstant.RelayModeImagesGenerations, + ChannelMeta: &relaycommon.ChannelMeta{ + ChannelBaseUrl: "https://xgapi.top", + }, + } + + converted, err := adaptor.ConvertImageRequest(nil, info, dto.ImageRequest{ + Model: "gpt-image-2", + Prompt: "A clean studio product photo.", + Size: "1792x1024", + }) + if err != nil { + t.Fatalf("ConvertImageRequest returned error: %v", err) + } + + request, ok := converted.(dto.ImageRequest) + if !ok { + t.Fatalf("converted request type = %T, want dto.ImageRequest", converted) + } + if !strings.Contains(request.Prompt, "Required image aspect ratio: 16:9.") { + t.Fatalf("prompt did not contain xgapi aspect-ratio instruction: %q", request.Prompt) + } +} + +func TestConvertImageRequestSkipsAspectRatioAppendForNonXGAPI(t *testing.T) { + adaptor := &Adaptor{} + info := &relaycommon.RelayInfo{ + RelayMode: relayconstant.RelayModeImagesGenerations, + ChannelMeta: &relaycommon.ChannelMeta{ + ChannelBaseUrl: "https://api.openai.com", + }, + } + + converted, err := adaptor.ConvertImageRequest(nil, info, dto.ImageRequest{ + Model: "gpt-image-2", + Prompt: "A clean studio product photo.", + Size: "1792x1024", + }) + if err != nil { + t.Fatalf("ConvertImageRequest returned error: %v", err) + } + + request, ok := converted.(dto.ImageRequest) + if !ok { + t.Fatalf("converted request type = %T, want dto.ImageRequest", converted) + } + if strings.Contains(request.Prompt, "Required image aspect ratio") { + t.Fatalf("non-xgapi prompt should not be modified: %q", request.Prompt) + } +} + +func TestAppendXGAPIImageAspectRatioToPromptDoesNotDuplicateExplicitRatio(t *testing.T) { + request := dto.ImageRequest{ + Prompt: "Create a poster in 16:9.", + Size: "1792x1024", + } + + got := appendXGAPIImageAspectRatioToPrompt(request.Prompt, request) + + if got != request.Prompt { + t.Fatalf("prompt with explicit ratio should be unchanged, got %q", got) + } +} + +func TestXGAPIImageAspectRatioFromExtraBody(t *testing.T) { + request := dto.ImageRequest{ + Prompt: "Create a poster.", + ExtraBody: []byte(`{"imageConfig":{"aspectRatio":"9:16"}}`), + } + + got := appendXGAPIImageAspectRatioToPrompt(request.Prompt, request) + + if !strings.Contains(got, "Required image aspect ratio: 9:16.") { + t.Fatalf("prompt did not contain aspect ratio from extra_body: %q", got) + } +} diff --git a/relay/channel/openai/constant.go b/relay/channel/openai/constant.go index 14e3d442d4ef..757285ecb486 100644 --- a/relay/channel/openai/constant.go +++ b/relay/channel/openai/constant.go @@ -65,7 +65,9 @@ var ModelList = []string{ "text-davinci-edit-001", "davinci-002", "babbage-002", "dall-e-2", "dall-e-3", - "gpt-image-1", "gpt-image-1-mini", "gpt-image-1.5", + "gpt-image-1", "gpt-image-1-mini", "gpt-image-1.5", "gpt-image-2", + "gemini_3.0_pro_image_preview", "gemini_3.0_pro_image_preview_4K", + "gemini_3.1_flash_image_preview", "gemini_3.1_flash_image_preview_4K", "chatgpt-image-latest", "whisper-1", "tts-1", "tts-1-1106", "tts-1-hd", "tts-1-hd-1106", diff --git a/relay/channel/siliconflow/adaptor.go b/relay/channel/siliconflow/adaptor.go index 3e9bee55adf6..2e638004d202 100644 --- a/relay/channel/siliconflow/adaptor.go +++ b/relay/channel/siliconflow/adaptor.go @@ -1,10 +1,14 @@ package siliconflow import ( + "encoding/base64" + "encoding/json" "errors" "fmt" "io" "net/http" + "sort" + "strings" "github.com/QuantumNous/new-api/common" "github.com/QuantumNous/new-api/dto" @@ -39,24 +43,41 @@ func (a *Adaptor) ConvertAudioRequest(c *gin.Context, info *relaycommon.RelayInf func (a *Adaptor) ConvertImageRequest(c *gin.Context, info *relaycommon.RelayInfo, request dto.ImageRequest) (any, error) { // 解析extra到SFImageRequest里，以填入SiliconFlow特殊字段。若失败重建一个空的。 sfRequest := &SFImageRequest{} - extra, err := common.Marshal(request.Extra) - if err == nil { - err = common.Unmarshal(extra, sfRequest) + if len(request.ExtraBody) > 0 { + if err := common.Unmarshal(request.ExtraBody, sfRequest); err != nil { + return nil, fmt.Errorf("invalid extra_body field: %w", err) + } + } + if len(request.Extra) > 0 { + extra, err := common.Marshal(request.Extra) if err != nil { - sfRequest = &SFImageRequest{} + return nil, fmt.Errorf("invalid extra fields: %w", err) + } + if err = common.Unmarshal(extra, sfRequest); err != nil { + return nil, fmt.Errorf("invalid extra fields: %w", err) } } sfRequest.Model = request.Model sfRequest.Prompt = request.Prompt // 优先使用image_size/batch_size，否则使用OpenAI标准的size/n - if sfRequest.ImageSize == "" { - sfRequest.ImageSize = request.Size + if sfRequest.ImageSize == "" && siliconflowSupportsImageSize(request.Model) { + sfRequest.ImageSize = siliconflowImageSize(request.Model, request.Size) } - if sfRequest.BatchSize == 0 { + if sfRequest.BatchSize == nil { if request.N != nil { - sfRequest.BatchSize = lo.FromPtr(request.N) + sfRequest.BatchSize = lo.ToPtr(lo.FromPtr(request.N)) + } + } + if sfRequest.OutputFormat == "" && len(request.OutputFormat) > 0 { + var outputFormat string + if err := common.Unmarshal(request.OutputFormat, &outputFormat); err != nil { + return nil, fmt.Errorf("invalid output_format field: %w", err) } + sfRequest.OutputFormat = outputFormat + } + if err := applySiliconFlowImageInputs(c, request, sfRequest); err != nil { + return nil, err } return sfRequest, nil @@ -69,12 +90,18 @@ func (a *Adaptor) GetRequestURL(info *relaycommon.RelayInfo) (string, error) { if info.RelayMode == constant.RelayModeRerank { return fmt.Sprintf("%s/v1/rerank", info.ChannelBaseUrl), nil } + if info.RelayMode == constant.RelayModeImagesGenerations || info.RelayMode == constant.RelayModeImagesEdits { + return fmt.Sprintf("%s/v1/images/generations", strings.TrimRight(info.ChannelBaseUrl, "/")), nil + } return relaycommon.GetFullRequestURL(info.ChannelBaseUrl, info.RequestURLPath, info.ChannelType), nil } func (a *Adaptor) SetupRequestHeader(c *gin.Context, req *http.Header, info *relaycommon.RelayInfo) error { channel.SetupApiRequestHeader(info, c, req) req.Set("Authorization", fmt.Sprintf("Bearer %s", info.ApiKey)) + if info.RelayMode == constant.RelayModeImagesGenerations || info.RelayMode == constant.RelayModeImagesEdits { + req.Set("Content-Type", "application/json") + } return nil } @@ -98,6 +125,9 @@ func (a *Adaptor) ConvertOpenAIResponsesRequest(c *gin.Context, info *relaycommo } func (a *Adaptor) DoRequest(c *gin.Context, info *relaycommon.RelayInfo, requestBody io.Reader) (any, error) { + if info.RelayMode == constant.RelayModeImagesGenerations || info.RelayMode == constant.RelayModeImagesEdits { + return channel.DoApiRequest(a, c, info, requestBody) + } adaptor := openai.Adaptor{} return adaptor.DoRequest(c, info, requestBody) } @@ -114,6 +144,8 @@ func (a *Adaptor) DoResponse(c *gin.Context, resp *http.Response, info *relaycom switch info.RelayMode { case constant.RelayModeRerank: usage, err = siliconflowRerankHandler(c, info, resp) + case constant.RelayModeImagesGenerations, constant.RelayModeImagesEdits: + usage, err = siliconflowImageHandler(c, info, resp) default: adaptor := openai.Adaptor{} usage, err = adaptor.DoResponse(c, resp, info) @@ -128,3 +160,175 @@ func (a *Adaptor) GetModelList() []string { func (a *Adaptor) GetChannelName() string { return ChannelName } + +func siliconflowSupportsImageSize(modelName string) bool { + modelName = strings.TrimSpace(modelName) + return !strings.EqualFold(modelName, "Qwen/Qwen-Image-Edit") && + !strings.EqualFold(modelName, "Qwen/Qwen-Image-Edit-2509") +} + +func siliconflowImageSize(modelName string, size string) string { + size = strings.TrimSpace(size) + if strings.EqualFold(strings.TrimSpace(modelName), "Qwen/Qwen-Image") { + switch size { + case "", "256x256", "512x512", "1024x1024": + return "1328x1328" + case "1792x1024": + return "1664x928" + case "1024x1792": + return "928x1664" + case "1536x1024": + return "1584x1056" + case "1024x1536": + return "1056x1584" + default: + return size + } + } + if size == "" { + return "1024x1024" + } + return size +} + +func applySiliconFlowImageInputs(c *gin.Context, request dto.ImageRequest, sfRequest *SFImageRequest) error { + var imageValues []string + if values, err := parseSiliconFlowImageValues(request.Image); err != nil { + return fmt.Errorf("invalid image field: %w", err) + } else { + imageValues = append(imageValues, values...) + } + if values, err := parseSiliconFlowImageValues(request.Images); err != nil { + return fmt.Errorf("invalid images field: %w", err) + } else { + imageValues = append(imageValues, values...) + } + if values, err := siliconflowMultipartImageValues(c); err != nil { + return err + } else { + imageValues = append(imageValues, values...) + } + return setSiliconFlowImages(sfRequest, imageValues) +} + +func parseSiliconFlowImageValues(raw json.RawMessage) ([]string, error) { + trimmed := strings.TrimSpace(string(raw)) + if trimmed == "" || trimmed == "null" { + return nil, nil + } + + var imageValue string + if err := common.Unmarshal(raw, &imageValue); err == nil { + imageValue = strings.TrimSpace(imageValue) + if imageValue == "" { + return nil, nil + } + return []string{imageValue}, nil + } + + var images []json.RawMessage + if err := common.Unmarshal(raw, &images); err == nil { + values := make([]string, 0, len(images)) + for _, item := range images { + itemValues, itemErr := parseSiliconFlowImageValues(item) + if itemErr != nil { + return nil, itemErr + } + values = append(values, itemValues...) + } + return values, nil + } + + var object map[string]json.RawMessage + if err := common.Unmarshal(raw, &object); err != nil { + return nil, err + } + for _, key := range []string{"url", "image", "image_url"} { + if len(object[key]) == 0 { + continue + } + if key != "image_url" { + return parseSiliconFlowImageValues(object[key]) + } + values, err := parseSiliconFlowImageValues(object[key]) + if err == nil && len(values) > 0 { + return values, nil + } + var imageURLObject map[string]json.RawMessage + if err := common.Unmarshal(object[key], &imageURLObject); err != nil { + return nil, err + } + return parseSiliconFlowImageValues(imageURLObject["url"]) + } + return nil, nil +} + +func siliconflowMultipartImageValues(c *gin.Context) ([]string, error) { + if c == nil || c.Request == nil || !strings.Contains(c.Request.Header.Get("Content-Type"), "multipart/form-data") { + return nil, nil + } + form, err := common.ParseMultipartFormReusable(c) + if err != nil { + return nil, fmt.Errorf("failed to parse image edit form request: %w", err) + } + + var files []string + if _, ok := form.File["image"]; ok { + files = append(files, "image") + } + if _, ok := form.File["image[]"]; ok { + files = append(files, "image[]") + } + var indexedKeys []string + for key := range form.File { + if strings.HasPrefix(key, "image[") && key != "image[]" { + indexedKeys = append(indexedKeys, key) + } + } + sort.Strings(indexedKeys) + files = append(files, indexedKeys...) + + var values []string + for _, key := range files { + for _, fileHeader := range form.File[key] { + file, err := fileHeader.Open() + if err != nil { + return nil, fmt.Errorf("failed to open image file: %w", err) + } + imageData, readErr := io.ReadAll(file) + _ = file.Close() + if readErr != nil { + return nil, fmt.Errorf("failed to read image file: %w", readErr) + } + + mimeType := strings.TrimSpace(fileHeader.Header.Get("Content-Type")) + if mimeType == "" || mimeType == "application/octet-stream" { + mimeType = http.DetectContentType(imageData) + } + values = append(values, fmt.Sprintf("data:%s;base64,%s", mimeType, base64.StdEncoding.EncodeToString(imageData))) + } + } + return values, nil +} + +func setSiliconFlowImages(sfRequest *SFImageRequest, imageValues []string) error { + fields := []*string{&sfRequest.Image, &sfRequest.Image2, &sfRequest.Image3} + for _, imageValue := range imageValues { + imageValue = strings.TrimSpace(imageValue) + if imageValue == "" { + continue + } + assigned := false + for _, field := range fields { + if *field == "" { + *field = imageValue + assigned = true + break + } + } + if !assigned { + return errors.New("SiliconFlow image models support at most 3 input images") + } + } + return nil +} diff --git a/relay/channel/siliconflow/constant.go b/relay/channel/siliconflow/constant.go index fea6fcd4896b..3635963c6208 100644 --- a/relay/channel/siliconflow/constant.go +++ b/relay/channel/siliconflow/constant.go @@ -9,6 +9,10 @@ var ModelList = []string{ //"stabilityai/sd-turbo", //"stabilityai/sdxl-turbo", "ByteDance/SDXL-Lightning", + "baidu/ERNIE-Image-Turbo", + "Qwen/Qwen-Image", + "Tongyi-MAI/Z-Image", + "Qwen/Qwen-Image-Edit-2509", "deepseek-ai/deepseek-llm-67b-chat", "Qwen/Qwen1.5-14B-Chat", "Qwen/Qwen1.5-7B-Chat", diff --git a/relay/channel/siliconflow/dto.go b/relay/channel/siliconflow/dto.go index 1009751074a8..fbf3c3aab377 100644 --- a/relay/channel/siliconflow/dto.go +++ b/relay/channel/siliconflow/dto.go @@ -17,16 +17,31 @@ type SFRerankResponse struct { } type SFImageRequest struct { - Model string `json:"model"` - Prompt string `json:"prompt"` - NegativePrompt string `json:"negative_prompt,omitempty"` - ImageSize string `json:"image_size,omitempty"` - BatchSize uint `json:"batch_size,omitempty"` - Seed uint64 `json:"seed,omitempty"` - NumInferenceSteps uint `json:"num_inference_steps,omitempty"` - GuidanceScale float64 `json:"guidance_scale,omitempty"` - Cfg float64 `json:"cfg,omitempty"` - Image string `json:"image,omitempty"` - Image2 string `json:"image2,omitempty"` - Image3 string `json:"image3,omitempty"` + Model string `json:"model"` + Prompt string `json:"prompt"` + NegativePrompt string `json:"negative_prompt,omitempty"` + ImageSize string `json:"image_size,omitempty"` + BatchSize *uint `json:"batch_size,omitempty"` + Seed *uint64 `json:"seed,omitempty"` + NumInferenceSteps *uint `json:"num_inference_steps,omitempty"` + GuidanceScale *float64 `json:"guidance_scale,omitempty"` + Cfg *float64 `json:"cfg,omitempty"` + OutputFormat string `json:"output_format,omitempty"` + Image string `json:"image,omitempty"` + Image2 string `json:"image2,omitempty"` + Image3 string `json:"image3,omitempty"` +} + +type SFImageResponse struct { + Images []SFImageResponseItem `json:"images"` + Timings any `json:"timings,omitempty"` + Seed any `json:"seed,omitempty"` + Code any `json:"code,omitempty"` + Message string `json:"message,omitempty"` + Data any `json:"data,omitempty"` +} + +type SFImageResponseItem struct { + Url string `json:"url"` + B64Json string `json:"b64_json,omitempty"` } diff --git a/relay/channel/siliconflow/relay-siliconflow.go b/relay/channel/siliconflow/relay-siliconflow.go index 421731fb1a96..4a89a4e516b0 100644 --- a/relay/channel/siliconflow/relay-siliconflow.go +++ b/relay/channel/siliconflow/relay-siliconflow.go @@ -1,10 +1,11 @@ package siliconflow import ( - "encoding/json" + "fmt" "io" "net/http" + "github.com/QuantumNous/new-api/common" "github.com/QuantumNous/new-api/dto" relaycommon "github.com/QuantumNous/new-api/relay/common" "github.com/QuantumNous/new-api/service" @@ -20,7 +21,7 @@ func siliconflowRerankHandler(c *gin.Context, info *relaycommon.RelayInfo, resp } service.CloseResponseBodyGracefully(resp) var siliconflowResp SFRerankResponse - err = json.Unmarshal(responseBody, &siliconflowResp) + err = common.Unmarshal(responseBody, &siliconflowResp) if err != nil { return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError) } @@ -34,7 +35,7 @@ func siliconflowRerankHandler(c *gin.Context, info *relaycommon.RelayInfo, resp Usage: *usage, } - jsonResponse, err := json.Marshal(rerankResp) + jsonResponse, err := common.Marshal(rerankResp) if err != nil { return nil, types.NewError(err, types.ErrorCodeBadResponseBody) } @@ -43,3 +44,45 @@ func siliconflowRerankHandler(c *gin.Context, info *relaycommon.RelayInfo, resp service.IOCopyBytesGracefully(c, resp, jsonResponse) return usage, nil } + +func siliconflowImageHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Response) (*dto.Usage, *types.NewAPIError) { + responseBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, types.NewOpenAIError(err, types.ErrorCodeReadResponseBodyFailed, http.StatusInternalServerError) + } + service.CloseResponseBodyGracefully(resp) + + var siliconflowResp SFImageResponse + if err = common.Unmarshal(responseBody, &siliconflowResp); err != nil { + return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError) + } + if len(siliconflowResp.Images) == 0 && siliconflowResp.Message != "" { + return nil, types.WithOpenAIError(types.OpenAIError{ + Message: siliconflowResp.Message, + Type: "siliconflow_error", + Code: fmt.Sprintf("%v", siliconflowResp.Code), + }, resp.StatusCode) + } + + imageResponse := dto.ImageResponse{ + Created: info.StartTime.Unix(), + Metadata: responseBody, + } + for _, image := range siliconflowResp.Images { + imageResponse.Data = append(imageResponse.Data, dto.ImageData{ + Url: image.Url, + B64Json: image.B64Json, + }) + } + if len(imageResponse.Data) > 0 { + info.PriceData.AddOtherRatio("n", float64(len(imageResponse.Data))) + } + + jsonResponse, err := common.Marshal(imageResponse) + if err != nil { + return nil, types.NewError(err, types.ErrorCodeBadResponseBody) + } + c.Writer.Header().Set("Content-Type", "application/json") + service.IOCopyBytesGracefully(c, resp, jsonResponse) + return &dto.Usage{}, nil +} diff --git a/relay/channel/task/openaivideo/adaptor.go b/relay/channel/task/openaivideo/adaptor.go new file mode 100644 index 000000000000..4807d33735d6 --- /dev/null +++ b/relay/channel/task/openaivideo/adaptor.go @@ -0,0 +1,384 @@ +package openaivideo + +import ( + "bytes" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/textproto" + "strconv" + "strings" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/constant" + "github.com/QuantumNous/new-api/dto" + "github.com/QuantumNous/new-api/model" + "github.com/QuantumNous/new-api/relay/channel" + taskcommon "github.com/QuantumNous/new-api/relay/channel/task/taskcommon" + relaycommon "github.com/QuantumNous/new-api/relay/common" + "github.com/QuantumNous/new-api/service" + + "github.com/gin-gonic/gin" + "github.com/pkg/errors" +) + +type TaskAdaptor struct { + taskcommon.BaseBilling + ChannelType int + apiKey string + baseURL string + prov provider +} + +func (a *TaskAdaptor) Init(info *relaycommon.RelayInfo) { + a.ChannelType = info.ChannelType + a.baseURL = info.ChannelBaseUrl + a.apiKey = info.ApiKey + a.prov = getProviderForRelayInfo(info) +} + +func (a *TaskAdaptor) ValidateRequestAndSetAction(c *gin.Context, info *relaycommon.RelayInfo) (taskErr *dto.TaskError) { + if info.Action == constant.TaskActionRemix { + return validateRemixRequest(c) + } + return relaycommon.ValidateMultipartDirect(c, info) +} + +func validateRemixRequest(c *gin.Context) *dto.TaskError { + var req relaycommon.TaskSubmitReq + if err := common.UnmarshalBodyReusable(c, &req); err != nil { + return service.TaskErrorWrapperLocal(err, "invalid_request", http.StatusBadRequest) + } + if strings.TrimSpace(req.Prompt) == "" { + return service.TaskErrorWrapperLocal(fmt.Errorf("field prompt is required"), "invalid_request", http.StatusBadRequest) + } + c.Set("task_request", req) + return nil +} + +func (a *TaskAdaptor) EstimateBilling(c *gin.Context, info *relaycommon.RelayInfo) map[string]float64 { + if info.Action == constant.TaskActionRemix { + return nil + } + + req, err := relaycommon.GetTaskRequest(c) + if err != nil { + return nil + } + + seconds, _ := strconv.Atoi(req.Seconds) + if seconds == 0 { + seconds = req.Duration + } + if seconds <= 0 { + seconds = 8 + } + if _, ok := a.prov.(*xbSoraProvider); ok { + seconds = normalizeXBSoraDuration(seconds, info.UpstreamModelName) + } + + ratios := map[string]float64{ + "seconds": float64(seconds), + "size": 1, + } + size := req.Size + if size == "1792x1024" || size == "1024x1792" { + ratios["size"] = 1.666667 + } + return ratios +} + +func (a *TaskAdaptor) BuildRequestURL(info *relaycommon.RelayInfo) (string, error) { + return a.prov.submitURL(a.baseURL), nil +} + +func (a *TaskAdaptor) BuildRequestHeader(c *gin.Context, req *http.Request, info *relaycommon.RelayInfo) error { + if headerSetter, ok := a.prov.(requestHeaderSetter); ok { + headerSetter.setupRequestHeader(req, a.apiKey) + } else { + req.Header.Set("Authorization", "Bearer "+a.apiKey) + } + req.Header.Set("Content-Type", c.Request.Header.Get("Content-Type")) + return nil +} + +func (a *TaskAdaptor) BuildRequestBody(c *gin.Context, info *relaycommon.RelayInfo) (io.Reader, error) { + storage, err := common.GetBodyStorage(c) + if err != nil { + return nil, errors.Wrap(err, "get_request_body_failed") + } + cachedBody, err := storage.Bytes() + if err != nil { + return nil, errors.Wrap(err, "read_body_bytes_failed") + } + contentType := c.GetHeader("Content-Type") + + if strings.HasPrefix(contentType, "application/json") { + var bodyMap map[string]interface{} + if err := common.Unmarshal(cachedBody, &bodyMap); err == nil { + imageCount := countRequestImages(bodyMap) + hasImages := imageCount > 0 + bodyMap["model"] = a.prov.mapModelForImages(info.UpstreamModelName, hasImages) + if normalizer, ok := a.prov.(requestNormalizer); ok { + normalizer.normalizeJSONRequest(bodyMap, info.OriginModelName, info.UpstreamModelName, imageCount) + } + + if a.prov.needsMultipart() { + return a.jsonToMultipart(c, bodyMap) + } + + if newBody, err := common.Marshal(bodyMap); err == nil { + return bytes.NewReader(newBody), nil + } + } + return bytes.NewReader(cachedBody), nil + } + + if strings.Contains(contentType, "multipart/form-data") { + formData, err := common.ParseMultipartFormReusable(c) + if err != nil { + return bytes.NewReader(cachedBody), nil + } + hasImages := len(formData.Value["images"]) > 0 || len(formData.Value["image"]) > 0 || len(formData.File["images"]) > 0 || len(formData.File["image"]) > 0 + imageCount := len(formData.Value["images"]) + len(formData.Value["image"]) + len(formData.File["images"]) + len(formData.File["image"]) + mappedModel := a.prov.mapModelForImages(info.UpstreamModelName, hasImages) + if normalizer, ok := a.prov.(requestNormalizer); ok { + formData.Value["model"] = []string{mappedModel} + normalizer.normalizeMultipartRequest(formData.Value, info.OriginModelName, mappedModel, imageCount) + if modelValue := firstValue(formData.Value["model"]); modelValue != "" { + mappedModel = modelValue + } + } + if jsonProvider, ok := a.prov.(jsonBodyProvider); ok && jsonProvider.forceJSONBody() { + if len(formData.File) > 0 { + return nil, fmt.Errorf("multipart file upload is not supported by this video provider; use image URLs") + } + bodyMap := multipartValuesToMap(formData.Value) + bodyMap["model"] = mappedModel + newBody, err := common.Marshal(bodyMap) + if err != nil { + return nil, err + } + c.Request.Header.Set("Content-Type", "application/json") + return bytes.NewReader(newBody), nil + } + var buf bytes.Buffer + writer := multipart.NewWriter(&buf) + writer.WriteField("model", mappedModel) + for key, values := range formData.Value { + if key == "model" { + continue + } + for _, v := range values { + writer.WriteField(key, v) + } + } + for fieldName, fileHeaders := range formData.File { + for _, fh := range fileHeaders { + f, err := fh.Open() + if err != nil { + continue + } + ct := fh.Header.Get("Content-Type") + if ct == "" || ct == "application/octet-stream" { + buf512 := make([]byte, 512) + n, _ := io.ReadFull(f, buf512) + ct = http.DetectContentType(buf512[:n]) + f.Close() + f, err = fh.Open() + if err != nil { + continue + } + } + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, fieldName, fh.Filename)) + h.Set("Content-Type", ct) + part, err := writer.CreatePart(h) + if err != nil { + f.Close() + continue + } + io.Copy(part, f) + f.Close() + } + } + writer.Close() + c.Request.Header.Set("Content-Type", writer.FormDataContentType()) + return &buf, nil + } + + return common.ReaderOnly(storage), nil +} + +func countRequestImages(bodyMap map[string]interface{}) int { + count := 0 + for _, key := range []string{"images", "image", "image_urls", "reference_images", "reference_image_urls", "image_url", "file_paths"} { + if value, ok := bodyMap[key]; ok { + count += countImageValue(value) + } + } + if refs, ok := bodyMap["image_reference"]; ok { + count += countImageValue(refs) + } + return count +} + +func countImageValue(value interface{}) int { + switch v := value.(type) { + case []interface{}: + count := 0 + for _, item := range v { + if countImageValue(item) > 0 { + count++ + } + } + return count + case []string: + count := 0 + for _, item := range v { + if strings.TrimSpace(item) != "" { + count++ + } + } + return count + case map[string]interface{}: + if imageURL, ok := v["image_url"].(map[string]interface{}); ok { + if url, ok := imageURL["url"].(string); ok && strings.TrimSpace(url) != "" { + return 1 + } + } + if url, ok := v["url"].(string); ok && strings.TrimSpace(url) != "" { + return 1 + } + case string: + if strings.TrimSpace(v) != "" { + return 1 + } + } + return 0 +} + +func (a *TaskAdaptor) jsonToMultipart(c *gin.Context, bodyMap map[string]interface{}) (io.Reader, error) { + var buf bytes.Buffer + writer := multipart.NewWriter(&buf) + + for key, val := range bodyMap { + switch v := val.(type) { + case string: + writer.WriteField(key, v) + case float64, int, int64: + writer.WriteField(key, fmt.Sprintf("%v", v)) + case bool: + writer.WriteField(key, fmt.Sprintf("%v", v)) + default: + b, _ := common.Marshal(v) + writer.WriteField(key, string(b)) + } + } + + writer.Close() + c.Request.Header.Set("Content-Type", writer.FormDataContentType()) + return &buf, nil +} + +func (a *TaskAdaptor) DoRequest(c *gin.Context, info *relaycommon.RelayInfo, requestBody io.Reader) (*http.Response, error) { + return channel.DoTaskApiRequest(a, c, info, requestBody) +} + +func (a *TaskAdaptor) DoResponse(c *gin.Context, resp *http.Response, info *relaycommon.RelayInfo) (taskID string, taskData []byte, taskErr *dto.TaskError) { + responseBody, err := io.ReadAll(resp.Body) + if err != nil { + taskErr = service.TaskErrorWrapper(err, "read_response_body_failed", http.StatusInternalServerError) + return + } + _ = resp.Body.Close() + + upstreamID, err := a.prov.parseSubmitResponse(responseBody) + if err != nil { + taskErr = service.TaskErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError) + return + } + + c.JSON(http.StatusOK, a.prov.buildSubmitResponseBody(info, upstreamID)) + return upstreamID, responseBody, nil +} + +func (a *TaskAdaptor) FetchTask(baseUrl, key string, body map[string]any, proxy string) (*http.Response, error) { + taskID, ok := body["task_id"].(string) + if !ok { + return nil, fmt.Errorf("invalid task_id") + } + + prov := getProviderForTaskFetch(baseUrl, body) + uri := prov.queryURL(baseUrl, taskID) + + req, err := http.NewRequest(http.MethodGet, uri, nil) + if err != nil { + return nil, err + } + + if headerSetter, ok := prov.(requestHeaderSetter); ok { + headerSetter.setupRequestHeader(req, key) + } else { + req.Header.Set("Authorization", "Bearer "+key) + } + + client, err := service.GetHttpClientWithProxy(proxy) + if err != nil { + return nil, fmt.Errorf("new proxy http client failed: %w", err) + } + return client.Do(req) +} + +func multipartValuesToMap(values map[string][]string) map[string]interface{} { + bodyMap := make(map[string]interface{}, len(values)) + for key, vals := range values { + if len(vals) == 0 { + continue + } + if len(vals) == 1 { + bodyMap[key] = vals[0] + continue + } + copied := make([]string, len(vals)) + copy(copied, vals) + bodyMap[key] = copied + } + return bodyMap +} + +func (a *TaskAdaptor) GetModelList() []string { + return ModelList +} + +func (a *TaskAdaptor) GetChannelName() string { + return ChannelName +} + +func (a *TaskAdaptor) ParseTaskResult(respBody []byte) (*relaycommon.TaskInfo, error) { + return a.prov.parseQueryResponse(respBody) +} + +func (a *TaskAdaptor) ConvertToOpenAIVideo(task *model.Task) ([]byte, error) { + video := dto.NewOpenAIVideo() + video.ID = task.TaskID + video.Model = task.Properties.OriginModelName + if video.Model == "" { + video.Model = task.Properties.UpstreamModelName + } + video.Status = task.Status.ToVideoStatus() + video.SetProgressStr(task.Progress) + video.VideoURL = task.GetResultURL() + if strings.HasPrefix(video.VideoURL, "runway:") || isXBSoraProtectedResultURL(video.VideoURL) || isLK888ResultURL(video.VideoURL) { + video.VideoURL = taskcommon.BuildProxyURL(task.TaskID) + } + video.CreatedAt = task.CreatedAt + if task.FinishTime > 0 { + video.CompletedAt = task.FinishTime + } else if task.UpdatedAt > 0 { + video.CompletedAt = task.UpdatedAt + } + + return common.Marshal(video) +} diff --git a/relay/channel/task/openaivideo/apexerapi.go b/relay/channel/task/openaivideo/apexerapi.go new file mode 100644 index 000000000000..0d2ba44d4204 --- /dev/null +++ b/relay/channel/task/openaivideo/apexerapi.go @@ -0,0 +1,141 @@ +package openaivideo + +import ( + "fmt" + "strings" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/model" + relaycommon "github.com/QuantumNous/new-api/relay/common" + "github.com/pkg/errors" +) + +type apexerapiProvider struct{} + +type apexerapiSubmitResp struct { + ID string `json:"id"` + TaskID string `json:"task_id"` + Status string `json:"status"` + StatusUpdateTime int64 `json:"status_update_time"` +} + +func (p *apexerapiProvider) submitURL(baseURL string) string { + return baseURL + "/v1/videos" +} + +func (p *apexerapiProvider) queryURL(baseURL, taskID string) string { + return baseURL + "/v1/videos/" + taskID +} + +func (p *apexerapiProvider) parseSubmitResponse(body []byte) (string, error) { + var resp apexerapiSubmitResp + if err := common.Unmarshal(body, &resp); err != nil { + return "", errors.Wrap(err, "unmarshal apexerapi submit response failed") + } + if resp.ID == "" { + return "", errors.Errorf("apexerapi submit returned empty id, body=%s", string(body)) + } + return resp.ID, nil +} + +func (p *apexerapiProvider) parseQueryResponse(body []byte) (*relaycommon.TaskInfo, error) { + var resp struct { + ID string `json:"id"` + Status string `json:"status"` + VideoURL *string `json:"video_url"` + Progress int `json:"progress"` + CompletedAt int64 `json:"completed_at"` + Error *struct { + Message string `json:"message"` + Code string `json:"code"` + } `json:"error,omitempty"` + } + if err := common.Unmarshal(body, &resp); err != nil { + return nil, errors.Wrap(err, "unmarshal apexerapi query response failed") + } + + taskInfo := &relaycommon.TaskInfo{ + TaskID: resp.ID, + Status: statusToTaskStatus(resp.Status), + } + + if taskInfo.Status == model.TaskStatusSuccess && resp.VideoURL != nil && *resp.VideoURL != "" { + taskInfo.Url = *resp.VideoURL + } + + if taskInfo.Status == model.TaskStatusFailure { + if resp.Error != nil { + taskInfo.Reason = resp.Error.Message + } else { + taskInfo.Reason = resp.Status + } + } + + if resp.Progress > 0 && resp.Progress < 100 { + taskInfo.Progress = fmt.Sprintf("%d%%", resp.Progress) + } + + return taskInfo, nil +} + +func (p *apexerapiProvider) buildSubmitResponseBody(info *relaycommon.RelayInfo, upstreamTaskID string) any { + return map[string]any{ + "id": info.PublicTaskID, + "task_id": info.PublicTaskID, + "object": "video", + "model": info.OriginModelName, + "status": "queued", + "progress": 0, + "created_at": 0, + } +} + +func (p *apexerapiProvider) needsMultipart() bool { return false } + +func (p *apexerapiProvider) mapModelForImages(model string, hasImages bool) string { + if mapped, ok := apexerModelMap[model]; ok { + return mapped + } + return strings.ReplaceAll(model, "-", "_") +} + +func (p *apexerapiProvider) normalizeJSONRequest(bodyMap map[string]interface{}, originModel, upstreamModel string, imageCount int) { + if _, ok := bodyMap["type"]; !ok { + bodyMap["type"] = inferApexerVideoType(originModel, upstreamModel, imageCount) + } +} + +func (p *apexerapiProvider) normalizeMultipartRequest(values map[string][]string, originModel, upstreamModel string, imageCount int) { + if _, ok := values["type"]; !ok { + values["type"] = []string{fmt.Sprintf("%d", inferApexerVideoType(originModel, upstreamModel, imageCount))} + } +} + +func inferApexerVideoType(originModel, upstreamModel string, imageCount int) int { + model := originModel + if model == "" { + model = upstreamModel + } + model = strings.ToLower(model) + + if strings.Contains(model, "components") || imageCount > 2 { + return 3 + } + if imageCount > 0 { + return 2 + } + return 1 +} + +var apexerModelMap = map[string]string{ + "veo3.1": "veo3.1_relaxed", + "veo3.1-fast": "veo3.1_fast", + "veo3.1-pro": "veo3.1_pro", + "veo3.1-4k": "veo3.1_relaxed_4k", + "veo3.1-fast-4k": "veo3.1_fast_4k", + "veo3.1-pro-4k": "veo3.1_pro_4k", + "veo3.1-components": "veo3.1_relaxed", + "veo3.1-fast-components": "veo3.1_fast", + "veo3.1-components-4k": "veo3.1_relaxed_4k", + "veo3.1-fast-components-4k": "veo3.1_fast_4k", +} diff --git a/relay/channel/task/openaivideo/bltcy.go b/relay/channel/task/openaivideo/bltcy.go new file mode 100644 index 000000000000..74daf4437021 --- /dev/null +++ b/relay/channel/task/openaivideo/bltcy.go @@ -0,0 +1,110 @@ +package openaivideo + +import ( + "fmt" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/model" + relaycommon "github.com/QuantumNous/new-api/relay/common" + "github.com/pkg/errors" +) + +type bltcySubmitResponse struct { + TaskID string `json:"task_id"` +} + +type bltcyQueryResponse struct { + TaskID string `json:"task_id"` + Status string `json:"status"` + FailReason string `json:"fail_reason"` + Progress string `json:"progress"` + Data struct { + Output string `json:"output"` + } `json:"data"` + Error *struct { + Message string `json:"message"` + Code string `json:"code"` + } `json:"error,omitempty"` +} + +type bltcyProvider struct{} + +func (p *bltcyProvider) submitURL(baseURL string) string { + return fmt.Sprintf("%s/v2/videos/generations", baseURL) +} + +func (p *bltcyProvider) queryURL(baseURL, taskID string) string { + return fmt.Sprintf("%s/v2/videos/generations/%s", baseURL, taskID) +} + +func (p *bltcyProvider) parseSubmitResponse(body []byte) (string, error) { + var resp bltcySubmitResponse + if err := common.Unmarshal(body, &resp); err != nil { + return "", errors.Wrap(err, "unmarshal bltcy submit response failed") + } + if resp.TaskID == "" { + return "", fmt.Errorf("bltcy response task_id is empty") + } + return resp.TaskID, nil +} + +func (p *bltcyProvider) parseQueryResponse(body []byte) (*relaycommon.TaskInfo, error) { + var resp bltcyQueryResponse + if err := common.Unmarshal(body, &resp); err != nil { + return nil, errors.Wrap(err, "unmarshal bltcy query response failed") + } + + ti := &relaycommon.TaskInfo{Code: 0} + ti.Status = statusToTaskStatus(resp.Status) + + if ti.Status == model.TaskStatusSuccess && resp.Data.Output != "" { + ti.Url = resp.Data.Output + } + if ti.Status == model.TaskStatusFailure { + if resp.FailReason != "" { + ti.Reason = resp.FailReason + } else if resp.Error != nil { + ti.Reason = resp.Error.Message + } else { + ti.Reason = "task failed" + } + } + if resp.Progress != "" { + ti.Progress = resp.Progress + } + return ti, nil +} + +func (p *bltcyProvider) buildSubmitResponseBody(info *relaycommon.RelayInfo, upstreamTaskID string) any { + return map[string]string{ + "id": info.PublicTaskID, + "task_id": info.PublicTaskID, + } +} + +func (p *bltcyProvider) needsMultipart() bool { return false } + +func (p *bltcyProvider) mapModelForImages(model string, hasImages bool) string { + if !hasImages { + return model + } + + if mapped, ok := bltcyFramesModelMap[model]; ok { + return mapped + } + + return model +} + +var bltcyFramesModelMap = map[string]string{ + "veo3.1": "veo3.1", + "veo3.1-fast": "veo3.1-fast", + "veo3.1-pro": "veo3.1-pro", + "veo3.1-pro-4k": "veo3.1-pro-4k", + "veo3.1-components": "veo3.1-components", + "veo3.1-fast-components": "veo3.1-fast-components", + "veo3": "veo3-pro-frames", + "veo3-fast": "veo3-fast-frames", + "veo2-fast": "veo2-fast-frames", + "veo2": "veo2-fast-frames", +} diff --git a/relay/channel/task/openaivideo/constants.go b/relay/channel/task/openaivideo/constants.go new file mode 100644 index 000000000000..0adc052b5364 --- /dev/null +++ b/relay/channel/task/openaivideo/constants.go @@ -0,0 +1,61 @@ +package openaivideo + +var ModelList = []string{ + "veo2", + "veo2-fast", + "veo2-fast-frames", + "veo2-fast-components", + "veo2-pro", + "veo3", + "veo3-fast", + "veo3-pro", + "veo3-pro-frames", + "veo3-fast-frames", + "veo3.1-fast", + "veo3.1", + "veo3.1-pro", + "veo3.1-pro-4k", + "veo3.1-components", + "veo3.1-lite", + "veo3.1-lite-4k", + "veo-3.1", + "veo3.1-fast-4k", + "veo3.1-4k", + "veo3.1-components-4k", + "veo3.1-fast-components", + "veo3.1-fast-components-4k", + "sora-2", + "sora-2-pro", + "sora-2-8s", + "sora-2-12s", + "grok-imagine-1.0-video", + "grok-imagine-1.0-video-20s", + "grok-imagine-1.0-video-30s", + "grok-imagine-video", + "grok-imagine-video-1.5-preview", + "xb-sora2", + "openai-sora-2", + "sora-2-pro-text-to-video", + "sora-2-image-to-video", + "seedance-2", + "gen4-turbo", + "wan-2.6-flash", + "kling-2.5-turbo-standard", + "gen4.5", + "happyhorse-1", + "wan-2.6", + "kling-2.5-turbo-pro", + "kling-2.6", + "wan-2.2-animate", + "kling-3.0-pro", + "kling-3.0-standard", + "kling-3.0-4k", + "kling-3.0-motion-control", + "kling-o3-pro", + "kling-o3-standard", + "kling-o3-4k", + "kling-2.6-motion-control", + "grok-video-3", +} + +var ChannelName = "openaivideo" diff --git a/relay/channel/task/openaivideo/lk888.go b/relay/channel/task/openaivideo/lk888.go new file mode 100644 index 000000000000..aeebb5a4c214 --- /dev/null +++ b/relay/channel/task/openaivideo/lk888.go @@ -0,0 +1,396 @@ +package openaivideo + +import ( + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/model" + relaycommon "github.com/QuantumNous/new-api/relay/common" + "github.com/pkg/errors" +) + +type lk888Provider struct{} + +type lk888SubmitResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + TaskID any `json:"task_id"` + TaskIDs []any `json:"任务ids"` + } `json:"data"` + Error *struct { + Message string `json:"message"` + Type string `json:"type"` + } `json:"error,omitempty"` +} + +type lk888StatusResponse struct { + TaskID any `json:"task_id"` + Model string `json:"model"` + State string `json:"state"` + Status string `json:"status"` + StatusGroup string `json:"status_group"` + Progress string `json:"progress"` + IsFinal bool `json:"is_final"` + ResultURL string `json:"result_url"` + ResultType string `json:"result_type"` + Cost float64 `json:"cost"` + Error any `json:"error"` + Refunded bool `json:"refunded"` +} + +func (p *lk888Provider) submitURL(baseURL string) string { + return lk888APIBase(baseURL) + "/v1/media/generate" +} + +func (p *lk888Provider) queryURL(baseURL, taskID string) string { + return lk888APIBase(baseURL) + "/v1/skills/task-status?task_id=" + url.QueryEscape(taskID) +} + +func (p *lk888Provider) parseSubmitResponse(body []byte) (string, error) { + var resp lk888SubmitResponse + if err := common.Unmarshal(body, &resp); err != nil { + return "", errors.Wrap(err, "unmarshal lk888 submit response failed") + } + if resp.Code != 200 { + if resp.Error != nil && resp.Error.Message != "" { + return "", fmt.Errorf("lk888 submit failed: %s", resp.Error.Message) + } + return "", fmt.Errorf("lk888 submit failed: %s", resp.Msg) + } + if taskID := lk888StringID(resp.Data.TaskID); taskID != "" { + return taskID, nil + } + for _, id := range resp.Data.TaskIDs { + if taskID := lk888StringID(id); taskID != "" { + return taskID, nil + } + } + return "", fmt.Errorf("lk888 response data.task_id is empty") +} + +func (p *lk888Provider) parseQueryResponse(body []byte) (*relaycommon.TaskInfo, error) { + var resp lk888StatusResponse + if err := common.Unmarshal(body, &resp); err != nil { + return nil, errors.Wrap(err, "unmarshal lk888 task status response failed") + } + + ti := &relaycommon.TaskInfo{ + Code: 0, + TaskID: lk888StringID(resp.TaskID), + Status: lk888TaskStatus(resp), + Progress: lk888Progress(resp.Progress), + } + if ti.Status == model.TaskStatusSuccess { + if strings.TrimSpace(resp.ResultURL) == "" { + ti.Status = model.TaskStatusFailure + ti.Reason = "lk888 completed without result_url" + } else { + ti.Url = strings.TrimSpace(resp.ResultURL) + } + } + if ti.Status == model.TaskStatusFailure { + ti.Reason = lk888ErrorMessage(resp.Error) + if ti.Reason == "" { + ti.Reason = firstNonEmpty(resp.Status, resp.State, "task failed") + } + } + return ti, nil +} + +func (p *lk888Provider) buildSubmitResponseBody(info *relaycommon.RelayInfo, upstreamTaskID string) any { + return map[string]any{ + "id": info.PublicTaskID, + "task_id": info.PublicTaskID, + "object": "video", + "model": info.OriginModelName, + "status": "queued", + "progress": 0, + "created_at": 0, + } +} + +func (p *lk888Provider) needsMultipart() bool { return false } + +func (p *lk888Provider) forceJSONBody() bool { return true } + +func (p *lk888Provider) setupRequestHeader(req *http.Request, apiKey string) { + req.Header.Set("Authorization", "Bearer "+apiKey) +} + +func (p *lk888Provider) mapModelForImages(model string, hasImages bool) string { + return strings.TrimSpace(model) +} + +func (p *lk888Provider) normalizeJSONRequest(bodyMap map[string]interface{}, originModel, upstreamModel string, imageCount int) { + normalizeLK888RequestMap(bodyMap, upstreamModel) +} + +func (p *lk888Provider) normalizeMultipartRequest(values map[string][]string, originModel, upstreamModel string, imageCount int) { + bodyMap := multipartValuesToMap(values) + normalizeLK888RequestMap(bodyMap, upstreamModel) + for k := range values { + delete(values, k) + } + for k, v := range bodyMap { + values[k] = []string{fmt.Sprintf("%v", v)} + } +} + +func lk888APIBase(baseURL string) string { + baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/") + if baseURL == "" { + return "https://api.lk888.ai/api" + } + return baseURL +} + +func isLK888ResultURL(resultURL string) bool { + lower := strings.ToLower(strings.TrimSpace(resultURL)) + return strings.Contains(lower, "lingkeai.vip") || + strings.Contains(lower, "lk888.ai") || + strings.Contains(lower, "lk666.ai") || + strings.Contains(lower, "storage-googleapis.com") +} + +func normalizeLK888RequestMap(bodyMap map[string]interface{}, upstreamModel string) { + modelName := strings.TrimSpace(fmtString(bodyMap["model"])) + if upstreamModel != "" { + modelName = strings.TrimSpace(upstreamModel) + bodyMap["model"] = modelName + } + + params := lk888ParamsMap(bodyMap["params"]) + for key, value := range bodyMap { + if lk888TopLevelRequestKey(key) { + continue + } + lk888AddParam(params, modelName, key, value) + } + + normalized := map[string]interface{}{ + "model": modelName, + "prompt": fmtString(bodyMap["prompt"]), + "params": params, + } + if count := intFromXBSoraValue(bodyMap["count"]); count > 0 { + normalized["count"] = count + } + for k := range bodyMap { + delete(bodyMap, k) + } + for k, v := range normalized { + bodyMap[k] = v + } +} + +func lk888ParamsMap(value interface{}) map[string]interface{} { + params := make(map[string]interface{}) + if value == nil { + return params + } + if existing, ok := value.(map[string]interface{}); ok { + for k, v := range existing { + params[k] = v + } + } + return params +} + +func lk888TopLevelRequestKey(key string) bool { + switch key { + case "model", "prompt", "params", "count": + return true + default: + return false + } +} + +func lk888AddParam(params map[string]interface{}, modelName, key string, value interface{}) { + if value == nil || lk888DropRequestKey(key) { + return + } + paramKey := lk888ParamKey(modelName, key) + if paramKey == "" { + return + } + if _, exists := params[paramKey]; exists { + return + } + params[paramKey] = lk888ParamValue(modelName, paramKey, key, value) +} + +func lk888DropRequestKey(key string) bool { + switch key { + case "n", "response_format", "user": + return true + default: + return false + } +} + +func lk888ParamKey(modelName, key string) string { + switch key { + case "seconds": + return "duration" + case "image", "input_reference", "image_url": + return "images" + case "orientation", "ratio", "size": + if strings.EqualFold(modelName, "sora-2") { + return "orientation" + } + return "aspect_ratio" + default: + return key + } +} + +func lk888ParamValue(modelName, paramKey, originalKey string, value interface{}) interface{} { + if paramKey == "duration" { + return lk888StringValue(value) + } + if paramKey == "aspect_ratio" { + return lk888AspectRatioValue(modelName, value) + } + if paramKey == "orientation" { + return lk888OrientationValue(value) + } + if paramKey == "images" { + if images := stringsFromXBSoraValue(value); len(images) > 0 { + if len(images) == 1 { + return images[0] + } + return images + } + } + return value +} + +func lk888StringValue(value interface{}) string { + switch v := value.(type) { + case string: + return strings.TrimSpace(v) + case float64: + if v == float64(int(v)) { + return strconv.Itoa(int(v)) + } + return strconv.FormatFloat(v, 'f', -1, 64) + case int: + return strconv.Itoa(v) + case int64: + return strconv.FormatInt(v, 10) + default: + return fmt.Sprintf("%v", v) + } +} + +func lk888AspectRatioValue(modelName string, value interface{}) string { + raw := strings.ToLower(strings.TrimSpace(lk888StringValue(value))) + switch raw { + case "portrait", "vertical", "9:16", "720x1280", "1024x1792": + if strings.EqualFold(modelName, "grok-video-3") { + return "2:3" + } + return "9:16" + case "landscape", "horizontal", "16:9", "1280x720", "1792x1024": + if strings.EqualFold(modelName, "grok-video-3") { + return "3:2" + } + return "16:9" + case "1:1": + return "1:1" + default: + if raw != "" { + return raw + } + return "16:9" + } +} + +func lk888OrientationValue(value interface{}) string { + raw := strings.ToLower(strings.TrimSpace(lk888StringValue(value))) + switch raw { + case "portrait", "vertical", "9:16", "720x1280", "1024x1792": + return "portrait" + default: + return "landscape" + } +} + +func lk888StringID(value any) string { + switch v := value.(type) { + case string: + return strings.TrimSpace(v) + case float64: + return strconv.FormatInt(int64(v), 10) + case int: + return strconv.Itoa(v) + case int64: + return strconv.FormatInt(v, 10) + default: + return "" + } +} + +func lk888TaskStatus(resp lk888StatusResponse) string { + state := strings.ToLower(strings.TrimSpace(resp.State)) + switch state { + case "success", "succeeded", "completed", "complete": + return model.TaskStatusSuccess + case "failed", "failure", "error": + return model.TaskStatusFailure + case "processing", "running": + return model.TaskStatusInProgress + case "pending", "queued": + return model.TaskStatusQueued + } + + switch strings.TrimSpace(resp.StatusGroup) { + case "已完成": + return model.TaskStatusSuccess + case "失败": + return model.TaskStatusFailure + case "处理中": + return model.TaskStatusInProgress + case "等待中": + return model.TaskStatusQueued + } + if resp.IsFinal { + if strings.TrimSpace(resp.ResultURL) != "" { + return model.TaskStatusSuccess + } + return model.TaskStatusFailure + } + return model.TaskStatusInProgress +} + +func lk888Progress(progress string) string { + progress = strings.TrimSpace(progress) + if progress == "" || progress == "100%" || progress == "100" || progress == "0" || progress == "0%" { + return "" + } + if strings.HasSuffix(progress, "%") { + return progress + } + return progress + "%" +} + +func lk888ErrorMessage(value any) string { + switch v := value.(type) { + case string: + return strings.TrimSpace(v) + case map[string]interface{}: + for _, key := range []string{"message", "details", "error"} { + if msg, ok := v[key].(string); ok && strings.TrimSpace(msg) != "" { + return strings.TrimSpace(msg) + } + } + default: + return "" + } + return "" +} diff --git a/relay/channel/task/openaivideo/manxiaobai.go b/relay/channel/task/openaivideo/manxiaobai.go new file mode 100644 index 000000000000..85621f81f840 --- /dev/null +++ b/relay/channel/task/openaivideo/manxiaobai.go @@ -0,0 +1,178 @@ +package openaivideo + +import ( + "fmt" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/model" + relaycommon "github.com/QuantumNous/new-api/relay/common" + "github.com/pkg/errors" +) + +// manxiaobaiProvider 适配漫小白（api.manxiaobai.online）。 +// 标准 OpenAI Video 协议：POST /v1/videos（JSON）、GET /v1/videos/{id}、 +// 下载 GET /v1/videos/{id}/content。注意 seconds 必须是字符串，数字会被 400 拒绝。 +// grok-imagine-video 支持文生/参考图；grok-imagine-video-1.5-preview 必须带参考图 +// （上游另有 /v1/video-reference-images 预上传接口，待账号充值后实测补充）。 +type manxiaobaiSubmitResponse struct { + ID string `json:"id"` + TaskID string `json:"task_id"` + Status string `json:"status"` +} + +type manxiaobaiQueryResponse struct { + ID string `json:"id"` + TaskID string `json:"task_id"` + Status string `json:"status"` + Progress int `json:"progress"` + VideoURL *string `json:"video_url"` + URL *string `json:"url"` + DownloadURL *string `json:"download_url"` + Error *struct { + Message string `json:"message"` + Code string `json:"code"` + } `json:"error,omitempty"` +} + +type manxiaobaiProvider struct{} + +func (p *manxiaobaiProvider) submitURL(baseURL string) string { + return fmt.Sprintf("%s/v1/videos", baseURL) +} + +func (p *manxiaobaiProvider) queryURL(baseURL, taskID string) string { + return fmt.Sprintf("%s/v1/videos/%s", baseURL, taskID) +} + +func (p *manxiaobaiProvider) parseSubmitResponse(body []byte) (string, error) { + var resp manxiaobaiSubmitResponse + if err := common.Unmarshal(body, &resp); err != nil { + return "", errors.Wrap(err, "unmarshal manxiaobai submit response failed") + } + id := resp.ID + if id == "" { + id = resp.TaskID + } + if id == "" { + return "", fmt.Errorf("manxiaobai response id/task_id is empty") + } + return id, nil +} + +func (p *manxiaobaiProvider) parseQueryResponse(body []byte) (*relaycommon.TaskInfo, error) { + var resp manxiaobaiQueryResponse + if err := common.Unmarshal(body, &resp); err != nil { + return nil, errors.Wrap(err, "unmarshal manxiaobai query response failed") + } + + ti := &relaycommon.TaskInfo{Code: 0} + ti.Status = statusToTaskStatus(resp.Status) + + if ti.Status == model.TaskStatusSuccess { + for _, candidate := range []*string{resp.VideoURL, resp.URL, resp.DownloadURL} { + if candidate != nil && *candidate != "" { + ti.Url = *candidate + break + } + } + } + if ti.Status == model.TaskStatusFailure { + if resp.Error != nil { + ti.Reason = resp.Error.Message + } else { + ti.Reason = "task failed" + } + } + if resp.Progress > 0 && resp.Progress < 100 { + ti.Progress = fmt.Sprintf("%d%%", resp.Progress) + } + return ti, nil +} + +func (p *manxiaobaiProvider) buildSubmitResponseBody(info *relaycommon.RelayInfo, upstreamTaskID string) any { + return map[string]any{ + "id": info.PublicTaskID, + "task_id": info.PublicTaskID, + "object": "video", + "model": info.OriginModelName, + "status": "queued", + "progress": 0, + "created_at": 0, + } +} + +// 上游 /v1/videos 的 JSON 解析存在缺陷（model 字段丢失报 Field required）， +// 文档推荐且实测可用的是 multipart 表单，统一转表单提交。 +func (p *manxiaobaiProvider) needsMultipart() bool { return true } + +func (p *manxiaobaiProvider) mapModelForImages(model string, hasImages bool) string { + return model +} + +// normalizeJSONRequest 收敛漫小白的字段要求： +// - seconds 必须是字符串（数字会被上游 400 拒绝），duration 互补； +// - 默认补横屏尺寸（上游仅支持 1792x1024 / 1024x1792）。 +func (p *manxiaobaiProvider) normalizeJSONRequest(bodyMap map[string]interface{}, originModel, upstreamModel string, imageCount int) { + seconds := "" + switch v := bodyMap["seconds"].(type) { + case string: + seconds = v + case float64: + seconds = fmt.Sprintf("%d", int(v)) + } + if seconds == "" { + if d, ok := bodyMap["duration"].(float64); ok && d > 0 { + seconds = fmt.Sprintf("%d", int(d)) + } + } + if seconds == "" { + seconds = "10" + } + bodyMap["seconds"] = seconds + delete(bodyMap, "duration") + + if size, _ := bodyMap["size"].(string); size != "1792x1024" && size != "1024x1792" { + if fmtString(bodyMap["aspect_ratio"]) == "9:16" || isPortraitSize(size) { + bodyMap["size"] = "1024x1792" + } else { + bodyMap["size"] = "1792x1024" + } + } + delete(bodyMap, "aspect_ratio") +} + +// normalizeMultipartRequest 表单路径：seconds 本身是字符串，补默认值并归一尺寸。 +func (p *manxiaobaiProvider) normalizeMultipartRequest(values map[string][]string, originModel, upstreamModel string, imageCount int) { + getFirst := func(key string) string { + if v := values[key]; len(v) > 0 { + return v[0] + } + return "" + } + if getFirst("seconds") == "" { + if d := getFirst("duration"); d != "" { + values["seconds"] = []string{d} + } else { + values["seconds"] = []string{"10"} + } + } + delete(values, "duration") + + if size := getFirst("size"); size != "1792x1024" && size != "1024x1792" { + if getFirst("aspect_ratio") == "9:16" || isPortraitSize(size) { + values["size"] = []string{"1024x1792"} + } else { + values["size"] = []string{"1792x1024"} + } + } + delete(values, "aspect_ratio") +} + +// isPortraitSize 判断 WxH 尺寸是否竖屏。 +func isPortraitSize(size string) bool { + var w, h int + if _, err := fmt.Sscanf(size, "%dx%d", &w, &h); err != nil { + return false + } + return h > w +} diff --git a/relay/channel/task/openaivideo/newapi.go b/relay/channel/task/openaivideo/newapi.go new file mode 100644 index 000000000000..a49bd8d6a2e4 --- /dev/null +++ b/relay/channel/task/openaivideo/newapi.go @@ -0,0 +1,96 @@ +package openaivideo + +import ( + "fmt" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/model" + relaycommon "github.com/QuantumNous/new-api/relay/common" + "github.com/pkg/errors" +) + +type newapiSubmitResponse struct { + ID string `json:"id"` + TaskID string `json:"task_id"` + Object string `json:"object"` + Model string `json:"model"` + Status string `json:"status"` + Progress int `json:"progress"` + CreatedAt int64 `json:"created_at"` +} + +type newapiQueryResponse struct { + ID string `json:"id"` + TaskID string `json:"task_id"` + Status string `json:"status"` + Progress int `json:"progress"` + Error *struct { + Message string `json:"message"` + Code string `json:"code"` + } `json:"error,omitempty"` +} + +type newapiProvider struct{} + +func (p *newapiProvider) submitURL(baseURL string) string { + return fmt.Sprintf("%s/v1/video/generations", baseURL) +} + +func (p *newapiProvider) queryURL(baseURL, taskID string) string { + return fmt.Sprintf("%s/v1/video/generations/%s", baseURL, taskID) +} + +func (p *newapiProvider) parseSubmitResponse(body []byte) (string, error) { + var resp newapiSubmitResponse + if err := common.Unmarshal(body, &resp); err != nil { + return "", errors.Wrap(err, "unmarshal newapi submit response failed") + } + id := resp.TaskID + if id == "" { + id = resp.ID + } + if id == "" { + return "", fmt.Errorf("newapi response task_id/id is empty") + } + return id, nil +} + +func (p *newapiProvider) parseQueryResponse(body []byte) (*relaycommon.TaskInfo, error) { + var resp newapiQueryResponse + if err := common.Unmarshal(body, &resp); err != nil { + return nil, errors.Wrap(err, "unmarshal newapi query response failed") + } + + ti := &relaycommon.TaskInfo{Code: 0} + ti.Status = statusToTaskStatus(resp.Status) + + if ti.Status == model.TaskStatusFailure { + if resp.Error != nil { + ti.Reason = resp.Error.Message + } else { + ti.Reason = "task failed" + } + } + if resp.Progress > 0 && resp.Progress < 100 { + ti.Progress = fmt.Sprintf("%d%%", resp.Progress) + } + return ti, nil +} + +func (p *newapiProvider) buildSubmitResponseBody(info *relaycommon.RelayInfo, upstreamTaskID string) any { + return map[string]any{ + "id": info.PublicTaskID, + "task_id": info.PublicTaskID, + "object": "video", + "model": info.OriginModelName, + "status": "queued", + "progress": 0, + "created_at": 0, + } +} + +func (p *newapiProvider) needsMultipart() bool { return true } + +func (p *newapiProvider) mapModelForImages(model string, hasImages bool) string { + return model +} diff --git a/relay/channel/task/openaivideo/provider.go b/relay/channel/task/openaivideo/provider.go new file mode 100644 index 000000000000..bdcb9991da91 --- /dev/null +++ b/relay/channel/task/openaivideo/provider.go @@ -0,0 +1,212 @@ +package openaivideo + +import ( + "net/http" + "strings" + + "github.com/QuantumNous/new-api/model" + relaycommon "github.com/QuantumNous/new-api/relay/common" +) + +type provider interface { + submitURL(baseURL string) string + queryURL(baseURL, taskID string) string + parseSubmitResponse(body []byte) (upstreamTaskID string, err error) + parseQueryResponse(body []byte) (*relaycommon.TaskInfo, error) + buildSubmitResponseBody(info *relaycommon.RelayInfo, upstreamTaskID string) any + needsMultipart() bool + mapModelForImages(model string, hasImages bool) string +} + +type requestHeaderSetter interface { + setupRequestHeader(req *http.Request, apiKey string) +} + +type jsonBodyProvider interface { + forceJSONBody() bool +} + +type requestNormalizer interface { + normalizeJSONRequest(bodyMap map[string]interface{}, originModel, upstreamModel string, imageCount int) + normalizeMultipartRequest(values map[string][]string, originModel, upstreamModel string, imageCount int) +} + +func getProvider(channelOther string) provider { + if prov, ok := getProviderByHint(channelOther); ok { + return prov + } + return &bltcyProvider{} +} + +func getProviderByHint(channelOther string) (provider, bool) { + switch { + case containsAny(channelOther, "runway"): + return &runwayProvider{}, true + case containsAny(channelOther, "lk888", "lk666", "jiushi", "ai聚合站"): + return &lk888Provider{}, true + case containsAny(channelOther, "xb-sora2", "xbsora2", "xb-sora", "xbsora", "hongniao"): + return &xbSoraProvider{}, true + case containsAny(channelOther, "xgapi", "xingguang"): + return &xgapiProvider{}, true + case containsAny(channelOther, "937qq", "qilin"): + return &qilinProvider{}, true + case containsAny(channelOther, "apexer"): + return &apexerapiProvider{}, true + case containsAny(channelOther, "manxiaobai", "漫小白", "mxb"): + return &manxiaobaiProvider{}, true + case containsAny(channelOther, "newapi"): + return &newapiProvider{}, true + default: + return nil, false + } +} + +func getProviderByBaseURL(baseURL string) provider { + switch { + case isRunwayBaseURL(baseURL): + return &runwayProvider{} + case isLK888BaseURL(baseURL): + return &lk888Provider{} + case containsAny(baseURL, "xgapi"): + return &xgapiProvider{} + case containsAny(baseURL, "937qq", "qilin"): + return &qilinProvider{} + case containsAny(baseURL, "apexer"): + return &apexerapiProvider{} + case containsAny(baseURL, "manxiaobai"): + return &manxiaobaiProvider{} + case containsAny(baseURL, "newapi"): + return &newapiProvider{} + case isXBSoraBaseURL(baseURL): + return &xbSoraProvider{} + default: + return &bltcyProvider{} + } +} + +func getProviderForRelayInfo(info *relaycommon.RelayInfo) provider { + if info == nil { + return &bltcyProvider{} + } + baseURL := info.ChannelBaseUrl + if prov, ok := getProviderByHint(info.ChannelOther); ok { + return prov + } + switch { + case isRunwayBaseURL(baseURL): + return &runwayProvider{} + case isLK888BaseURL(baseURL): + return &lk888Provider{} + case containsAny(baseURL, "xgapi"): + return &xgapiProvider{} + case containsAny(baseURL, "937qq", "qilin"): + return &qilinProvider{} + case containsAny(baseURL, "apexer"): + return &apexerapiProvider{} + case containsAny(baseURL, "manxiaobai"): + return &manxiaobaiProvider{} + case containsAny(baseURL, "newapi"): + return &newapiProvider{} + case isXBSoraBaseURL(baseURL), isXBSoraModelName(info.OriginModelName), isXBSoraModelName(info.UpstreamModelName): + return &xbSoraProvider{} + default: + return &bltcyProvider{} + } +} + +func getProviderForTaskFetch(baseURL string, body map[string]any) provider { + if body != nil { + for _, key := range []string{"provider", "channel_other", "other"} { + if prov, ok := getProviderByHint(fmtString(body[key])); ok { + return prov + } + } + } + switch { + case isRunwayBaseURL(baseURL): + return &runwayProvider{} + case isLK888BaseURL(baseURL): + return &lk888Provider{} + case containsAny(baseURL, "xgapi"): + return &xgapiProvider{} + case containsAny(baseURL, "937qq", "qilin"): + return &qilinProvider{} + case containsAny(baseURL, "apexer"): + return &apexerapiProvider{} + case containsAny(baseURL, "manxiaobai"): + return &manxiaobaiProvider{} + case containsAny(baseURL, "newapi"): + return &newapiProvider{} + case isXBSoraBaseURL(baseURL): + return &xbSoraProvider{} + } + if body != nil { + for _, key := range []string{"origin_model_name", "upstream_model_name", "model"} { + if isXBSoraModelName(fmtString(body[key])) { + return &xbSoraProvider{} + } + } + } + return &bltcyProvider{} +} + +func isRunwayBaseURL(baseURL string) bool { + baseURL = strings.ToLower(strings.TrimRight(strings.TrimSpace(baseURL), "/")) + return strings.Contains(baseURL, "runway") || + strings.Contains(baseURL, "127.0.0.1:8787") || + strings.Contains(baseURL, "localhost:8787") +} + +func isLK888BaseURL(baseURL string) bool { + baseURL = strings.ToLower(strings.TrimRight(strings.TrimSpace(baseURL), "/")) + return strings.Contains(baseURL, "api.lk888.ai") || + strings.Contains(baseURL, "jiushi.lk666.ai") +} + +func isXBSoraBaseURL(baseURL string) bool { + baseURL = strings.ToLower(strings.TrimRight(strings.TrimSpace(baseURL), "/")) + return containsAny(baseURL, "xb-sora2", "xbsora2", "xb-sora", "xbsora") || + strings.HasSuffix(baseURL, "/api/v1") || + strings.HasSuffix(baseURL, "/v1") +} + +func fmtString(value any) string { + if s, ok := value.(string); ok { + return s + } + return "" +} + +func isXBSoraModelName(modelName string) bool { + switch strings.TrimSpace(modelName) { + case "xb-sora2", "xb-sora-2", "sora-2", "sora-2-pro", "openai-sora-2", "sora-2-pro-text-to-video", "sora-2-image-to-video": + return true + default: + return false + } +} + +func containsAny(s string, keywords ...string) bool { + s = strings.ToLower(s) + for _, k := range keywords { + if strings.Contains(s, strings.ToLower(k)) { + return true + } + } + return false +} + +func statusToTaskStatus(status string) string { + switch status { + case "NOT_START", "queued", "pending", "submitted": + return model.TaskStatusQueued + case "processing", "in_progress", "running", "RUNNING", "IN_PROGRESS": + return model.TaskStatusInProgress + case "SUCCESS", "SUCCEEDED", "completed", "succeed", "succeeded", "success": + return model.TaskStatusSuccess + case "FAILED", "failed", "cancelled", "canceled": + return model.TaskStatusFailure + default: + return "" + } +} diff --git a/relay/channel/task/openaivideo/qilin.go b/relay/channel/task/openaivideo/qilin.go new file mode 100644 index 000000000000..cc60c950bc5a --- /dev/null +++ b/relay/channel/task/openaivideo/qilin.go @@ -0,0 +1,351 @@ +package openaivideo + +import ( + "fmt" + "strconv" + "strings" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/model" + relaycommon "github.com/QuantumNous/new-api/relay/common" + "github.com/pkg/errors" +) + +type qilinProvider struct{} + +type qilinSubmitResponse struct { + ID string `json:"id"` + TaskID string `json:"task_id"` + Object string `json:"object"` + Model string `json:"model"` + Status string `json:"status"` + Progress int `json:"progress"` + CreatedAt int64 `json:"created_at"` +} + +type qilinQueryResponse struct { + ID string `json:"id"` + TaskID string `json:"task_id"` + Status string `json:"status"` + Progress int `json:"progress"` + URL *string `json:"url"` + VideoURL *string `json:"video_url"` + Output *struct { + URL string `json:"url"` + } `json:"output,omitempty"` + CompletedAt int64 `json:"completed_at"` + Error *struct { + Message string `json:"message"` + Code string `json:"code"` + } `json:"error,omitempty"` +} + +const qilinGrokBaseVideoModel = "grok-imagine-1.0-video" + +var qilinGrokLongModelByDuration = map[int]string{ + 20: "grok-imagine-1.0-video-20s", + 30: "grok-imagine-1.0-video-30s", +} + +var qilinGrokLockedDurationByModel = map[string]int{ + "grok-imagine-1.0-video-20s": 20, + "grok-imagine-1.0-video-30s": 30, +} + +func (p *qilinProvider) submitURL(baseURL string) string { + return fmt.Sprintf("%s/v1/videos", baseURL) +} + +func (p *qilinProvider) queryURL(baseURL, taskID string) string { + return fmt.Sprintf("%s/v1/videos/%s", baseURL, taskID) +} + +func (p *qilinProvider) parseSubmitResponse(body []byte) (string, error) { + var resp qilinSubmitResponse + if err := common.Unmarshal(body, &resp); err != nil { + return "", errors.Wrap(err, "unmarshal qilin submit response failed") + } + id := resp.ID + if id == "" { + id = resp.TaskID + } + if id == "" { + return "", fmt.Errorf("qilin submit response id/task_id is empty") + } + return id, nil +} + +func (p *qilinProvider) parseQueryResponse(body []byte) (*relaycommon.TaskInfo, error) { + var resp qilinQueryResponse + if err := common.Unmarshal(body, &resp); err != nil { + return nil, errors.Wrap(err, "unmarshal qilin query response failed") + } + + ti := &relaycommon.TaskInfo{Code: 0} + ti.Status = statusToTaskStatus(resp.Status) + if ti.Status == model.TaskStatusSuccess { + if resp.VideoURL != nil && *resp.VideoURL != "" { + ti.Url = *resp.VideoURL + } else if resp.Output != nil && resp.Output.URL != "" { + ti.Url = resp.Output.URL + } else if resp.URL != nil && *resp.URL != "" { + ti.Url = *resp.URL + } + } + if ti.Status == model.TaskStatusFailure { + if resp.Error != nil { + ti.Reason = resp.Error.Message + } else { + ti.Reason = "task failed" + } + } + if resp.Progress > 0 && resp.Progress < 100 { + ti.Progress = fmt.Sprintf("%d%%", resp.Progress) + } + return ti, nil +} + +func (p *qilinProvider) buildSubmitResponseBody(info *relaycommon.RelayInfo, upstreamTaskID string) any { + return map[string]any{ + "id": info.PublicTaskID, + "task_id": info.PublicTaskID, + "object": "video", + "model": info.OriginModelName, + "status": "queued", + "progress": 0, + "created_at": 0, + } +} + +func (p *qilinProvider) needsMultipart() bool { return false } + +func (p *qilinProvider) mapModelForImages(model string, hasImages bool) string { + return model +} + +func (p *qilinProvider) normalizeJSONRequest(bodyMap map[string]interface{}, originModel, upstreamModel string, imageCount int) { + normalizeQilinDuration(bodyMap) + normalizeQilinSeconds(bodyMap) + normalizeQilinGrokTransportModel(bodyMap) + if _, ok := bodyMap["resolution"]; !ok { + bodyMap["resolution"] = "720p" + } + if _, ok := bodyMap["quality"]; !ok { + bodyMap["quality"] = qilinQualityForResolution(bodyMap["resolution"]) + } + normalizeQilinImageReference(bodyMap) + if _, ok := bodyMap["size"]; !ok { + ratio, _ := bodyMap["aspect_ratio"].(string) + if ratio == "" { + ratio, _ = bodyMap["ratio"].(string) + } + if size := qilinSizeForAspectRatio(ratio); size != "" { + bodyMap["size"] = size + } + } +} + +func (p *qilinProvider) normalizeMultipartRequest(values map[string][]string, originModel, upstreamModel string, imageCount int) { + if len(values["duration"]) == 0 { + if seconds := firstValue(values["seconds"]); seconds != "" { + values["duration"] = []string{seconds} + } + } + if len(values["seconds"]) == 0 { + if duration := firstValue(values["duration"]); duration != "" { + values["seconds"] = []string{duration} + } + } + normalizeQilinGrokTransportModelForValues(values) + if len(values["resolution"]) == 0 { + values["resolution"] = []string{"720p"} + } + if len(values["quality"]) == 0 { + values["quality"] = []string{qilinQualityForResolution(firstValue(values["resolution"]))} + } + if len(values["size"]) > 0 { + return + } + ratio := firstValue(values["aspect_ratio"]) + if ratio == "" { + ratio = firstValue(values["ratio"]) + } + if size := qilinSizeForAspectRatio(ratio); size != "" { + values["size"] = []string{size} + } +} + +func normalizeQilinImageReference(bodyMap map[string]interface{}) { + if _, ok := bodyMap["image_reference"]; ok { + return + } + urls := collectQilinImageReferenceURLs(bodyMap) + if len(urls) == 0 { + return + } + refs := make([]map[string]interface{}, 0, len(urls)) + for _, url := range urls { + refs = append(refs, map[string]interface{}{ + "type": "image_url", + "image_url": map[string]interface{}{ + "url": url, + }, + }) + } + bodyMap["image_reference"] = refs +} + +func collectQilinImageReferenceURLs(bodyMap map[string]interface{}) []string { + keys := []string{"images", "image", "image_urls", "reference_images", "reference_image_urls", "image_url", "file_paths"} + urls := make([]string, 0) + seen := make(map[string]struct{}) + add := func(url string) { + url = strings.TrimSpace(url) + if url == "" { + return + } + if _, ok := seen[url]; ok { + return + } + seen[url] = struct{}{} + urls = append(urls, url) + } + for _, key := range keys { + value, ok := bodyMap[key] + if !ok { + continue + } + switch v := value.(type) { + case string: + add(v) + case []string: + for _, item := range v { + add(item) + } + case []interface{}: + for _, item := range v { + if s, ok := item.(string); ok { + add(s) + } + } + } + } + return urls +} + +func normalizeQilinDuration(bodyMap map[string]interface{}) { + if _, ok := bodyMap["duration"]; ok { + return + } + v, ok := bodyMap["seconds"] + if !ok { + return + } + if duration := intFromQilinValue(v); duration > 0 { + bodyMap["duration"] = duration + } +} + +func normalizeQilinSeconds(bodyMap map[string]interface{}) { + if _, ok := bodyMap["seconds"]; ok { + return + } + if duration := intFromQilinValue(bodyMap["duration"]); duration > 0 { + bodyMap["seconds"] = strconv.Itoa(duration) + } +} + +func normalizeQilinGrokTransportModel(bodyMap map[string]interface{}) { + modelName, _ := bodyMap["model"].(string) + duration := intFromQilinValue(bodyMap["duration"]) + transportModel, normalizedDuration := qilinGrokTransportModelAndDuration(modelName, duration) + if transportModel != "" { + bodyMap["model"] = transportModel + } + if normalizedDuration > 0 { + bodyMap["duration"] = normalizedDuration + bodyMap["seconds"] = strconv.Itoa(normalizedDuration) + } +} + +func normalizeQilinGrokTransportModelForValues(values map[string][]string) { + modelName := firstValue(values["model"]) + duration := intFromQilinValue(firstValue(values["duration"])) + transportModel, normalizedDuration := qilinGrokTransportModelAndDuration(modelName, duration) + if transportModel != "" { + values["model"] = []string{transportModel} + } + if normalizedDuration > 0 { + v := strconv.Itoa(normalizedDuration) + values["duration"] = []string{v} + values["seconds"] = []string{v} + } +} + +func qilinGrokTransportModelAndDuration(modelName string, duration int) (string, int) { + modelName = strings.TrimSpace(modelName) + if lockedDuration, ok := qilinGrokLockedDurationByModel[modelName]; ok { + return modelName, lockedDuration + } + if modelName == qilinGrokBaseVideoModel { + if transportModel, ok := qilinGrokLongModelByDuration[duration]; ok { + return transportModel, duration + } + } + return "", 0 +} + +func intFromQilinValue(value interface{}) int { + switch v := value.(type) { + case string: + raw := strings.TrimSuffix(strings.TrimSpace(strings.ToLower(v)), "s") + n, err := strconv.ParseFloat(raw, 64) + if err != nil { + return 0 + } + return int(n) + case float64: + return int(v) + case int: + return v + case int64: + return int(v) + } + return 0 +} + +func qilinQualityForResolution(resolution interface{}) string { + raw := strings.TrimSpace(strings.ToLower(fmt.Sprintf("%v", resolution))) + switch raw { + case "720p", "1080p", "hd", "high", "高清": + return "high" + default: + return "standard" + } +} + +func firstValue(values []string) string { + if len(values) == 0 { + return "" + } + return values[0] +} + +func qilinSizeForAspectRatio(aspectRatio string) string { + switch strings.TrimSpace(aspectRatio) { + case "1:1": + return "1024x1024" + case "9:16": + return "720x1280" + case "16:9": + return "1280x720" + case "4:3": + return "1152x864" + case "3:4": + return "864x1152" + case "21:9": + return "1680x720" + default: + return "" + } +} diff --git a/relay/channel/task/openaivideo/qilin_test.go b/relay/channel/task/openaivideo/qilin_test.go new file mode 100644 index 000000000000..8ef98ac46b5c --- /dev/null +++ b/relay/channel/task/openaivideo/qilin_test.go @@ -0,0 +1,79 @@ +package openaivideo + +import ( + "testing" + + "github.com/QuantumNous/new-api/model" +) + +func TestQilinGrokLongDurationTransportModel(t *testing.T) { + p := &qilinProvider{} + body := map[string]interface{}{ + "model": "grok-imagine-1.0-video", + "prompt": "test", + "duration": 20, + } + + p.normalizeJSONRequest(body, "grok-imagine-1.0-video", "grok-imagine-1.0-video", 0) + + if got := body["model"]; got != "grok-imagine-1.0-video-20s" { + t.Fatalf("model = %#v", got) + } + if got := body["duration"]; got != 20 { + t.Fatalf("duration = %#v", got) + } + if got := body["seconds"]; got != "20" { + t.Fatalf("seconds = %#v", got) + } +} + +func TestQilinGrokLockedDurationModel(t *testing.T) { + p := &qilinProvider{} + body := map[string]interface{}{ + "model": "grok-imagine-1.0-video-30s", + "prompt": "test", + "duration": 10, + "seconds": "10", + } + + p.normalizeJSONRequest(body, "grok-imagine-1.0-video-30s", "grok-imagine-1.0-video-30s", 0) + + if got := body["model"]; got != "grok-imagine-1.0-video-30s" { + t.Fatalf("model = %#v", got) + } + if got := body["duration"]; got != 30 { + t.Fatalf("duration = %#v", got) + } + if got := body["seconds"]; got != "30" { + t.Fatalf("seconds = %#v", got) + } +} + +func TestQilinGrokAdditionalAspectRatios(t *testing.T) { + p := &qilinProvider{} + body := map[string]interface{}{ + "model": "grok-imagine-1.0-video", + "prompt": "test", + "aspect_ratio": "21:9", + } + + p.normalizeJSONRequest(body, "grok-imagine-1.0-video", "grok-imagine-1.0-video", 0) + + if got := body["size"]; got != "1680x720" { + t.Fatalf("size = %#v", got) + } +} + +func TestQilinParseQueryPrefersOutputURLBeforeProxyURL(t *testing.T) { + p := &qilinProvider{} + info, err := p.parseQueryResponse([]byte(`{"id":"task_1","status":"completed","progress":100,"url":"https://proxy.example.com/v1/videos/task_1/content","output":{"url":"https://cdn.example.com/video.mp4"}}`)) + if err != nil { + t.Fatalf("parseQueryResponse error: %v", err) + } + if info.Status != model.TaskStatusSuccess { + t.Fatalf("status = %q", info.Status) + } + if info.Url != "https://cdn.example.com/video.mp4" { + t.Fatalf("url = %q", info.Url) + } +} diff --git a/relay/channel/task/openaivideo/runway.go b/relay/channel/task/openaivideo/runway.go new file mode 100644 index 000000000000..d65aa2ef2c68 --- /dev/null +++ b/relay/channel/task/openaivideo/runway.go @@ -0,0 +1,264 @@ +package openaivideo + +import ( + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/model" + relaycommon "github.com/QuantumNous/new-api/relay/common" + "github.com/pkg/errors" +) + +type runwayProvider struct{} + +type runwaySubmitResponse struct { + JobID string `json:"jobId"` + Status string `json:"status"` + Error any `json:"error,omitempty"` +} + +type runwayQueryResponse struct { + JobID string `json:"jobId"` + Kind string `json:"kind"` + Status string `json:"status"` + TaskID string `json:"taskId,omitempty"` + Result *struct { + TaskID string `json:"taskId"` + Status string `json:"status"` + Files []string `json:"files"` + FileURLs []string `json:"fileUrls"` + } `json:"result,omitempty"` + Error any `json:"error,omitempty"` +} + +func (p *runwayProvider) submitURL(baseURL string) string { + return strings.TrimRight(baseURL, "/") + "/video" +} + +func (p *runwayProvider) queryURL(baseURL, taskID string) string { + return strings.TrimRight(baseURL, "/") + "/jobs/" + taskID +} + +func (p *runwayProvider) parseSubmitResponse(body []byte) (string, error) { + var resp runwaySubmitResponse + if err := common.Unmarshal(body, &resp); err != nil { + return "", errors.Wrap(err, "unmarshal runway submit response failed") + } + if resp.JobID == "" { + return "", fmt.Errorf("runway response jobId is empty: %s", runwayErrorMessage(resp.Error)) + } + return resp.JobID, nil +} + +func (p *runwayProvider) parseQueryResponse(body []byte) (*relaycommon.TaskInfo, error) { + var resp runwayQueryResponse + if err := common.Unmarshal(body, &resp); err != nil { + return nil, errors.Wrap(err, "unmarshal runway query response failed") + } + + ti := &relaycommon.TaskInfo{ + Code: 0, + TaskID: firstNonEmpty(resp.TaskID, resp.JobID), + Status: statusToTaskStatus(resp.Status), + } + + if ti.Status == model.TaskStatusSuccess && resp.Result != nil { + if len(resp.Result.FileURLs) > 0 && strings.TrimSpace(resp.Result.FileURLs[0]) != "" { + ti.Url = "runway:" + strings.TrimSpace(resp.Result.FileURLs[0]) + } + } + if ti.Status == model.TaskStatusFailure { + ti.Reason = runwayErrorMessage(resp.Error) + if ti.Reason == "" { + ti.Reason = "task failed" + } + } + return ti, nil +} + +func (p *runwayProvider) buildSubmitResponseBody(info *relaycommon.RelayInfo, upstreamTaskID string) any { + return map[string]any{ + "id": info.PublicTaskID, + "task_id": info.PublicTaskID, + "object": "video", + "model": info.OriginModelName, + "status": "queued", + "progress": 0, + "created_at": 0, + } +} + +func (p *runwayProvider) needsMultipart() bool { return false } + +func (p *runwayProvider) forceJSONBody() bool { return true } + +func (p *runwayProvider) setupRequestHeader(req *http.Request, apiKey string) { + req.Header.Set("X-API-Key", apiKey) +} + +func (p *runwayProvider) mapModelForImages(model string, hasImages bool) string { + model = strings.TrimSpace(model) + if model == "" { + return "seedance-2" + } + return model +} + +func (p *runwayProvider) normalizeJSONRequest(bodyMap map[string]interface{}, originModel, upstreamModel string, imageCount int) { + normalizeRunwayRequestMap(bodyMap) +} + +func (p *runwayProvider) normalizeMultipartRequest(values map[string][]string, originModel, upstreamModel string, imageCount int) { + bodyMap := multipartValuesToMap(values) + normalizeRunwayRequestMap(bodyMap) + for k := range values { + delete(values, k) + } + for k, v := range bodyMap { + values[k] = []string{fmt.Sprintf("%v", v)} + } +} + +func normalizeRunwayRequestMap(bodyMap map[string]interface{}) { + if _, ok := bodyMap["async"]; !ok { + bodyMap["async"] = true + } + if _, ok := bodyMap["exploreMode"]; !ok { + bodyMap["exploreMode"] = true + } + if _, ok := bodyMap["aspectRatio"]; !ok { + if ratio := firstStringValue(bodyMap, "aspect_ratio", "ratio"); ratio != "" { + bodyMap["aspectRatio"] = ratio + } + } + if _, ok := bodyMap["duration"]; !ok { + if duration := intFromRunwayValue(bodyMap["seconds"]); duration > 0 { + bodyMap["duration"] = duration + } + } + if _, ok := bodyMap["imageURL"]; !ok { + if imageURL := firstRunwayImageURL(bodyMap); imageURL != "" { + bodyMap["imageURL"] = imageURL + } + } + if _, ok := bodyMap["imageAssetID"]; !ok { + if assetID := firstStringValue(bodyMap, "image_asset_id", "imageAssetId", "asset_id", "assetId"); assetID != "" { + bodyMap["imageAssetID"] = assetID + } + } + if _, ok := bodyMap["videoURL"]; !ok { + if videoURL := firstStringValue(bodyMap, "video_url", "source_video_url"); videoURL != "" { + bodyMap["videoURL"] = videoURL + } + } + if _, ok := bodyMap["videoAssetID"]; !ok { + if assetID := firstStringValue(bodyMap, "video_asset_id", "videoAssetId"); assetID != "" { + bodyMap["videoAssetID"] = assetID + } + } + + for _, key := range []string{ + "size", + "seconds", + "aspect_ratio", + "ratio", + "image", + "images", + "image_url", + "image_urls", + "reference_images", + "reference_image_urls", + "input_reference", + "video_url", + "video_asset_id", + "source_video_url", + } { + delete(bodyMap, key) + } +} + +func firstRunwayImageURL(bodyMap map[string]interface{}) string { + for _, key := range []string{"imageURL", "image_url", "image", "input_reference", "images", "image_urls", "reference_images", "reference_image_urls"} { + if imageURL := firstURLFromRunwayValue(bodyMap[key]); imageURL != "" { + return imageURL + } + } + return "" +} + +func firstURLFromRunwayValue(value interface{}) string { + switch v := value.(type) { + case string: + return strings.TrimSpace(v) + case []string: + for _, item := range v { + if s := strings.TrimSpace(item); s != "" { + return s + } + } + case []interface{}: + for _, item := range v { + if s := firstURLFromRunwayValue(item); s != "" { + return s + } + } + case map[string]interface{}: + for _, key := range []string{"url", "imageURL", "image_url"} { + if s := firstURLFromRunwayValue(v[key]); s != "" { + return s + } + } + if imageURL, ok := v["image_url"].(map[string]interface{}); ok { + return firstURLFromRunwayValue(imageURL["url"]) + } + } + return "" +} + +func firstStringValue(bodyMap map[string]interface{}, keys ...string) string { + for _, key := range keys { + if s, ok := bodyMap[key].(string); ok && strings.TrimSpace(s) != "" { + return strings.TrimSpace(s) + } + } + return "" +} + +func intFromRunwayValue(value interface{}) int { + switch v := value.(type) { + case int: + return v + case int64: + return int(v) + case float64: + return int(v) + case string: + n, _ := strconv.Atoi(strings.TrimSpace(v)) + return n + default: + return 0 + } +} + +func runwayErrorMessage(value any) string { + switch v := value.(type) { + case nil: + return "" + case string: + return strings.TrimSpace(v) + case map[string]any: + for _, key := range []string{"message", "error", "reason"} { + if s, ok := v[key].(string); ok && strings.TrimSpace(s) != "" { + return strings.TrimSpace(s) + } + } + default: + if b, err := common.Marshal(v); err == nil { + return string(b) + } + } + return "" +} diff --git a/relay/channel/task/openaivideo/xb_sora.go b/relay/channel/task/openaivideo/xb_sora.go new file mode 100644 index 000000000000..dc4319fb7734 --- /dev/null +++ b/relay/channel/task/openaivideo/xb_sora.go @@ -0,0 +1,505 @@ +package openaivideo + +import ( + "encoding/json" + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/model" + relaycommon "github.com/QuantumNous/new-api/relay/common" + "github.com/pkg/errors" +) + +type xbSoraProvider struct{} + +type xbSoraSubmitResponse struct { + Code any `json:"code"` + Message string `json:"message"` + Msg string `json:"msg,omitempty"` + Data struct { + TaskID string `json:"task_id"` + Status string `json:"status"` + Model string `json:"model"` + } `json:"data"` + Error *xbSoraError `json:"error,omitempty"` +} + +type xbSoraQueryResponse struct { + Code any `json:"code"` + Message string `json:"message"` + Msg string `json:"msg,omitempty"` + Data struct { + TaskID string `json:"task_id"` + Status string `json:"status"` + Progress int `json:"progress"` + Model string `json:"model"` + Result *struct { + VideoURL string `json:"video_url"` + ResultURLs []string `json:"resultUrls"` + ThumbnailURL string `json:"thumbnail_url"` + Duration float64 `json:"duration"` + Format string `json:"format"` + ExpiresAt int64 `json:"expires_at"` + } `json:"result,omitempty"` + Error *xbSoraError `json:"error,omitempty"` + } `json:"data"` + Error *xbSoraError `json:"error,omitempty"` +} + +type xbSoraError struct { + Type string `json:"type,omitempty"` + Code string `json:"code,omitempty"` + Message string `json:"message,omitempty"` + Details string `json:"details,omitempty"` +} + +type xbSoraResponseEnvelope struct { + Code any `json:"code"` + Message string `json:"message"` + Msg string `json:"msg"` + Data json.RawMessage `json:"data"` +} + +func (p *xbSoraProvider) submitURL(baseURL string) string { + return fmt.Sprintf("%s/videos/generate", xbSoraAPIBase(baseURL)) +} + +func (p *xbSoraProvider) queryURL(baseURL, taskID string) string { + return fmt.Sprintf("%s/videos/%s", xbSoraAPIBase(baseURL), taskID) +} + +func (p *xbSoraProvider) parseSubmitResponse(body []byte) (string, error) { + var err error + body, err = unwrapXBSoraNestedResponse(body) + if err != nil { + return "", err + } + var resp xbSoraSubmitResponse + if err := common.Unmarshal(body, &resp); err != nil { + return "", errors.Wrap(err, "unmarshal xb-sora2 submit response failed") + } + if !xbSoraCodeOK(resp.Code) { + return "", fmt.Errorf("xb-sora2 submit failed: %s", xbSoraErrorMessage(firstNonEmpty(resp.Message, resp.Msg), resp.Error)) + } + if resp.Data.TaskID == "" { + return "", fmt.Errorf("xb-sora2 response data.task_id is empty") + } + return resp.Data.TaskID, nil +} + +func (p *xbSoraProvider) parseQueryResponse(body []byte) (*relaycommon.TaskInfo, error) { + var err error + body, err = unwrapXBSoraNestedResponse(body) + if err != nil { + return nil, err + } + var resp xbSoraQueryResponse + if err := common.Unmarshal(body, &resp); err != nil { + return nil, errors.Wrap(err, "unmarshal xb-sora2 query response failed") + } + if !xbSoraCodeOK(resp.Code) { + return nil, fmt.Errorf("xb-sora2 query failed: %s", xbSoraErrorMessage(firstNonEmpty(resp.Message, resp.Msg), resp.Error)) + } + + ti := &relaycommon.TaskInfo{ + Code: 0, + TaskID: resp.Data.TaskID, + Status: statusToTaskStatus(resp.Data.Status), + } + if ti.Status == model.TaskStatusSuccess && resp.Data.Result != nil { + ti.Url = firstNonEmpty(resp.Data.Result.VideoURL, firstXBSoraResultURL(resp.Data.Result.ResultURLs)) + if ti.Url == "" { + ti.Status = model.TaskStatusFailure + ti.Reason = "xb-sora2 completed without video_url" + } + } + if ti.Status == model.TaskStatusFailure { + if resp.Data.Error != nil { + ti.Reason = xbSoraErrorMessage("", resp.Data.Error) + } else { + ti.Reason = "task failed" + } + } + if resp.Data.Progress > 0 && resp.Data.Progress < 100 { + ti.Progress = fmt.Sprintf("%d%%", resp.Data.Progress) + } + return ti, nil +} + +func firstXBSoraResultURL(urls []string) string { + for _, u := range urls { + if strings.TrimSpace(u) != "" { + return strings.TrimSpace(u) + } + } + return "" +} + +func (p *xbSoraProvider) buildSubmitResponseBody(info *relaycommon.RelayInfo, upstreamTaskID string) any { + return map[string]any{ + "id": info.PublicTaskID, + "task_id": info.PublicTaskID, + "object": "video", + "model": info.OriginModelName, + "status": "queued", + "progress": 0, + "created_at": 0, + } +} + +func (p *xbSoraProvider) needsMultipart() bool { return false } + +func (p *xbSoraProvider) forceJSONBody() bool { return true } + +func (p *xbSoraProvider) setupRequestHeader(req *http.Request, apiKey string) { + req.Header.Set("X-API-Key", apiKey) +} + +func (p *xbSoraProvider) mapModelForImages(model string, hasImages bool) string { + model = strings.TrimSpace(model) + switch model { + case "xb-sora2": + return model + case "xb-sora-2", "sora-2", "openai-sora-2", "sora-2-image-to-video": + return "xb-sora2" + case "sora-2-pro", "sora-2-pro-text-to-video": + return "sora-2-pro(线路BF)" + default: + return model + } +} + +func (p *xbSoraProvider) normalizeJSONRequest(bodyMap map[string]interface{}, originModel, upstreamModel string, imageCount int) { + normalizeXBSoraRequestMap(bodyMap, imageCount, upstreamModel) +} + +func (p *xbSoraProvider) normalizeMultipartRequest(values map[string][]string, originModel, upstreamModel string, imageCount int) { + bodyMap := multipartValuesToMap(values) + normalizeXBSoraRequestMap(bodyMap, imageCount, upstreamModel) + for k := range values { + delete(values, k) + } + for k, v := range bodyMap { + values[k] = []string{fmt.Sprintf("%v", v)} + } + if images, ok := bodyMap["images"].([]string); ok { + values["images"] = images + } +} + +func xbSoraAPIBase(baseURL string) string { + baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/") + if strings.HasSuffix(baseURL, "/api/v1") || strings.HasSuffix(baseURL, "/v1") { + return baseURL + } + return baseURL + "/api/v1" +} + +func normalizeXBSoraRequestMap(bodyMap map[string]interface{}, imageCount int, modelName string) { + bodyMap["duration"] = normalizeXBSoraDuration(getXBSoraDuration(bodyMap), modelName) + if isXBSoraGrokModelName(modelName) { + bodyMap["aspect_ratio"] = normalizeXBSoraGrokAspectRatio(bodyMap) + if images := collectXBSoraImages(bodyMap); len(images) > 0 { + bodyMap["images"] = images + } + for _, key := range []string{ + "seconds", + "input_reference", + "image", + "ratio", + "orientation", + "size", + "width", + "height", + "fps", + "seed", + "n", + "response_format", + "user", + } { + delete(bodyMap, key) + } + return + } + if orientation := getXBSoraOrientation(bodyMap); orientation != "" { + bodyMap["orientation"] = orientation + } + if images := collectXBSoraImages(bodyMap); len(images) > 0 { + bodyMap["images"] = images + } + + for _, key := range []string{ + "size", + "seconds", + "input_reference", + "image", + "aspect_ratio", + "ratio", + "width", + "height", + "fps", + "seed", + "n", + "response_format", + "user", + } { + delete(bodyMap, key) + } + + if imageCount > 0 && bodyMap["model"] == "" { + bodyMap["model"] = "xb-sora2" + } +} + +func normalizeXBSoraDuration(duration int, modelName string) int { + supported := supportedXBSoraDurations(modelName) + if len(supported) == 0 { + if duration > 0 { + return duration + } + return 8 + } + if duration <= 0 { + return defaultXBSoraDuration(modelName, supported) + } + for _, allowed := range supported { + if duration == allowed { + return duration + } + if duration < allowed { + return allowed + } + } + return supported[len(supported)-1] +} + +func supportedXBSoraDurations(modelName string) []int { + lower := strings.ToLower(strings.TrimSpace(modelName)) + switch { + case isXBSoraGrokModelName(modelName): + return []int{6, 10} + case strings.Contains(modelName, "全能视频"): + return []int{4, 5, 8, 10, 15} + case lower == "openai-sora-2" || lower == "sora-2-image-to-video": + return []int{10, 15} + case strings.Contains(lower, "sora"): + return []int{4, 8, 12} + default: + return nil + } +} + +func isXBSoraGrokModelName(modelName string) bool { + lower := strings.ToLower(strings.TrimSpace(modelName)) + return lower == "je-grok" || strings.Contains(lower, "grok") +} + +func isXBSoraProtectedResultURL(resultURL string) bool { + lower := strings.ToLower(strings.TrimSpace(resultURL)) + return strings.Contains(lower, "open.hongniaoai.com") || + strings.Contains(lower, ".33502349.xyz/") +} + +func normalizeXBSoraGrokAspectRatio(bodyMap map[string]interface{}) string { + for _, key := range []string{"size", "aspect_ratio", "ratio"} { + if aspectRatio := grokAspectRatioFromXBSoraValue(bodyMap[key]); aspectRatio != "" { + return aspectRatio + } + } + switch getXBSoraOrientation(bodyMap) { + case "portrait": + return "720x1280" + default: + return "1280x720" + } +} + +func grokAspectRatioFromXBSoraValue(value interface{}) string { + s, ok := value.(string) + if !ok { + return "" + } + s = strings.TrimSpace(s) + switch strings.ToLower(s) { + case "landscape", "horizontal": + return "1280x720" + case "portrait", "vertical": + return "720x1280" + case "16:9", "1280x720": + return "1280x720" + case "9:16", "720x1280": + return "720x1280" + case "横屏 16:9": + return "1280x720" + case "竖屏 9:16": + return "720x1280" + default: + return "" + } +} + +func defaultXBSoraDuration(modelName string, supported []int) int { + lower := strings.ToLower(strings.TrimSpace(modelName)) + switch { + case lower == "ss-sora-2": + return 4 + case strings.Contains(modelName, "全能视频"): + return 15 + case len(supported) > 1: + return supported[1] + default: + return supported[0] + } +} + +func getXBSoraDuration(bodyMap map[string]interface{}) int { + for _, key := range []string{"duration", "seconds"} { + if duration := intFromXBSoraValue(bodyMap[key]); duration > 0 { + return duration + } + } + return 0 +} + +func getXBSoraOrientation(bodyMap map[string]interface{}) string { + for _, key := range []string{"orientation", "aspect_ratio", "ratio", "size"} { + if orientation := orientationFromXBSoraValue(bodyMap[key]); orientation != "" { + return orientation + } + } + return "landscape" +} + +func collectXBSoraImages(bodyMap map[string]interface{}) []string { + seen := make(map[string]struct{}) + images := make([]string, 0) + for _, key := range []string{"images", "image", "input_reference", "image_url"} { + for _, image := range stringsFromXBSoraValue(bodyMap[key]) { + image = strings.TrimSpace(image) + if image == "" { + continue + } + if _, ok := seen[image]; ok { + continue + } + seen[image] = struct{}{} + images = append(images, image) + } + } + return images +} + +func intFromXBSoraValue(value interface{}) int { + switch v := value.(type) { + case int: + return v + case int64: + return int(v) + case float64: + return int(v) + case string: + i, _ := strconv.Atoi(strings.TrimSpace(v)) + return i + default: + return 0 + } +} + +func orientationFromXBSoraValue(value interface{}) string { + switch v := value.(type) { + case string: + switch strings.ToLower(strings.TrimSpace(v)) { + case "landscape", "horizontal", "16:9", "1280x720", "1792x1024": + return "landscape" + case "portrait", "vertical", "9:16", "720x1280", "1024x1792": + return "portrait" + } + } + return "" +} + +func stringsFromXBSoraValue(value interface{}) []string { + switch v := value.(type) { + case []string: + return v + case []interface{}: + out := make([]string, 0, len(v)) + for _, item := range v { + out = append(out, stringsFromXBSoraValue(item)...) + } + return out + case map[string]interface{}: + if imageURL, ok := v["image_url"]; ok { + return stringsFromXBSoraValue(imageURL) + } + if url, ok := v["url"].(string); ok { + return []string{url} + } + case string: + return []string{v} + } + return nil +} + +func xbSoraErrorMessage(message string, errInfo *xbSoraError) string { + if errInfo == nil { + return message + } + for _, msg := range []string{errInfo.Message, errInfo.Details, errInfo.Code, errInfo.Type, message} { + if strings.TrimSpace(msg) != "" { + return msg + } + } + return "unknown error" +} + +func unwrapXBSoraNestedResponse(body []byte) ([]byte, error) { + var env xbSoraResponseEnvelope + if err := common.Unmarshal(body, &env); err != nil { + return body, nil + } + if !xbSoraCodeOK(env.Code) { + return nil, fmt.Errorf("xb-sora2 request failed: %s", firstNonEmpty(env.Message, env.Msg)) + } + if len(env.Data) == 0 { + return body, nil + } + var nested xbSoraResponseEnvelope + if err := common.Unmarshal(env.Data, &nested); err != nil { + return body, nil + } + if len(nested.Data) == 0 { + return body, nil + } + return env.Data, nil +} + +func xbSoraCodeOK(code any) bool { + switch v := code.(type) { + case nil: + return true + case int: + return v == 0 || v == http.StatusOK + case int64: + return v == 0 || v == http.StatusOK + case float64: + return v == 0 || v == http.StatusOK + case string: + v = strings.TrimSpace(v) + return v == "" || v == "0" || v == "0000" || v == "200" + default: + return false + } +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return value + } + } + return "" +} diff --git a/relay/channel/task/openaivideo/xb_sora_test.go b/relay/channel/task/openaivideo/xb_sora_test.go new file mode 100644 index 000000000000..e2de2d06f3ac --- /dev/null +++ b/relay/channel/task/openaivideo/xb_sora_test.go @@ -0,0 +1,326 @@ +package openaivideo + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/QuantumNous/new-api/model" + relaycommon "github.com/QuantumNous/new-api/relay/common" + "github.com/QuantumNous/new-api/service" + "github.com/gin-gonic/gin" +) + +func TestXBSoraProviderURLsAndHeader(t *testing.T) { + p := &xbSoraProvider{} + + if got := p.submitURL("https://example.com"); got != "https://example.com/api/v1/videos/generate" { + t.Fatalf("submitURL host root = %q", got) + } + if got := p.submitURL("https://example.com/api/v1/"); got != "https://example.com/api/v1/videos/generate" { + t.Fatalf("submitURL api base = %q", got) + } + if got := p.submitURL("https://localhost:3000/v1"); got != "https://localhost:3000/v1/videos/generate" { + t.Fatalf("submitURL v1 base = %q", got) + } + if got := p.queryURL("https://example.com/api/v1", "task_123"); got != "https://example.com/api/v1/videos/task_123" { + t.Fatalf("queryURL = %q", got) + } + + req, err := http.NewRequest(http.MethodGet, "https://example.com", nil) + if err != nil { + t.Fatal(err) + } + p.setupRequestHeader(req, "sk_test") + if got := req.Header.Get("X-API-Key"); got != "sk_test" { + t.Fatalf("X-API-Key = %q", got) + } + if got := req.Header.Get("Authorization"); got != "" { + t.Fatalf("Authorization should be empty, got %q", got) + } +} + +func TestXBSoraProviderSelection(t *testing.T) { + if _, ok := getProviderByBaseURL("https://localhost:3000/v1").(*xbSoraProvider); !ok { + t.Fatalf("localhost /v1 base URL should select xb-sora2 provider") + } + if _, ok := getProviderByBaseURL("https://example.com/api/v1").(*xbSoraProvider); !ok { + t.Fatalf("/api/v1 base URL should select xb-sora2 provider") + } + if _, ok := getProviderByBaseURL("https://xgapi.top/api/v1").(*xgapiProvider); !ok { + t.Fatalf("explicit xgapi base URL should keep xgapi provider") + } + if _, ok := getProviderForRelayInfo(&relaycommon.RelayInfo{ + OriginModelName: "xb-sora2", + ChannelMeta: &relaycommon.ChannelMeta{ + ChannelBaseUrl: "https://example.com", + UpstreamModelName: "xb-sora2", + }, + }).(*xbSoraProvider); !ok { + t.Fatalf("xb-sora2 model should select xb-sora2 provider") + } + if _, ok := getProviderForRelayInfo(&relaycommon.RelayInfo{ + OriginModelName: "future-video-model", + ChannelMeta: &relaycommon.ChannelMeta{ + ChannelBaseUrl: "https://example.com", + ChannelOther: "xb-sora2", + UpstreamModelName: "future-video-model", + }, + }).(*xbSoraProvider); !ok { + t.Fatalf("xb-sora2 channel hint should select xb-sora2 provider for future models") + } + if _, ok := getProviderForTaskFetch("https://example.com", map[string]any{ + "task_id": "up_task", + "channel_other": "xb-sora2", + "upstream_model_name": "future-video-model", + }).(*xbSoraProvider); !ok { + t.Fatalf("xb-sora2 task fetch hint should select xb-sora2 provider") + } +} + +func TestXBSoraModelMappingAndNormalize(t *testing.T) { + p := &xbSoraProvider{} + + if got := p.mapModelForImages("xb-sora2", false); got != "xb-sora2" { + t.Fatalf("xb-sora2 mapped to %q", got) + } + if got := p.mapModelForImages("sora-2-pro", false); got != "sora-2-pro(线路BF)" { + t.Fatalf("sora-2-pro mapped to %q", got) + } + if got := p.mapModelForImages("openai-sora-2", true); got != "xb-sora2" { + t.Fatalf("image request mapped to %q", got) + } + if got := p.mapModelForImages("future-video-model", false); got != "future-video-model" { + t.Fatalf("unknown model should pass through, got %q", got) + } + if got := p.mapModelForImages("future-video-model", true); got != "future-video-model" { + t.Fatalf("unknown image-capable model should pass through, got %q", got) + } + + body := map[string]interface{}{ + "model": "openai-sora-2", + "prompt": "test", + "seconds": "12", + "size": "720x1280", + "input_reference": "https://example.com/a.png", + "image": "https://example.com/a.png", + } + p.normalizeJSONRequest(body, "xb-sora2", "xb-sora2", 1) + + if got := body["duration"]; got != 12 { + t.Fatalf("duration = %#v", got) + } + if got := body["orientation"]; got != "portrait" { + t.Fatalf("orientation = %#v", got) + } + images, ok := body["images"].([]string) + if !ok || len(images) != 1 || images[0] != "https://example.com/a.png" { + t.Fatalf("images = %#v", body["images"]) + } + if _, ok := body["seconds"]; ok { + t.Fatalf("seconds should be removed") + } + if _, ok := body["input_reference"]; ok { + t.Fatalf("input_reference should be removed") + } + + grokBody := map[string]interface{}{ + "model": "je-grok", + "prompt": "test", + "duration": float64(6), + "orientation": "landscape", + } + p.normalizeJSONRequest(grokBody, "je-grok", "je-grok", 0) + + if got := grokBody["duration"]; got != 6 { + t.Fatalf("grok duration = %#v", got) + } + if got := grokBody["aspect_ratio"]; got != "1280x720" { + t.Fatalf("grok aspect_ratio = %#v", got) + } + if _, ok := grokBody["orientation"]; ok { + t.Fatalf("grok orientation should be removed") + } + if _, ok := grokBody["size"]; ok { + t.Fatalf("grok size should be removed") + } +} + +func TestXBSoraParseResponses(t *testing.T) { + p := &xbSoraProvider{} + + taskID, err := p.parseSubmitResponse([]byte(`{"code":200,"message":"ok","data":{"task_id":"up_task","status":"pending","model":"openai-sora-2"}}`)) + if err != nil { + t.Fatalf("parseSubmitResponse error: %v", err) + } + if taskID != "up_task" { + t.Fatalf("taskID = %q", taskID) + } + + taskID, err = p.parseSubmitResponse([]byte(`{"code":"0000","msg":"success","data":{"code":200,"message":"ok","data":{"task_id":"up_task_nested","status":"pending","model":"xb-sora2"}}}`)) + if err != nil { + t.Fatalf("parse nested parseSubmitResponse error: %v", err) + } + if taskID != "up_task_nested" { + t.Fatalf("nested taskID = %q", taskID) + } + + info, err := p.parseQueryResponse([]byte(`{"code":200,"message":"ok","data":{"task_id":"up_task","status":"completed","progress":100,"result":{"video_url":"https://cdn.example.com/a.mp4","duration":10,"format":"mp4"}}}`)) + if err != nil { + t.Fatalf("parseQueryResponse completed error: %v", err) + } + if info.Status != model.TaskStatusSuccess { + t.Fatalf("status = %q", info.Status) + } + if info.Url != "https://cdn.example.com/a.mp4" { + t.Fatalf("url = %q", info.Url) + } + + info, err = p.parseQueryResponse([]byte(`{"code":"0000","msg":"success","data":{"code":200,"message":"ok","data":{"task_id":"up_task","status":"completed","progress":100,"result":{"video_url":"https://cdn.example.com/b.mp4","duration":8,"format":"mp4"}}}}`)) + if err != nil { + t.Fatalf("parse nested parseQueryResponse completed error: %v", err) + } + if info.Status != model.TaskStatusSuccess || info.Url != "https://cdn.example.com/b.mp4" { + t.Fatalf("nested info = %+v", info) + } + + info, err = p.parseQueryResponse([]byte(`{"code":200,"message":"ok","data":{"task_id":"up_task","status":"failed","error":{"code":"generation_failed","message":"failed upstream"}}}`)) + if err != nil { + t.Fatalf("parseQueryResponse failed error: %v", err) + } + if info.Status != model.TaskStatusFailure || info.Reason != "failed upstream" { + t.Fatalf("failed info = %+v", info) + } +} + +func TestXBSoraSubmitResponseBodyUsesPublicTaskID(t *testing.T) { + p := &xbSoraProvider{} + body := p.buildSubmitResponseBody(&relaycommon.RelayInfo{ + OriginModelName: "xb-sora2", + TaskRelayInfo: &relaycommon.TaskRelayInfo{ + PublicTaskID: "task_public", + }, + }, "upstream_task").(map[string]any) + + if body["id"] != "task_public" || body["task_id"] != "task_public" { + t.Fatalf("public ids not used: %+v", body) + } + if body["model"] != "xb-sora2" { + t.Fatalf("model = %#v", body["model"]) + } +} + +func TestXBSoraAdaptorSubmitAndFetchHTTP(t *testing.T) { + gin.SetMode(gin.TestMode) + service.InitHttpClient() + + var submitSeen bool + var fetchSeen bool + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("X-API-Key"); got != "sk_test" { + t.Fatalf("X-API-Key = %q", got) + } + if got := r.Header.Get("Authorization"); got != "" { + t.Fatalf("Authorization should be empty, got %q", got) + } + + switch { + case r.Method == http.MethodPost && r.URL.Path == "/api/v1/videos/generate": + submitSeen = true + if got := r.Header.Get("Content-Type"); got != "application/json" { + t.Fatalf("submit Content-Type = %q", got) + } + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatal(err) + } + var req map[string]any + if err := json.Unmarshal(body, &req); err != nil { + t.Fatalf("submit body is not json: %v", err) + } + if req["model"] != "xb-sora2" { + t.Fatalf("model = %#v", req["model"]) + } + if req["duration"] != float64(12) { + t.Fatalf("duration = %#v", req["duration"]) + } + if req["orientation"] != "portrait" { + t.Fatalf("orientation = %#v", req["orientation"]) + } + for _, key := range []string{"seconds", "size", "input_reference"} { + if _, ok := req[key]; ok { + t.Fatalf("%s should not be sent: %#v", key, req) + } + } + _, _ = w.Write([]byte(`{"code":"0000","msg":"success","data":{"code":200,"message":"ok","data":{"task_id":"up_task","status":"pending","model":"xb-sora2"}}}`)) + case r.Method == http.MethodGet && r.URL.Path == "/api/v1/videos/up_task": + fetchSeen = true + _, _ = w.Write([]byte(`{"code":"0000","msg":"success","data":{"code":200,"message":"ok","data":{"task_id":"up_task","status":"completed","progress":100,"model":"xb-sora2","result":{"video_url":"https://cdn.example.com/a.mp4","duration":12,"format":"mp4"}}}}`)) + default: + t.Fatalf("unexpected upstream request: %s %s", r.Method, r.URL.Path) + } + })) + defer server.Close() + + info := &relaycommon.RelayInfo{ + OriginModelName: "xb-sora2", + ChannelMeta: &relaycommon.ChannelMeta{ + ChannelBaseUrl: server.URL + "/api/v1", + ApiKey: "sk_test", + UpstreamModelName: "xb-sora2", + }, + TaskRelayInfo: &relaycommon.TaskRelayInfo{ + PublicTaskID: "task_public", + }, + } + adaptor := &TaskAdaptor{} + adaptor.Init(info) + + body := []byte(`{"model":"xb-sora2","prompt":"test","seconds":"12","size":"720x1280","input_reference":"https://example.com/a.png"}`) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/videos", bytes.NewReader(body)) + c.Request.Header.Set("Content-Type", "application/json") + + requestBody, err := adaptor.BuildRequestBody(c, info) + if err != nil { + t.Fatalf("BuildRequestBody error: %v", err) + } + resp, err := adaptor.DoRequest(c, info, requestBody) + if err != nil { + t.Fatalf("DoRequest error: %v", err) + } + upstreamID, _, taskErr := adaptor.DoResponse(c, resp, info) + if taskErr != nil { + t.Fatalf("DoResponse taskErr: %v", taskErr) + } + if upstreamID != "up_task" { + t.Fatalf("upstreamID = %q", upstreamID) + } + if !submitSeen { + t.Fatalf("submit endpoint was not called") + } + + resp, err = adaptor.FetchTask(server.URL+"/api/v1", "sk_test", map[string]any{"task_id": upstreamID}, "") + if err != nil { + t.Fatalf("FetchTask error: %v", err) + } + defer resp.Body.Close() + responseBody, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read fetch response: %v", err) + } + taskInfo, err := adaptor.ParseTaskResult(responseBody) + if err != nil { + t.Fatalf("ParseTaskResult error: %v", err) + } + if taskInfo.Status != model.TaskStatusSuccess || taskInfo.Url != "https://cdn.example.com/a.mp4" { + t.Fatalf("taskInfo = %+v", taskInfo) + } + if !fetchSeen { + t.Fatalf("fetch endpoint was not called") + } +} diff --git a/relay/channel/task/openaivideo/xgapi.go b/relay/channel/task/openaivideo/xgapi.go new file mode 100644 index 000000000000..7b2afa45f282 --- /dev/null +++ b/relay/channel/task/openaivideo/xgapi.go @@ -0,0 +1,97 @@ +package openaivideo + +import ( + "fmt" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/model" + relaycommon "github.com/QuantumNous/new-api/relay/common" + "github.com/pkg/errors" +) + +type xgapiSubmitResponse struct { + ID string `json:"id"` + Object string `json:"object"` + Model string `json:"model"` + Status string `json:"status"` + Progress int `json:"progress"` + CreatedAt int64 `json:"created_at"` + Seconds string `json:"seconds"` + Size string `json:"size"` +} + +type xgapiQueryResponse struct { + ID string `json:"id"` + Status string `json:"status"` + Progress int `json:"progress"` + VideoURL *string `json:"video_url"` + CompletedAt int64 `json:"completed_at"` + Error *struct { + Message string `json:"message"` + Code string `json:"code"` + } `json:"error,omitempty"` +} + +type xgapiProvider struct{} + +func (p *xgapiProvider) submitURL(baseURL string) string { + return fmt.Sprintf("%s/v1/videos", baseURL) +} + +func (p *xgapiProvider) queryURL(baseURL, taskID string) string { + return fmt.Sprintf("%s/v1/videos/%s", baseURL, taskID) +} + +func (p *xgapiProvider) parseSubmitResponse(body []byte) (string, error) { + var resp xgapiSubmitResponse + if err := common.Unmarshal(body, &resp); err != nil { + return "", errors.Wrap(err, "unmarshal xgapi submit response failed") + } + if resp.ID == "" { + return "", fmt.Errorf("xgapi response id is empty") + } + return resp.ID, nil +} + +func (p *xgapiProvider) parseQueryResponse(body []byte) (*relaycommon.TaskInfo, error) { + var resp xgapiQueryResponse + if err := common.Unmarshal(body, &resp); err != nil { + return nil, errors.Wrap(err, "unmarshal xgapi query response failed") + } + + ti := &relaycommon.TaskInfo{Code: 0} + ti.Status = statusToTaskStatus(resp.Status) + + if ti.Status == model.TaskStatusSuccess && resp.VideoURL != nil && *resp.VideoURL != "" { + ti.Url = *resp.VideoURL + } + if ti.Status == model.TaskStatusFailure { + if resp.Error != nil { + ti.Reason = resp.Error.Message + } else { + ti.Reason = "task failed" + } + } + if resp.Progress > 0 && resp.Progress < 100 { + ti.Progress = fmt.Sprintf("%d%%", resp.Progress) + } + return ti, nil +} + +func (p *xgapiProvider) buildSubmitResponseBody(info *relaycommon.RelayInfo, upstreamTaskID string) any { + return map[string]any{ + "id": info.PublicTaskID, + "task_id": info.PublicTaskID, + "object": "video", + "model": info.OriginModelName, + "status": "queued", + "progress": 0, + "created_at": 0, + } +} + +func (p *xgapiProvider) needsMultipart() bool { return true } + +func (p *xgapiProvider) mapModelForImages(model string, hasImages bool) string { + return model +} diff --git a/relay/common/relay_info.go b/relay/common/relay_info.go index 2f7afd398599..434cc3d4e4a5 100644 --- a/relay/common/relay_info.go +++ b/relay/common/relay_info.go @@ -71,6 +71,7 @@ type ChannelMeta struct { ApiKey string Organization string ChannelCreateTime int64 + ChannelOther string ParamOverride map[string]interface{} HeadersOverride map[string]interface{} ChannelSetting dto.ChannelSettings @@ -204,6 +205,7 @@ func (info *RelayInfo) InitChannelMeta(c *gin.Context) { ApiKey: common.GetContextKeyString(c, constant.ContextKeyChannelKey), Organization: c.GetString("channel_organization"), ChannelCreateTime: c.GetInt64("channel_create_time"), + ChannelOther: common.GetContextKeyString(c, constant.ContextKeyChannelOther), ParamOverride: paramOverride, HeadersOverride: headerOverride, UpstreamModelName: common.GetContextKeyString(c, constant.ContextKeyOriginalModel), @@ -290,8 +292,8 @@ func (info *RelayInfo) ToString() string { // Channel metadata (mask ApiKey) if info.ChannelMeta != nil { cm := info.ChannelMeta - fmt.Fprintf(b, "ChannelMeta{ Type: %d, Id: %d, IsMultiKey: %t, MultiKeyIndex: %d, BaseURL: %q, ApiType: %d, ApiVersion: %q, Organization: %q, CreateTime: %d, UpstreamModelName: %q, IsModelMapped: %t, SupportStreamOptions: %t, ApiKey: ***masked*** }, ", - cm.ChannelType, cm.ChannelId, cm.ChannelIsMultiKey, cm.ChannelMultiKeyIndex, cm.ChannelBaseUrl, cm.ApiType, cm.ApiVersion, cm.Organization, cm.ChannelCreateTime, cm.UpstreamModelName, cm.IsModelMapped, cm.SupportStreamOptions) + fmt.Fprintf(b, "ChannelMeta{ Type: %d, Id: %d, IsMultiKey: %t, MultiKeyIndex: %d, BaseURL: %q, Other: %q, ApiType: %d, ApiVersion: %q, Organization: %q, CreateTime: %d, UpstreamModelName: %q, IsModelMapped: %t, SupportStreamOptions: %t, ApiKey: ***masked*** }, ", + cm.ChannelType, cm.ChannelId, cm.ChannelIsMultiKey, cm.ChannelMultiKeyIndex, cm.ChannelBaseUrl, cm.ChannelOther, cm.ApiType, cm.ApiVersion, cm.Organization, cm.ChannelCreateTime, cm.UpstreamModelName, cm.IsModelMapped, cm.SupportStreamOptions) } // Responses usage info (non-sensitive) @@ -691,6 +693,9 @@ type TaskSubmitReq struct { Duration int `json:"duration,omitempty"` Seconds string `json:"seconds,omitempty"` InputReference string `json:"input_reference,omitempty"` + AspectRatio string `json:"aspect_ratio,omitempty"` + EnhancePrompt *bool `json:"enhance_prompt,omitempty"` + EnableUpsample *bool `json:"enable_upsample,omitempty"` Metadata map[string]interface{} `json:"metadata,omitempty"` } diff --git a/relay/common/relay_utils.go b/relay/common/relay_utils.go index 18df77a645d6..fc5046c28cc1 100644 --- a/relay/common/relay_utils.go +++ b/relay/common/relay_utils.go @@ -3,6 +3,7 @@ package common import ( "fmt" "net/http" + "net/url" "strconv" "strings" @@ -23,17 +24,97 @@ type HasImage interface { } func GetFullRequestURL(baseURL string, requestURL string, channelType int) string { - fullRequestURL := fmt.Sprintf("%s%s", baseURL, requestURL) - if strings.HasPrefix(baseURL, "https://gateway.ai.cloudflare.com") { switch channelType { case constant.ChannelTypeOpenAI: - fullRequestURL = fmt.Sprintf("%s%s", baseURL, strings.TrimPrefix(requestURL, "/v1")) + return joinRequestURL(baseURL, strings.TrimPrefix(requestURL, "/v1")) case constant.ChannelTypeAzure: - fullRequestURL = fmt.Sprintf("%s%s", baseURL, strings.TrimPrefix(requestURL, "/openai/deployments")) + return joinRequestURL(baseURL, strings.TrimPrefix(requestURL, "/openai/deployments")) + } + } + return joinRequestURL(baseURL, requestURL) +} + +func joinRequestURL(baseURL string, requestURL string) string { + baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/") + requestURL = strings.TrimSpace(requestURL) + if baseURL == "" { + return requestURL + } + if requestURL == "" { + return baseURL + } + + request, err := url.Parse(requestURL) + if err != nil { + return simpleJoinRequestURL(baseURL, requestURL) + } + if request.IsAbs() { + return request.String() + } + + if base, err := url.Parse(baseURL); err == nil { + overlap := overlappingPathSegments(base.Path, request.Path) + if overlap > 0 { + request.Path = trimLeadingPathSegments(request.Path, overlap) + request.RawPath = "" + } + } + + relative := request.String() + if relative == "" { + return baseURL + } + if strings.HasPrefix(relative, "/") || strings.HasPrefix(relative, "?") || strings.HasPrefix(relative, "#") { + return baseURL + relative + } + return baseURL + "/" + relative +} + +func simpleJoinRequestURL(baseURL string, requestURL string) string { + if strings.HasPrefix(requestURL, "/") { + return baseURL + requestURL + } + return baseURL + "/" + requestURL +} + +func overlappingPathSegments(basePath string, requestPath string) int { + baseSegments := splitPathSegments(basePath) + requestSegments := splitPathSegments(requestPath) + maxOverlap := len(baseSegments) + if len(requestSegments) < maxOverlap { + maxOverlap = len(requestSegments) + } + + for overlap := maxOverlap; overlap > 0; overlap-- { + matched := true + for i := 0; i < overlap; i++ { + if baseSegments[len(baseSegments)-overlap+i] != requestSegments[i] { + matched = false + break + } + } + if matched { + return overlap } } - return fullRequestURL + return 0 +} + +func splitPathSegments(path string) []string { + path = strings.Trim(path, "/") + if path == "" { + return nil + } + return strings.Split(path, "/") +} + +func trimLeadingPathSegments(path string, count int) string { + segments := splitPathSegments(path) + if count >= len(segments) { + return "" + } + return "/" + strings.Join(segments[count:], "/") } func GetAPIVersion(c *gin.Context) string { @@ -157,6 +238,16 @@ func ValidateMultipartDirect(c *gin.Context, info *RelayInfo) *dto.TaskError { if hasInputReference { action = constant.TaskActionGenerate } + + if isFramesModel(model) { + if hasInputReference { + action = constant.TaskActionGenerate + } + if req.AspectRatio != "" && !lo.Contains([]string{"9:16", "16:9"}, req.AspectRatio) { + return createTaskError(fmt.Errorf("aspect_ratio must be 9:16 or 16:9"), "invalid_aspect_ratio", http.StatusBadRequest, true) + } + } + if strings.HasPrefix(model, "sora-2") { if size == "" { @@ -173,7 +264,6 @@ func ValidateMultipartDirect(c *gin.Context, info *RelayInfo) *dto.TaskError { if model == "sora-2-pro" && !lo.Contains([]string{"720x1280", "1280x720", "1792x1024", "1024x1792"}, size) { return createTaskError(fmt.Errorf("sora-2 size is invalid"), "invalid_size", http.StatusBadRequest, true) } - // OtherRatios 已移到 Sora adaptor 的 EstimateBilling 中设置 } storeTaskRequest(c, info, action, req) @@ -181,6 +271,10 @@ func ValidateMultipartDirect(c *gin.Context, info *RelayInfo) *dto.TaskError { return nil } +func isFramesModel(model string) bool { + return strings.HasSuffix(model, "-frames") || strings.HasSuffix(model, "-components") || strings.Contains(model, "-frames-") || strings.Contains(model, "-components-") +} + func isKnownTaskField(field string) bool { knownFields := map[string]bool{ "prompt": true, @@ -190,7 +284,10 @@ func isKnownTaskField(field string) bool { "images": true, "size": true, "duration": true, - "input_reference": true, // Sora 特有字段 + "input_reference": true, + "aspect_ratio": true, + "enhance_prompt": true, + "enable_upsample": true, } return knownFields[field] } diff --git a/relay/common/relay_utils_test.go b/relay/common/relay_utils_test.go new file mode 100644 index 000000000000..dbc5f5a340cc --- /dev/null +++ b/relay/common/relay_utils_test.go @@ -0,0 +1,76 @@ +package common + +import ( + "testing" + + "github.com/QuantumNous/new-api/constant" +) + +func TestGetFullRequestURL(t *testing.T) { + tests := []struct { + name string + baseURL string + requestURL string + channelType int + want string + }{ + { + name: "base path v1 does not duplicate request v1", + baseURL: "https://open.hongniaoai.com/v1", + requestURL: "/v1/images/generations", + channelType: constant.ChannelTypeOpenAI, + want: "https://open.hongniaoai.com/v1/images/generations", + }, + { + name: "base path suffix overlaps request prefix", + baseURL: "https://example.com/api/v1/", + requestURL: "/v1/images/generations?foo=bar", + channelType: constant.ChannelTypeOpenAI, + want: "https://example.com/api/v1/images/generations?foo=bar", + }, + { + name: "non overlapping base path is preserved", + baseURL: "https://api.marswave.ai/openapi", + requestURL: "/v1/images/generation", + channelType: constant.ChannelTypeListenHub, + want: "https://api.marswave.ai/openapi/v1/images/generation", + }, + { + name: "root base path joins request", + baseURL: "https://api.openai.com", + requestURL: "/v1/chat/completions", + channelType: constant.ChannelTypeOpenAI, + want: "https://api.openai.com/v1/chat/completions", + }, + { + name: "request without leading slash", + baseURL: "https://api.openai.com/v1", + requestURL: "chat/completions", + channelType: constant.ChannelTypeOpenAI, + want: "https://api.openai.com/v1/chat/completions", + }, + { + name: "cloudflare openai keeps gateway provider path", + baseURL: "https://gateway.ai.cloudflare.com/v1/account/gateway/openai", + requestURL: "/v1/chat/completions", + channelType: constant.ChannelTypeOpenAI, + want: "https://gateway.ai.cloudflare.com/v1/account/gateway/openai/chat/completions", + }, + { + name: "absolute request url is returned as-is", + baseURL: "https://api.openai.com/v1", + requestURL: "https://other.example.com/v1/models", + channelType: constant.ChannelTypeOpenAI, + want: "https://other.example.com/v1/models", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := GetFullRequestURL(tt.baseURL, tt.requestURL, tt.channelType) + if got != tt.want { + t.Fatalf("GetFullRequestURL() = %q, want %q", got, tt.want) + } + }) + } +} diff --git a/relay/helper/stream_scanner.go b/relay/helper/stream_scanner.go index bc0af1bb5b54..cbd2c0c7423d 100644 --- a/relay/helper/stream_scanner.go +++ b/relay/helper/stream_scanner.go @@ -23,7 +23,8 @@ import ( const ( InitialScannerBufferSize = 64 << 10 // 64KB (64*1024) - DefaultMaxScannerBufferSize = 128 << 20 // 64MB (64*1024*1024) default SSE buffer size + DefaultMaxScannerBufferSize = 128 << 20 // 128MB default SSE buffer size + DefaultStreamingTimeout = 300 * time.Second DefaultPingInterval = 10 * time.Second ) @@ -46,8 +47,9 @@ func StreamScannerHandler(c *gin.Context, resp *http.Response, info *relaycommon return } - // 无条件新建 StreamStatus - info.StreamStatus = relaycommon.NewStreamStatus() + if info.StreamStatus == nil { + info.StreamStatus = relaycommon.NewStreamStatus() + } // 确保响应体总是被关闭 defer func() { @@ -57,6 +59,9 @@ func StreamScannerHandler(c *gin.Context, resp *http.Response, info *relaycommon }() streamingTimeout := time.Duration(constant.StreamingTimeout) * time.Second + if streamingTimeout <= 0 { + streamingTimeout = DefaultStreamingTimeout + } var ( stopChan = make(chan bool, 3) // 增加缓冲区避免阻塞 diff --git a/relay/relay_adaptor.go b/relay/relay_adaptor.go index 3139c9a2dd4a..1ec719f8ff02 100644 --- a/relay/relay_adaptor.go +++ b/relay/relay_adaptor.go @@ -19,6 +19,7 @@ import ( "github.com/QuantumNous/new-api/relay/channel/gemini" "github.com/QuantumNous/new-api/relay/channel/jimeng" "github.com/QuantumNous/new-api/relay/channel/jina" + "github.com/QuantumNous/new-api/relay/channel/listenhub" "github.com/QuantumNous/new-api/relay/channel/minimax" "github.com/QuantumNous/new-api/relay/channel/mistral" "github.com/QuantumNous/new-api/relay/channel/mokaai" @@ -36,6 +37,7 @@ import ( "github.com/QuantumNous/new-api/relay/channel/task/hailuo" taskjimeng "github.com/QuantumNous/new-api/relay/channel/task/jimeng" "github.com/QuantumNous/new-api/relay/channel/task/kling" + taskopenaivideo "github.com/QuantumNous/new-api/relay/channel/task/openaivideo" tasksora "github.com/QuantumNous/new-api/relay/channel/task/sora" "github.com/QuantumNous/new-api/relay/channel/task/suno" taskvertex "github.com/QuantumNous/new-api/relay/channel/task/vertex" @@ -120,6 +122,8 @@ func GetAdaptor(apiType int) channel.Adaptor { return &replicate.Adaptor{} case constant.APITypeCodex: return &codex.Adaptor{} + case constant.APITypeListenHub: + return &listenhub.Adaptor{} } return nil } @@ -159,6 +163,8 @@ func GetTaskAdaptor(platform constant.TaskPlatform) channel.TaskAdaptor { return &taskGemini.TaskAdaptor{} case constant.ChannelTypeMiniMax: return &hailuo.TaskAdaptor{} + case constant.ChannelTypeOpenAIVideo: + return &taskopenaivideo.TaskAdaptor{} } } return nil diff --git a/relay/relay_task.go b/relay/relay_task.go index 098e23828b6c..bc54b683d240 100644 --- a/relay/relay_task.go +++ b/relay/relay_task.go @@ -221,7 +221,7 @@ func RelayTaskSubmit(c *gin.Context, info *relaycommon.RelayInfo) (*TaskSubmitRe if err != nil { return nil, service.TaskErrorWrapper(err, "do_request_failed", http.StatusInternalServerError) } - if resp != nil && resp.StatusCode != http.StatusOK { + if resp != nil && resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted { responseBody, _ := io.ReadAll(resp.Body) return nil, service.TaskErrorWrapper(fmt.Errorf("%s", string(responseBody)), "fail_to_fetch_task", resp.StatusCode) } @@ -438,8 +438,11 @@ func tryRealtimeFetch(task *model.Task, isOpenAIVideoAPI bool) []byte { } resp, err := adaptor.FetchTask(baseURL, channelModel.Key, map[string]any{ - "task_id": task.GetUpstreamTaskID(), - "action": task.Action, + "task_id": task.GetUpstreamTaskID(), + "action": task.Action, + "channel_other": channelModel.Other, + "origin_model_name": task.Properties.OriginModelName, + "upstream_model_name": task.Properties.UpstreamModelName, }, proxy) if err != nil || resp == nil { return nil diff --git a/router/api-router.go b/router/api-router.go index e98dc66ac048..330b36fe351b 100644 --- a/router/api-router.go +++ b/router/api-router.go @@ -238,6 +238,7 @@ func SetApiRouter(router *gin.Engine) { channelRoute.GET("/test/:id", controller.TestChannel) channelRoute.GET("/update_balance", controller.UpdateAllChannelsBalance) channelRoute.GET("/update_balance/:id", controller.UpdateChannelBalance) + channelRoute.GET("/balance_overview", controller.GetChannelBalanceOverview) channelRoute.POST("/", controller.AddChannel) channelRoute.PUT("/", controller.UpdateChannel) channelRoute.DELETE("/disabled", controller.DeleteDisabledChannel) diff --git a/service/task_polling.go b/service/task_polling.go index c5ec3ea33ead..3fc5c6175cec 100644 --- a/service/task_polling.go +++ b/service/task_polling.go @@ -360,8 +360,11 @@ func updateVideoSingleTask(ctx context.Context, adaptor TaskPollingAdaptor, ch * key = privateData.Key } resp, err := adaptor.FetchTask(baseURL, key, map[string]any{ - "task_id": task.GetUpstreamTaskID(), - "action": task.Action, + "task_id": task.GetUpstreamTaskID(), + "action": task.Action, + "channel_other": ch.Other, + "origin_model_name": task.Properties.OriginModelName, + "upstream_model_name": task.Properties.UpstreamModelName, }, proxy) if err != nil { return fmt.Errorf("fetchTask failed for task %s: %w", taskId, err) diff --git a/setting/operation_setting/quota_error.go b/setting/operation_setting/quota_error.go new file mode 100644 index 000000000000..6f8841beb146 --- /dev/null +++ b/setting/operation_setting/quota_error.go @@ -0,0 +1,45 @@ +package operation_setting + +import "strings" + +// UpstreamQuotaErrorKeywords 用于识别上游"余额/额度不足"类错误文案。 +// 命中后该错误会被视为可重试（换渠道转移），且渠道会进入短暂的 quota 冷却。 +// 可在运营设置中按行配置，新增下游平台时把其真实余额错误文案补充进来。 +var UpstreamQuotaErrorKeywords = []string{ + "insufficient credits", + "insufficient credit", + "insufficient balance", + "not enough credits", + "not enough credit", + "credit balance", + "quota exceeded", + "insufficient_user_quota", + "余额不足", + "额度不足", +} + +func UpstreamQuotaErrorKeywordsToString() string { + return strings.Join(UpstreamQuotaErrorKeywords, "\n") +} + +func UpstreamQuotaErrorKeywordsFromString(s string) { + keywords := []string{} + for _, k := range strings.Split(s, "\n") { + k = strings.ToLower(strings.TrimSpace(k)) + if k != "" { + keywords = append(keywords, k) + } + } + UpstreamQuotaErrorKeywords = keywords +} + +// IsUpstreamQuotaErrorMessage 判断错误文案是否命中余额/额度不足关键词。 +func IsUpstreamQuotaErrorMessage(message string) bool { + message = strings.ToLower(message) + for _, keyword := range UpstreamQuotaErrorKeywords { + if strings.Contains(message, keyword) { + return true + } + } + return false +} diff --git a/setting/ratio_setting/model_ratio.go b/setting/ratio_setting/model_ratio.go index 23fd360e366f..b49950cd8410 100644 --- a/setting/ratio_setting/model_ratio.go +++ b/setting/ratio_setting/model_ratio.go @@ -307,13 +307,104 @@ var defaultModelPrice = map[string]float64{ "mj_upscale": 0.05, "swap_face": 0.05, "mj_upload": 0.05, - "sora-2": 0.3, - "sora-2-pro": 0.5, - "gpt-4o-mini-tts": 0.3, - "veo-3.0-generate-001": 0.4, - "veo-3.0-fast-generate-001": 0.15, - "veo-3.1-generate-preview": 0.4, - "veo-3.1-fast-generate-preview": 0.15, + // 上游中转站采购价（参考）: + // bltcy.ai / ablai.top: veo3.1-fast=$0.2, veo3.1=$0.3, veo3.1-components=$0.3, veo3.1-pro=$1, veo3.1-pro-4k=$13 + // bltcy.ai / ablai.top: veo3.1-fast-components=$0.2, veo3.1-fast-4k=$1.5, veo3.1-4k=$1.5, veo3.1-components-4k=$1.5, veo3.1-fast-components-4k=$1.5 + // bltcy.ai / ablai.top: veo3-pro-frames≈$1, veo3-fast-frames≈$0.2, veo2-fast-frames≈$0.2, veo2-fast-components≈$0.2 + // bltcy.ai / ablai.top: sora-2=$0.3(按次), sora-2-pro=$0.5(按次) + // bltcy.ai: gemini-2.5-flash-image=$0.04, nano-banana=$0.08, nano-banana-hd=$0.12, gemini-3.1-flash-image=$0.1, nano-banana-pro=$0.2, gemini-3-pro-image=$0.2 + // xgapi.top: veo3.1-lite=$0.5, veo3.1-lite-4k=$0.55, sora-2=$0.2(按次,便宜) + // Apexer (www.aiapexers.com): veo2≈$0.2, veo2-fast≈$0.2, veo2-pro≈$0.5, veo3≈$0.3, veo3-fast≈$0.2, veo3-pro≈$1, veo3.1≈$0.3, veo3.1-pro≈$1 + // Qilin API (www.937qq.cn): grok-imagine-1.0-video≈$0.020833333333333332 + // 漫小白 (api.manxiaobai.online): gpt-image-2=$0.03, gpt-image-2-1k=$0.05, gpt-image-2-2k=$0.06, gpt-image-2-4k=$0.07 + // 漫小白 (api.manxiaobai.online): gemini-3-pro-image-preview=$0.125, gemini-3.1-flash-image-preview=$0.1 + // 漫小白 (api.manxiaobai.online): grok-imagine-video=$0.2(10s), grok-imagine-video-1.5-preview=$0.35(必须参考图,10/15s) + // Runway Explore adapter: pricing is estimated by public web-app credit rates; production requests remain async. + "sora-2": 0.4, + "sora-2-pro": 0.6, + "xb-sora2": 0.4, + "ss-sora-2": 0.4, + "openai-sora-2": 0.4, + "sora-2-pro-text-to-video": 0.6, + "sora-2-image-to-video": 0.6, + "sora-2(线路BF)": 0.4, + "sora-2-pro(线路BF)": 0.6, + "je-grok": 0.05, + "grok-video-3(线路W)": 0.05, + "全能视频2.0": 0.4, + "香蕉2(线路V)": 0.2, + "香蕉pro(线路G)": 0.3, + "gr-image-2": 0.3, + "gpt-image-2(线路XF)": 0.3, + "grok-imagine-1.0-video": 0.025, + "grok-imagine-1.0-video-20s": 0.025, + "grok-imagine-1.0-video-30s": 0.025, + "veo2": 0.3, + "veo2-fast": 0.3, + "veo2-pro": 0.6, + "veo2-fast-frames": 0.3, + "veo2-fast-components": 0.3, + "veo3": 0.4, + "veo3-fast": 0.3, + "veo3-pro": 1.5, + "veo3-pro-frames": 1.5, + "veo3-fast-frames": 0.3, + "veo3.1-fast": 0.3, + "veo3.1": 0.4, + "veo3.1-pro": 1.5, + "veo3.1-pro-4k": 15, + "veo3.1-components": 0.4, + "veo3.1-lite": 0.6, + "veo3.1-lite-4k": 0.65, + "veo-3.1": 0.4, + "veo3.1-fast-4k": 1.5, + "veo3.1-4k": 1.5, + "veo3.1-components-4k": 1.5, + "veo3.1-fast-components": 0.3, + "veo3.1-fast-components-4k": 1.5, + "seedance-2": 0.5, + "gen4-turbo": 0.1, + "wan-2.6-flash": 0.1, + "kling-2.5-turbo-standard": 0.15, + "gen4.5": 0.3, + "happyhorse-1": 0.3, + "wan-2.6": 0.2, + "kling-2.5-turbo-pro": 0.25, + "kling-2.6": 0.35, + "wan-2.2-animate": 0.25, + "kling-3.0-pro": 0.25, + "kling-3.0-standard": 0.2, + "kling-3.0-4k": 1.0, + "kling-3.0-motion-control": 0.25, + "kling-o3-pro": 0.3, + "kling-o3-standard": 0.25, + "kling-o3-4k": 1.0, + "kling-2.6-motion-control": 0.35, + "gemini-2.5-flash-image-preview": 0.14, + "gemini-2.5-flash-image": 0.14, + "nano-banana": 0.18, + "nano-banana-hd": 0.22, + "gemini-3.1-flash-image-preview": 0.2, + "gemini_3.0_pro_image_preview": 0.3, + "gemini_3.0_pro_image_preview_4K": 0.35, + "gemini_3.1_flash_image_preview": 0.25, + "gemini_3.1_flash_image_preview_4K": 0.3, + "gpt-image-2": 0.5, + "gpt-image-2-1k": 0.55, + "gpt-image-2-2k": 0.6, + "gpt-image-2-4k": 0.65, + "grok-imagine-video": 0.25, + "grok-imagine-video-1.5-preview": 0.45, + "nano-banana-pro": 0.3, + "gemini-3-pro-image-preview": 0.3, + "gpt-4o-mini-tts": 0.3, + "veo-3.0-generate-001": 0.4, + "veo-3.0-fast-generate-001": 0.15, + "veo-3.1-generate-preview": 0.4, + "veo-3.1-fast-generate-preview": 0.15, + "MiniMax-Hailuo-02": 0.4, + "MiniMax-Hailuo-2.3": 0.4, + "MiniMax-Hailuo-2.3-Fast": 0.3, } var defaultAudioRatio = map[string]float64{ diff --git a/web/classic/src/constants/channel.constants.js b/web/classic/src/constants/channel.constants.js index 9fa78779de8f..0b49c01d9994 100644 --- a/web/classic/src/constants/channel.constants.js +++ b/web/classic/src/constants/channel.constants.js @@ -189,6 +189,16 @@ export const CHANNEL_OPTIONS = [ color: 'blue', label: 'Codex (OpenAI OAuth)', }, + { + value: 58, + color: 'green', + label: 'OpenAI Video', + }, + { + value: 59, + color: 'blue', + label: 'ListenHub', + }, ]; // Channel types that support upstream model list fetching in UI. diff --git a/web/classic/src/index.jsx b/web/classic/src/index.jsx index f24f0ec08983..ba5e12f77d9b 100644 --- a/web/classic/src/index.jsx +++ b/web/classic/src/index.jsx @@ -22,7 +22,6 @@ For commercial licensing, please contact support@quantumnous.com import React from 'react'; import ReactDOM from 'react-dom/client'; import { BrowserRouter } from 'react-router-dom'; -import '@douyinfe/semi-ui/dist/css/semi.css'; import { UserProvider } from './context/User'; import 'react-toastify/dist/ReactToastify.css'; import { StatusProvider } from './context/Status'; diff --git a/web/default/package.json b/web/default/package.json index fa9dfd9c6c09..9cac8f6050eb 100644 --- a/web/default/package.json +++ b/web/default/package.json @@ -82,6 +82,7 @@ "@tanstack/react-router-devtools": "^1.167.0", "@tanstack/router-plugin": "^1.168.11", "@trivago/prettier-plugin-sort-imports": "^6.0.2", + "@types/hast": "^3.0.4", "@types/node": "^25.9.1", "@types/react": "^19.2.15", "@types/react-dom": "^19.2.3", diff --git a/web/default/src/features/channels/components/channels-columns.tsx b/web/default/src/features/channels/components/channels-columns.tsx index 0d79c1d3aeb8..9ca182c66e43 100644 --- a/web/default/src/features/channels/components/channels-columns.tsx +++ b/web/default/src/features/channels/components/channels-columns.tsx @@ -55,9 +55,11 @@ import { getCodexUsage } from '../api' import { CHANNEL_STATUS_CONFIG, MODEL_FETCHABLE_TYPES } from '../constants' import { formatBalance, + formatBalanceWithUnit, formatRelativeTime, formatResponseTime, getBalanceVariant, + parseBalanceMeta, getChannelTypeIcon, getChannelTypeLabel, getResponseTimeConfig, @@ -301,9 +303,39 @@ function BalanceCell({ channel }: { channel: Channel }) { tokenSuffix && value !== '-' ? `${value}${tokenSuffix}` : value const usedDisplay = withSuffix(formatQuotaValue(usedQuota)) - const remainingDisplay = withSuffix(formatBalance(balance)) const usedLabel = `${t('Used:')} ${usedDisplay}` - const remainingLabel = `${t('Remaining:')} ${remainingDisplay}` + + // 下游余额三档（balance / spend_only / console_only），来自后端写入 other_info + const balanceMeta = parseBalanceMeta(channel.other_info) + const balanceKind = balanceMeta?.balance_kind + const balanceUnit = balanceMeta?.balance_unit + + let remainingDisplay = withSuffix(formatBalance(balance)) + let remainingVariant: 'success' | 'warning' | 'danger' | 'neutral' | 'info' = + getBalanceVariant(balance) + let remainingTooltip = `${t('Remaining:')} ${remainingDisplay}` + let updatable = true + if (balanceKind === 'console_only') { + remainingDisplay = t('Console only') + remainingVariant = 'info' + remainingTooltip = t( + 'This upstream has no balance API. Check its web console.' + ) + updatable = false + } else if (balanceKind === 'spend_only') { + const spent = formatBalanceWithUnit(balanceMeta?.balance_used, balanceUnit) + remainingDisplay = `${t('Spent:')} ${spent}` + remainingVariant = 'warning' + remainingTooltip = t( + 'Unlimited-quota key: only cumulative spend is visible, not wallet balance. Configure console credentials in channel settings to fetch the real balance.' + ) + } else if (balanceKind === 'balance') { + const rem = balanceMeta?.balance_remaining ?? balance + remainingDisplay = formatBalanceWithUnit(rem, balanceUnit) + remainingVariant = getBalanceVariant(rem) + remainingTooltip = `${t('Remaining:')} ${remainingDisplay}` + } + const remainingLabel = remainingTooltip // Tag row: only show cumulative used quota if (isTagRow) { @@ -319,10 +351,10 @@ function BalanceCell({ channel }: { channel: Channel }) { } // Regular channel row: show used and remaining with click to update - const variant = getBalanceVariant(balance) const handleClickUpdate = async () => { if (isUpdating) return + if (!updatable) return setIsUpdating(true) if (channel.type === 57) { @@ -383,12 +415,12 @@ function BalanceCell({ channel }: { channel: Channel }) { ? 'info' : isUpdating ? 'neutral' - : variant + : remainingVariant } size='sm' copyable={false} showDot={false} - className='cursor-pointer' + className={updatable ? 'cursor-pointer' : 'cursor-help'} onClick={handleClickUpdate} /> } @@ -399,7 +431,9 @@ function BalanceCell({ channel }: { channel: Channel }) { ? t('Click to view Codex usage') : remainingLabel}

- {channel.type !== 57 &&

{t('Click to update balance')}

} + {channel.type !== 57 && updatable && ( +

{t('Click to update balance')}

+ )} diff --git a/web/default/src/features/channels/components/drawers/channel-mutate-drawer.tsx b/web/default/src/features/channels/components/drawers/channel-mutate-drawer.tsx index 25258d92fdd8..50f006afc6b1 100644 --- a/web/default/src/features/channels/components/drawers/channel-mutate-drawer.tsx +++ b/web/default/src/features/channels/components/drawers/channel-mutate-drawer.tsx @@ -44,6 +44,7 @@ import { Route, Settings, SlidersHorizontal, + Wallet, Wand2, } from 'lucide-react' import { useTranslation } from 'react-i18next' @@ -373,6 +374,7 @@ export function ChannelMutateDrawer({ const currentName = form.watch('name') const currentModelMapping = form.watch('model_mapping') const awsKeyType = form.watch('aws_key_type') + const balanceQueryMode = form.watch('balance_query_mode') const upstreamModelUpdateCheckEnabled = form.watch( 'upstream_model_update_check_enabled' ) @@ -3261,6 +3263,122 @@ export function ChannelMutateDrawer({ )} /> +
+ } + /> + ( + + {t('Balance Query Mode')} + + + {t( + 'Unlimited-quota relay keys can only report cumulative spend. Use console login to fetch the real wallet balance.' + )} + + + )} + /> + {balanceQueryMode === 'newapi_console' && ( + <> + ( + + + {t('Console Username')} + + + + + + )} + /> + ( + + + {t('Console Password')} + + + + + + )} + /> + + )} + ( + + + {t('Total Recharged (optional)')} + + + + field.onChange( + Number(e.target.value) || 0 + ) + } + /> + + + {t( + 'For spend-only upstreams: estimate remaining ≈ recharged − cumulative spend.' + )} + + + )} + /> +
+ {MODEL_FETCHABLE_TYPES.has(currentType) && (
{ diff --git a/web/default/src/features/channels/lib/channel-form.ts b/web/default/src/features/channels/lib/channel-form.ts index 4f0e5042511d..34a821b46bed 100644 --- a/web/default/src/features/channels/lib/channel-form.ts +++ b/web/default/src/features/channels/lib/channel-form.ts @@ -182,6 +182,13 @@ export const channelFormSchema = z pass_through_body_enabled: z.boolean().optional(), system_prompt: z.string().optional(), system_prompt_override: z.boolean().optional(), + // Downstream balance query (stored in setting.balance_query JSON) + balance_query_mode: z + .enum(['', 'auto', 'newapi_console', 'disabled']) + .optional(), + balance_query_username: z.string().optional(), + balance_query_password: z.string().optional(), + balance_query_recharged: z.number().optional(), // Type-specific settings (stored in settings JSON) is_enterprise_account: z.boolean().optional(), // OpenRouter specific vertex_key_type: z.enum(['json', 'api_key']).optional(), // Vertex AI specific @@ -300,6 +307,10 @@ export const CHANNEL_FORM_DEFAULT_VALUES: ChannelFormValues = { pass_through_body_enabled: false, system_prompt: '', system_prompt_override: false, + balance_query_mode: 'auto', + balance_query_username: '', + balance_query_password: '', + balance_query_recharged: 0, // Type-specific settings is_enterprise_account: false, vertex_key_type: 'json', @@ -336,11 +347,16 @@ export function transformChannelToFormDefaults( pass_through_body_enabled: false, system_prompt: '', system_prompt_override: false, + balance_query_mode: 'auto' as '' | 'auto' | 'newapi_console' | 'disabled', + balance_query_username: '', + balance_query_password: '', + balance_query_recharged: 0, } if (channel.setting) { try { const parsed = JSON.parse(channel.setting) + const bq = parsed.balance_query || {} extraSettings = { force_format: parsed.force_format || false, thinking_to_content: parsed.thinking_to_content || false, @@ -348,6 +364,10 @@ export function transformChannelToFormDefaults( pass_through_body_enabled: parsed.pass_through_body_enabled || false, system_prompt: parsed.system_prompt || '', system_prompt_override: parsed.system_prompt_override || false, + balance_query_mode: bq.mode || 'auto', + balance_query_username: bq.username || '', + balance_query_password: bq.password || '', + balance_query_recharged: bq.recharged || 0, } } catch (error) { // eslint-disable-next-line no-console @@ -450,7 +470,7 @@ export function transformChannelToFormDefaults( * Build the setting JSON string from form extra settings */ function buildSettingJSON(formData: ChannelFormValues): string { - const settingObj = { + const settingObj: Record = { force_format: formData.force_format || false, thinking_to_content: formData.thinking_to_content || false, proxy: formData.proxy || '', @@ -458,6 +478,17 @@ function buildSettingJSON(formData: ChannelFormValues): string { system_prompt: formData.system_prompt || '', system_prompt_override: formData.system_prompt_override || false, } + // 仅在非默认（auto）时写入 balance_query，保持 setting 精简 + const mode = formData.balance_query_mode || 'auto' + const recharged = formData.balance_query_recharged || 0 + if (mode !== 'auto' || recharged > 0) { + settingObj.balance_query = { + mode, + username: formData.balance_query_username || '', + password: formData.balance_query_password || '', + recharged, + } + } return JSON.stringify(settingObj) } diff --git a/web/default/src/features/channels/lib/channel-type-config.ts b/web/default/src/features/channels/lib/channel-type-config.ts index 097f942f81ab..7ea2d62d90b8 100644 --- a/web/default/src/features/channels/lib/channel-type-config.ts +++ b/web/default/src/features/channels/lib/channel-type-config.ts @@ -134,6 +134,29 @@ export const CHANNEL_TYPE_CONFIGS: Record = { baseUrl: 'Default: https://api.replicate.com', }, }, + 58: { + id: 58, + name: CHANNEL_TYPES[58], + icon: 'openai', + hints: { + baseUrl: 'e.g., https://api.bltcy.ai, https://api.ablai.top, https://xgapi.top, https://your-domain.com/api/v1', + key: 'API Key from relay provider', + models: 'veo3.1-fast,veo3.1,veo3.1-pro,sora-2,sora-2-pro,xb-sora2,openai-sora-2,sora-2-pro-text-to-video,sora-2-image-to-video', + other: 'OpenAI-compatible video generation relay (Veo/Sora via third-party API; xb-sora2 API uses X-API-Key)', + }, + }, + 59: { + id: 59, + name: CHANNEL_TYPES[59], + icon: 'openai', + defaultBaseUrl: 'https://api.marswave.ai/openapi', + hints: { + baseUrl: 'Default: https://api.marswave.ai/openapi', + key: 'ListenHub API Key', + models: 'gemini-3-pro-image-preview,gemini-3.1-flash-image-preview,gpt-image-2', + other: 'ListenHub image generation; upstream path is /v1/images/generation and this channel exposes /v1/images/generations.', + }, + }, } /** diff --git a/web/default/src/features/channels/lib/channel-utils.ts b/web/default/src/features/channels/lib/channel-utils.ts index 3b55f15eb63c..0c67ccd42d40 100644 --- a/web/default/src/features/channels/lib/channel-utils.ts +++ b/web/default/src/features/channels/lib/channel-utils.ts @@ -101,6 +101,8 @@ export function getChannelTypeIcon(type: number): string { 55: 'OpenAI', // Sora 54: 'Doubao', // DoubaoVideo 56: 'Replicate', // Replicate + 58: 'OpenAI', // OpenAIVideo + 59: 'OpenAI', // ListenHub // Tools & Platforms 37: 'Dify', // Dify @@ -322,6 +324,55 @@ export function getBalanceVariant( return 'success' } +// ============================================================================ +// Downstream balance metadata (written by backend into channel.other_info) +// 三档语义详见 docs/channel-balance-query.md +// ============================================================================ + +export type BalanceKind = 'balance' | 'spend_only' | 'console_only' + +export interface BalanceMeta { + balance_kind?: BalanceKind + balance_unit?: string + balance_used?: number + balance_remaining?: number + balance_provider?: string + balance_checked_time?: number +} + +/** + * Parse balance metadata from a channel's other_info JSON string. + */ +export function parseBalanceMeta( + otherInfo: string | null | undefined +): BalanceMeta | null { + if (!otherInfo) return null + try { + const parsed = JSON.parse(otherInfo) + if (parsed && typeof parsed === 'object' && parsed.balance_kind) { + return parsed as BalanceMeta + } + } catch { + return null + } + return null +} + +/** + * Format a numeric value with the upstream's native unit. + * USD goes through the currency formatter; other units (算力 / credits …) + * are shown as a plain number + unit suffix. + */ +export function formatBalanceWithUnit( + value: number | null | undefined, + unit: string | undefined +): string { + if (value == null || Number.isNaN(value)) return '-' + if (!unit || unit === 'USD') return formatBalance(value) + const rounded = Math.round(value * 10000) / 10000 + return `${rounded} ${unit}` +} + // ============================================================================ // Response Time Utilities // ============================================================================ diff --git a/web/default/src/features/channels/types.ts b/web/default/src/features/channels/types.ts index a282053a3a95..44036aed89a8 100644 --- a/web/default/src/features/channels/types.ts +++ b/web/default/src/features/channels/types.ts @@ -86,6 +86,17 @@ export interface ChannelSettings { pass_through_body_enabled?: boolean system_prompt?: string system_prompt_override?: boolean + balance_query?: BalanceQuerySettings +} + +// 下游余额查询配置，详见 docs/channel-balance-query.md +export interface BalanceQuerySettings { + // ''/'auto' 自动识别；'newapi_console' 账密登录拿真实钱包余额；'disabled' 不查询 + mode?: '' | 'auto' | 'newapi_console' | 'disabled' + username?: string + password?: string + // 已充值累计额（spend_only 档用于估算剩余 ≈ recharged - used） + recharged?: number } export interface ChannelOtherSettings { diff --git a/web/default/src/i18n/locales/zh.json b/web/default/src/i18n/locales/zh.json index 13d9bfa38831..5a438dec38aa 100644 --- a/web/default/src/i18n/locales/zh.json +++ b/web/default/src/i18n/locales/zh.json @@ -1,5 +1,20 @@ { "translation": { + "Console only": "仅控制台", + "This upstream has no balance API. Check its web console.": "该上游没有余额查询接口，请登录其 web 控制台查看。", + "Spent:": "已消费：", + "Unlimited-quota key: only cumulative spend is visible, not wallet balance. Configure console credentials in channel settings to fetch the real balance.": "无限额度 Key：只能看到累计消费，查不到钱包余额。可在渠道设置里配置控制台账密以获取真实余额。", + "Downstream Balance Query": "下游余额查询", + "Balance Query Mode": "余额查询模式", + "Auto (detect by base URL)": "自动（按 Base URL 识别）", + "new-api console login (fetch real wallet balance)": "new-api 控制台登录（获取真实钱包余额）", + "Unlimited-quota relay keys can only report cumulative spend. Use console login to fetch the real wallet balance.": "无限额度中转 Key 只能查到累计消费；用控制台登录可获取真实钱包余额。", + "Console Username": "控制台账号", + "Downstream site account": "下游站点账号", + "Console Password": "控制台密码", + "Downstream site password": "下游站点密码", + "Total Recharged (optional)": "累计充值额（选填）", + "For spend-only upstreams: estimate remaining ≈ recharged − cumulative spend.": "仅消费档上游：估算剩余 ≈ 累计充值 − 累计消费。", "360": "360", "1000": "1000", "10000": "10000",