diff --git a/common/model.go b/common/model.go index 4ebc7b532d74..ba2505a8af3e 100644 --- a/common/model.go +++ b/common/model.go @@ -13,6 +13,7 @@ var ( "dall-e-3", "dall-e-2", "gpt-image-1", + "gpt-image-2", "prefix:imagen-", "flux-", "flux.1-", diff --git a/controller/channel-test.go b/controller/channel-test.go index b225585ed7a3..cf9563cbf802 100644 --- a/controller/channel-test.go +++ b/controller/channel-test.go @@ -43,6 +43,12 @@ type testResult struct { newAPIError *types.NewAPIError } +func isCodexImageGenerationTestModel(channel *model.Channel, modelName string) bool { + return channel != nil && + channel.Type == constant.ChannelTypeCodex && + strings.EqualFold(strings.TrimSpace(modelName), "gpt-image-2") +} + func normalizeChannelTestEndpoint(channel *model.Channel, modelName, endpointType string) string { normalized := strings.TrimSpace(endpointType) if normalized != "" { @@ -51,6 +57,9 @@ func normalizeChannelTestEndpoint(channel *model.Channel, modelName, endpointTyp if strings.HasSuffix(modelName, ratio_setting.CompactModelSuffix) { return string(constant.EndpointTypeOpenAIResponseCompact) } + if isCodexImageGenerationTestModel(channel, modelName) { + return string(constant.EndpointTypeImageGeneration) + } if channel != nil && channel.Type == constant.ChannelTypeCodex { return string(constant.EndpointTypeOpenAIResponse) } @@ -122,6 +131,10 @@ func testChannel(channel *model.Channel, testModel string, endpointType string, requestPath = "/v1/images/generations" } + if isCodexImageGenerationTestModel(channel, testModel) { + requestPath = "/v1/images/generations" + } + // responses-only models if strings.Contains(strings.ToLower(testModel), "codex") { requestPath = "/v1/responses" @@ -773,6 +786,15 @@ func buildTestRequest(model string, endpointType string, channel *model.Channel, } } + if isCodexImageGenerationTestModel(channel, model) { + return &dto.ImageRequest{ + Model: model, + Prompt: "a cute cat", + N: lo.ToPtr(uint(1)), + Size: "1024x1024", + } + } + // Responses-only models (e.g. codex series) if strings.Contains(strings.ToLower(model), "codex") { return &dto.OpenAIResponsesRequest{ diff --git a/dto/openai_image.go b/dto/openai_image.go index 52986fbfd59d..9e426f4a0cf3 100644 --- a/dto/openai_image.go +++ b/dto/openai_image.go @@ -170,9 +170,14 @@ func (i *ImageRequest) SetModelName(modelName string) { } type ImageResponse struct { - Data []ImageData `json:"data"` - Created int64 `json:"created"` - Metadata json.RawMessage `json:"metadata,omitempty"` + Data []ImageData `json:"data"` + Created int64 `json:"created"` + Metadata json.RawMessage `json:"metadata,omitempty"` + Background string `json:"background,omitempty"` + OutputFormat string `json:"output_format,omitempty"` + Quality string `json:"quality,omitempty"` + Size string `json:"size,omitempty"` + Usage *Usage `json:"usage,omitempty"` } type ImageData struct { Url string `json:"url"` diff --git a/dto/openai_response.go b/dto/openai_response.go index 0e6b818dbd8b..ec98240807f4 100644 --- a/dto/openai_response.go +++ b/dto/openai_response.go @@ -338,16 +338,20 @@ type IncompleteDetails struct { } type ResponsesOutput struct { - Type string `json:"type"` - ID string `json:"id"` - Status string `json:"status"` - Role string `json:"role"` - Content []ResponsesOutputContent `json:"content"` - Quality string `json:"quality"` - Size string `json:"size"` - CallId string `json:"call_id,omitempty"` - Name string `json:"name,omitempty"` - Arguments json.RawMessage `json:"arguments,omitempty"` + Type string `json:"type"` + ID string `json:"id"` + Status string `json:"status"` + Role string `json:"role"` + Content []ResponsesOutputContent `json:"content"` + Quality string `json:"quality"` + Size string `json:"size"` + Result string `json:"result,omitempty"` + RevisedPrompt string `json:"revised_prompt,omitempty"` + OutputFormat string `json:"output_format,omitempty"` + Background string `json:"background,omitempty"` + CallId string `json:"call_id,omitempty"` + Name string `json:"name,omitempty"` + Arguments json.RawMessage `json:"arguments,omitempty"` } // ArgumentsString returns function call arguments in the string form expected by Chat Completions. diff --git a/relay/channel/codex/adaptor.go b/relay/channel/codex/adaptor.go index ef4d4fa04125..8a50ada1d8c1 100644 --- a/relay/channel/codex/adaptor.go +++ b/relay/channel/codex/adaptor.go @@ -16,6 +16,8 @@ import ( "github.com/QuantumNous/new-api/types" "github.com/gin-gonic/gin" + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" ) type Adaptor struct { @@ -34,7 +36,16 @@ func (a *Adaptor) ConvertAudioRequest(c *gin.Context, info *relaycommon.RelayInf } func (a *Adaptor) ConvertImageRequest(c *gin.Context, info *relaycommon.RelayInfo, request dto.ImageRequest) (any, error) { - return nil, errors.New("codex channel: endpoint not supported") + relayMode := relayconstant.RelayModeUnknown + if info != nil { + relayMode = info.RelayMode + } + switch relayMode { + case relayconstant.RelayModeImagesGenerations, relayconstant.RelayModeImagesEdits: + return buildCodexImageResponsesRequest(c, info, request) + default: + return nil, errors.New("codex channel: only /v1/images/generations and /v1/images/edits are supported for image requests") + } } func (a *Adaptor) Init(info *relaycommon.RelayInfo) { @@ -52,10 +63,170 @@ func (a *Adaptor) ConvertEmbeddingRequest(c *gin.Context, info *relaycommon.Rela return nil, errors.New("codex channel: /v1/embeddings endpoint not supported") } +func normalizeCodexResponsesInput(raw json.RawMessage) (json.RawMessage, error) { + if common.GetJsonType(raw) != "string" { + return raw, nil + } + var input string + if err := common.Unmarshal(raw, &input); err != nil { + return raw, err + } + return common.Marshal([]map[string]string{{ + "role": "user", + "content": input, + }}) +} + +func normalizeCodexResponsesTools(raw json.RawMessage) (json.RawMessage, error) { + if len(raw) == 0 || common.GetJsonType(raw) != "array" { + return raw, nil + } + var tools []map[string]any + if err := common.Unmarshal(raw, &tools); err != nil { + return raw, err + } + changed := false + for i := range tools { + if common.Interface2String(tools[i]["type"]) != imageGenerationTool { + continue + } + if strings.TrimSpace(common.Interface2String(tools[i]["model"])) == "" { + tools[i]["model"] = CodexImageModel + changed = true + } + } + if !changed { + return raw, nil + } + return common.Marshal(tools) +} + +func buildCodexRawResponsesRequest(c *gin.Context, info *relaycommon.RelayInfo, request dto.OpenAIResponsesRequest, isCompact bool) (json.RawMessage, bool, error) { + if c == nil || c.Request == nil || c.Request.Body == nil { + return nil, false, nil + } + storage, err := common.GetBodyStorage(c) + if err != nil { + return nil, false, err + } + raw, err := storage.Bytes() + if err != nil { + return nil, false, err + } + if len(raw) == 0 || common.GetJsonType(raw) != "object" { + return nil, false, nil + } + + out := string(raw) + if request.Model != "" && gjson.Get(out, "model").String() != request.Model { + out, err = sjson.Set(out, "model", request.Model) + if err != nil { + return nil, false, err + } + } + + input := gjson.Get(out, "input") + if input.Exists() && input.Type == gjson.String { + wrapped := []map[string]string{{ + "role": "user", + "content": input.String(), + }} + out, err = sjson.Set(out, "input", wrapped) + if err != nil { + return nil, false, err + } + } + + if tools := gjson.Get(out, "tools"); tools.Exists() && tools.IsArray() { + normalizedTools, err := normalizeCodexResponsesTools(json.RawMessage(tools.Raw)) + if err != nil { + return nil, false, err + } + if string(normalizedTools) != tools.Raw { + out, err = sjson.SetRaw(out, "tools", string(normalizedTools)) + if err != nil { + return nil, false, err + } + } + } + + if info != nil && info.ChannelMeta != nil && info.ChannelSetting.SystemPrompt != "" { + systemPrompt := info.ChannelSetting.SystemPrompt + instructions := gjson.Get(out, "instructions") + if !instructions.Exists() { + out, err = sjson.Set(out, "instructions", systemPrompt) + if err != nil { + return nil, false, err + } + } else if info.ChannelSetting.SystemPromptOverride { + if instructions.Type == gjson.String { + existing := strings.TrimSpace(instructions.String()) + if existing != "" { + systemPrompt += "\n" + existing + } + } + out, err = sjson.Set(out, "instructions", systemPrompt) + if err != nil { + return nil, false, err + } + } + } else if !gjson.Get(out, "instructions").Exists() { + out, err = sjson.Set(out, "instructions", "") + if err != nil { + return nil, false, err + } + } + + if !isCompact { + out, err = sjson.Set(out, "stream", true) + if err != nil { + return nil, false, err + } + out, err = sjson.Set(out, "store", false) + if err != nil { + return nil, false, err + } + out, err = sjson.Delete(out, "max_output_tokens") + if err != nil { + return nil, false, err + } + out, err = sjson.Delete(out, "temperature") + if err != nil { + return nil, false, err + } + } + + return json.RawMessage(out), true, nil +} + +func IsRawResponsesRequest(request any) bool { + _, ok := request.(json.RawMessage) + return ok +} + func (a *Adaptor) ConvertOpenAIResponsesRequest(c *gin.Context, info *relaycommon.RelayInfo, request dto.OpenAIResponsesRequest) (any, error) { isCompact := info != nil && info.RelayMode == relayconstant.RelayModeResponsesCompact - if info != nil && info.ChannelSetting.SystemPrompt != "" { + if rawRequest, ok, err := buildCodexRawResponsesRequest(c, info, request, isCompact); ok || err != nil { + return rawRequest, err + } + + if len(request.Input) > 0 { + normalizedInput, err := normalizeCodexResponsesInput(request.Input) + if err != nil { + return nil, err + } + request.Input = normalizedInput + } + if len(request.Tools) > 0 { + normalizedTools, err := normalizeCodexResponsesTools(request.Tools) + if err != nil { + return nil, err + } + request.Tools = normalizedTools + } + + if info != nil && info.ChannelMeta != nil && info.ChannelSetting.SystemPrompt != "" { systemPrompt := info.ChannelSetting.SystemPrompt if len(request.Instructions) == 0 { @@ -99,6 +270,7 @@ func (a *Adaptor) ConvertOpenAIResponsesRequest(c *gin.Context, info *relaycommo if isCompact { return request, nil } + request.Stream = common.GetPointer(true) // codex: store must be false request.Store = json.RawMessage("false") // rm max_output_tokens @@ -112,6 +284,13 @@ func (a *Adaptor) DoRequest(c *gin.Context, info *relaycommon.RelayInfo, request } func (a *Adaptor) DoResponse(c *gin.Context, resp *http.Response, info *relaycommon.RelayInfo) (usage any, err *types.NewAPIError) { + if isCodexImageRelayMode(info) { + return handleImageResponse(c, resp, info) + } + + if info == nil { + return nil, types.NewError(errors.New("codex channel: relay info is nil"), types.ErrorCodeInvalidRequest) + } if info.RelayMode != relayconstant.RelayModeResponses && info.RelayMode != relayconstant.RelayModeResponsesCompact { return nil, types.NewError(errors.New("codex channel: endpoint not supported"), types.ErrorCodeInvalidRequest) } @@ -121,9 +300,9 @@ func (a *Adaptor) DoResponse(c *gin.Context, resp *http.Response, info *relaycom } if info.IsStream { - return openai.OaiResponsesStreamHandler(c, info, resp) + return handleResponsesStream(c, resp, info) } - return openai.OaiResponsesHandler(c, info, resp) + return handleResponsesNonStream(c, resp, info) } func (a *Adaptor) GetModelList() []string { @@ -135,8 +314,10 @@ func (a *Adaptor) GetChannelName() string { } func (a *Adaptor) GetRequestURL(info *relaycommon.RelayInfo) (string, error) { - if info.RelayMode != relayconstant.RelayModeResponses && info.RelayMode != relayconstant.RelayModeResponsesCompact { - return "", errors.New("codex channel: only /v1/responses and /v1/responses/compact are supported") + if info.RelayMode != relayconstant.RelayModeResponses && + info.RelayMode != relayconstant.RelayModeResponsesCompact && + !isCodexImageRelayMode(info) { + return "", errors.New("codex channel: only /v1/responses, /v1/responses/compact, /v1/images/generations and /v1/images/edits are supported") } path := "/backend-api/codex/responses" if info.RelayMode == relayconstant.RelayModeResponsesCompact { @@ -182,7 +363,7 @@ func (a *Adaptor) SetupRequestHeader(c *gin.Context, req *http.Header, info *rel // Clients may omit it or include parameters like `application/json; charset=utf-8`, // which can be rejected by the upstream. Force the exact media type. req.Set("Content-Type", "application/json") - if info.IsStream { + if info.IsStream || info.RelayMode == relayconstant.RelayModeResponses || isCodexImageRelayMode(info) { req.Set("Accept", "text/event-stream") } else if req.Get("Accept") == "" { req.Set("Accept", "application/json") diff --git a/relay/channel/codex/adaptor_test.go b/relay/channel/codex/adaptor_test.go new file mode 100644 index 000000000000..cd99179da46f --- /dev/null +++ b/relay/channel/codex/adaptor_test.go @@ -0,0 +1,542 @@ +package codex + +import ( + "bytes" + "context" + "encoding/json" + "io" + "mime/multipart" + "net/http" + "net/http/httptest" + "net/textproto" + "strings" + "testing" + + "github.com/QuantumNous/new-api/common" + appconstant "github.com/QuantumNous/new-api/constant" + "github.com/QuantumNous/new-api/dto" + relaycommon "github.com/QuantumNous/new-api/relay/common" + relayconstant "github.com/QuantumNous/new-api/relay/constant" + "github.com/gin-gonic/gin" + "github.com/tidwall/gjson" +) + +func TestConvertOpenAIResponsesRequest_NormalizesStringInputAndForcesStream(t *testing.T) { + gin.SetMode(gin.TestMode) + adaptor := &Adaptor{} + + converted, err := adaptor.ConvertOpenAIResponsesRequest(nil, &relaycommon.RelayInfo{ + RelayMode: relayconstant.RelayModeResponses, + ChannelMeta: &relaycommon.ChannelMeta{}, + }, dto.OpenAIResponsesRequest{ + Model: "gpt-5.4", + Input: json.RawMessage(`"hello"`), + }) + if err != nil { + t.Fatalf("ConvertOpenAIResponsesRequest returned error: %v", err) + } + + request := converted.(dto.OpenAIResponsesRequest) + if request.Stream == nil || !*request.Stream { + t.Fatalf("expected stream=true, got %#v", request.Stream) + } + + var input []map[string]string + if err := common.Unmarshal(request.Input, &input); err != nil { + t.Fatalf("input is not a message list: %v", err) + } + if len(input) != 1 || input[0]["role"] != "user" || input[0]["content"] != "hello" { + t.Fatalf("unexpected normalized input: %#v", input) + } +} + +func TestConvertOpenAIResponsesRequest_ArrayInputUnchanged(t *testing.T) { + adaptor := &Adaptor{} + rawInput := json.RawMessage(`[{"role":"user","content":"hi"}]`) + + converted, err := adaptor.ConvertOpenAIResponsesRequest(nil, &relaycommon.RelayInfo{ + RelayMode: relayconstant.RelayModeResponses, + ChannelMeta: &relaycommon.ChannelMeta{}, + }, dto.OpenAIResponsesRequest{ + Model: "gpt-5.4", + Input: rawInput, + }) + if err != nil { + t.Fatalf("ConvertOpenAIResponsesRequest returned error: %v", err) + } + + request := converted.(dto.OpenAIResponsesRequest) + if string(request.Input) != string(rawInput) { + t.Fatalf("array input changed: got %s want %s", request.Input, rawInput) + } +} + +func TestConvertOpenAIResponsesRequest_RawPassthroughPreservesCodexInputItems(t *testing.T) { + gin.SetMode(gin.TestMode) + adaptor := &Adaptor{} + rawBody := []byte(`{ + "model":"gpt-5.4", + "input":[ + {"type":"tool_search_call","call_id":"search-1","execution":"client","arguments":{"query":"calendar"}}, + {"type":"function_call","name":"write_file","call_id":"call-1","arguments":"{\"path\":\"a.txt\"}"} + ], + "tools":[{"type":"image_generation","output_format":"png"}], + "client_metadata":{"x-codex-installation-id":"install-1"}, + "max_output_tokens":100, + "temperature":0.7 + }`) + var request dto.OpenAIResponsesRequest + if err := common.Unmarshal(rawBody, &request); err != nil { + t.Fatalf("request unmarshal failed: %v", err) + } + + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", bytes.NewReader(rawBody)) + c.Request.Header.Set("Content-Type", "application/json") + + converted, err := adaptor.ConvertOpenAIResponsesRequest(c, &relaycommon.RelayInfo{ + RelayMode: relayconstant.RelayModeResponses, + ChannelMeta: &relaycommon.ChannelMeta{}, + }, request) + if err != nil { + t.Fatalf("ConvertOpenAIResponsesRequest returned error: %v", err) + } + + raw, ok := converted.(json.RawMessage) + if !ok { + t.Fatalf("expected raw passthrough request, got %T", converted) + } + var out map[string]any + if err := common.Unmarshal(raw, &out); err != nil { + t.Fatalf("converted raw request unmarshal failed: %v", err) + } + input := out["input"].([]any) + toolSearch := input[0].(map[string]any) + if _, ok := toolSearch["arguments"].(map[string]any); !ok { + t.Fatalf("tool_search_call arguments should remain object: %#v", toolSearch["arguments"]) + } + functionCall := input[1].(map[string]any) + if _, ok := functionCall["arguments"].(string); !ok { + t.Fatalf("function_call arguments should remain string: %#v", functionCall["arguments"]) + } + if _, ok := out["client_metadata"].(map[string]any); !ok { + t.Fatalf("client_metadata should be preserved: %#v", out["client_metadata"]) + } + if out["stream"] != true || out["store"] != false { + t.Fatalf("codex stream/store defaults not applied: stream=%#v store=%#v", out["stream"], out["store"]) + } + if _, ok := out["max_output_tokens"]; ok { + t.Fatalf("max_output_tokens should be removed: %#v", out["max_output_tokens"]) + } + if _, ok := out["temperature"]; ok { + t.Fatalf("temperature should be removed: %#v", out["temperature"]) + } + tools := out["tools"].([]any) + imageTool := tools[0].(map[string]any) + if imageTool["model"] != CodexImageModel { + t.Fatalf("image_generation tool model was not defaulted: %#v", imageTool) + } +} + +func TestConvertOpenAIResponsesRequest_RawPassthroughPreservesSystemPromptSetting(t *testing.T) { + gin.SetMode(gin.TestMode) + adaptor := &Adaptor{} + rawBody := []byte(`{"model":"gpt-5.4","input":[{"role":"user","content":"hi"}],"instructions":"base"}`) + var request dto.OpenAIResponsesRequest + if err := common.Unmarshal(rawBody, &request); err != nil { + t.Fatalf("request unmarshal failed: %v", err) + } + + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", bytes.NewReader(rawBody)) + c.Request.Header.Set("Content-Type", "application/json") + + converted, err := adaptor.ConvertOpenAIResponsesRequest(c, &relaycommon.RelayInfo{ + RelayMode: relayconstant.RelayModeResponses, + ChannelMeta: &relaycommon.ChannelMeta{ + ChannelSetting: dto.ChannelSettings{ + SystemPrompt: "system", + SystemPromptOverride: true, + }, + }, + }, request) + if err != nil { + t.Fatalf("ConvertOpenAIResponsesRequest returned error: %v", err) + } + + raw, ok := converted.(json.RawMessage) + if !ok { + t.Fatalf("expected raw passthrough request, got %T", converted) + } + if got := gjson.GetBytes(raw, "instructions").String(); got != "system\nbase" { + t.Fatalf("unexpected instructions: %q", got) + } +} + +func TestConvertOpenAIResponsesRequest_DefaultsImageGenerationToolModel(t *testing.T) { + adaptor := &Adaptor{} + + converted, err := adaptor.ConvertOpenAIResponsesRequest(nil, &relaycommon.RelayInfo{ + RelayMode: relayconstant.RelayModeResponses, + ChannelMeta: &relaycommon.ChannelMeta{}, + }, dto.OpenAIResponsesRequest{ + Model: "gpt-5.4", + Input: json.RawMessage(`[{"role":"user","content":"hi"}]`), + Tools: json.RawMessage(`[{"type":"image_generation","size":"1024x1024"}]`), + }) + if err != nil { + t.Fatalf("ConvertOpenAIResponsesRequest returned error: %v", err) + } + + request := converted.(dto.OpenAIResponsesRequest) + var tools []map[string]any + if err := common.Unmarshal(request.Tools, &tools); err != nil { + t.Fatalf("tools unmarshal failed: %v", err) + } + if tools[0]["model"] != CodexImageModel { + t.Fatalf("image_generation tool model was not defaulted: %#v", tools[0]) + } +} + +func TestConvertOpenAIResponsesRequest_CompactDoesNotForceStream(t *testing.T) { + adaptor := &Adaptor{} + stream := false + + converted, err := adaptor.ConvertOpenAIResponsesRequest(nil, &relaycommon.RelayInfo{ + RelayMode: relayconstant.RelayModeResponsesCompact, + ChannelMeta: &relaycommon.ChannelMeta{}, + }, dto.OpenAIResponsesRequest{ + Model: "gpt-5.4", + Input: json.RawMessage(`[{"role":"user","content":"hi"}]`), + Stream: &stream, + }) + if err != nil { + t.Fatalf("ConvertOpenAIResponsesRequest returned error: %v", err) + } + + request := converted.(dto.OpenAIResponsesRequest) + if request.Stream == nil || *request.Stream { + t.Fatalf("compact stream should remain false, got %#v", request.Stream) + } +} + +func TestHandleResponsesNonStream_AggregatesCodexSSE(t *testing.T) { + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) + + body := `data: {"type":"response.completed","response":{"id":"resp_1","object":"response","created_at":1,"model":"gpt-5.4","output":[],"usage":{"input_tokens":2,"output_tokens":3,"total_tokens":5}}}` + "\n\n" + + "data: [DONE]\n\n" + resp := &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"text/event-stream"}}, + Body: io.NopCloser(strings.NewReader(body)), + } + + usage, err := handleResponsesNonStream(c, resp, &relaycommon.RelayInfo{}) + if err != nil { + t.Fatalf("handleResponsesNonStream returned error: %v", err) + } + if usage.PromptTokens != 2 || usage.CompletionTokens != 3 || usage.TotalTokens != 5 { + t.Fatalf("unexpected usage: %#v", usage) + } + if !strings.Contains(w.Body.String(), `"id":"resp_1"`) { + t.Fatalf("response body does not contain completed response: %s", w.Body.String()) + } + if contentType := w.Header().Get("Content-Type"); !strings.Contains(contentType, "application/json") { + t.Fatalf("expected application/json content type, got %q", contentType) + } +} + +func TestHandleResponsesNonStream_PreservesImageGenerationOutputItemDone(t *testing.T) { + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) + + body := `data: {"type":"response.output_item.done","item":{"type":"image_generation_call","status":"generating","result":"aGVsbG8=","output_format":"png","size":"1024x1024"}}` + "\n\n" + + `data: {"type":"response.completed","response":{"id":"resp_1","object":"response","created_at":1,"model":"gpt-5.4","output":[],"tool_usage":{"image_gen":{"input_tokens":4,"output_tokens":5,"total_tokens":9}}}}` + "\n\n" + resp := &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"text/event-stream"}}, + Body: io.NopCloser(strings.NewReader(body)), + } + + usage, err := handleResponsesNonStream(c, resp, &relaycommon.RelayInfo{}) + if err != nil { + t.Fatalf("handleResponsesNonStream returned error: %v", err) + } + if usage.PromptTokens != 4 || usage.CompletionTokens != 5 || usage.TotalTokens != 9 { + t.Fatalf("unexpected usage from tool_usage.image_gen: %#v", usage) + } + if !c.GetBool("image_generation_call") { + t.Fatalf("expected image_generation_call context marker") + } + if !strings.Contains(w.Body.String(), `"type":"image_generation_call"`) || !strings.Contains(w.Body.String(), `"result":"aGVsbG8="`) { + t.Fatalf("response body does not preserve image output item: %s", w.Body.String()) + } +} + +func TestHandleResponsesNonStream_PreservesTextOutputItemDoneWhenCompletedOutputEmpty(t *testing.T) { + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodPost, "/backend-api/codex/responses", nil) + + body := `data: {"type":"response.output_item.done","item":{"id":"msg_1","type":"message","status":"completed","role":"assistant","content":[{"type":"output_text","text":"OK","annotations":[]}]}}` + "\n\n" + + `data: {"type":"response.completed","response":{"id":"resp_1","object":"response","created_at":1,"model":"gpt-5.4","output":[],"usage":{"input_tokens":2,"output_tokens":1,"total_tokens":3}}}` + "\n\n" + resp := &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"text/event-stream"}}, + Body: io.NopCloser(strings.NewReader(body)), + } + + usage, err := handleResponsesNonStream(c, resp, &relaycommon.RelayInfo{}) + if err != nil { + t.Fatalf("handleResponsesNonStream returned error: %v", err) + } + if usage.PromptTokens != 2 || usage.CompletionTokens != 1 || usage.TotalTokens != 3 { + t.Fatalf("unexpected usage: %#v", usage) + } + if !strings.Contains(w.Body.String(), `"type":"message"`) || !strings.Contains(w.Body.String(), `"text":"OK"`) { + t.Fatalf("response body does not preserve text output item: %s", w.Body.String()) + } +} + +func TestHandleResponsesStream_MarksNativeImageGenerationTool(t *testing.T) { + oldStreamingTimeout := appconstant.StreamingTimeout + appconstant.StreamingTimeout = 30 + defer func() { + appconstant.StreamingTimeout = oldStreamingTimeout + }() + + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodPost, "/backend-api/codex/responses", nil) + + body := `data: {"type":"response.output_item.done","item":{"type":"image_generation_call","status":"generating","result":"aGVsbG8=","output_format":"png","size":"1024x1024"}}` + "\n\n" + + `data: {"type":"response.completed","response":{"id":"resp_1","object":"response","created_at":1,"model":"gpt-5.4","output":[],"tool_usage":{"image_gen":{"input_tokens":4,"output_tokens":5,"total_tokens":9}}}}` + "\n\n" + + "data: [DONE]\n\n" + resp := &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"text/event-stream"}}, + Body: io.NopCloser(strings.NewReader(body)), + } + info := &relaycommon.RelayInfo{ + RelayMode: relayconstant.RelayModeResponses, + ChannelMeta: &relaycommon.ChannelMeta{UpstreamModelName: "gpt-5.4"}, + ResponsesUsageInfo: &relaycommon.ResponsesUsageInfo{ + BuiltInTools: map[string]*relaycommon.BuildInToolInfo{ + imageGenerationTool: {ToolName: imageGenerationTool}, + }, + }, + } + + usage, err := handleResponsesStream(c, resp, info) + if err != nil { + t.Fatalf("handleResponsesStream returned error: %v", err) + } + if usage.PromptTokens != 4 || usage.CompletionTokens != 5 || usage.TotalTokens != 9 { + t.Fatalf("unexpected usage from tool_usage.image_gen: %#v", usage) + } + if !c.GetBool("image_generation_call") { + t.Fatalf("expected image_generation_call context marker") + } + if info.ResponsesUsageInfo.BuiltInTools[imageGenerationTool].CallCount != 1 { + t.Fatalf("expected image_generation tool call count to be recorded, got %#v", info.ResponsesUsageInfo.BuiltInTools[imageGenerationTool]) + } + if !strings.Contains(w.Body.String(), `"type":"image_generation_call"`) || !strings.Contains(w.Body.String(), `"result":"aGVsbG8="`) { + t.Fatalf("stream body does not preserve native image event: %s", w.Body.String()) + } +} + +func TestRelayErrorHandlerPlainText(t *testing.T) { + resp := &http.Response{ + StatusCode: http.StatusInternalServerError, + Status: "500 Internal Server Error", + Body: io.NopCloser(strings.NewReader("error upstream broke")), + } + + err := RelayErrorHandler(context.Background(), resp) + if err == nil { + t.Fatal("expected error") + } + if strings.Contains(err.Error(), "invalid character") { + t.Fatalf("error should not be JSON parse error: %s", err.Error()) + } + if !strings.Contains(err.Error(), "error upstream broke") { + t.Fatalf("plain text body was not preserved: %s", err.Error()) + } +} + +func TestBuildCodexImageGenerationResponsesRequest(t *testing.T) { + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + + request, err := buildCodexImageResponsesRequest(c, &relaycommon.RelayInfo{ + RelayMode: relayconstant.RelayModeImagesGenerations, + }, dto.ImageRequest{ + Model: CodexImageModel, + Prompt: "中文海报", + Size: "1024x1024", + ResponseFormat: "b64_json", + }) + if err != nil { + t.Fatalf("buildCodexImageResponsesRequest returned error: %v", err) + } + if request.Model != defaultImagesMainModel { + t.Fatalf("unexpected main model: %s", request.Model) + } + if request.Stream == nil || !*request.Stream { + t.Fatalf("expected upstream stream=true") + } + + var tools []map[string]any + if err := common.Unmarshal(request.Tools, &tools); err != nil { + t.Fatalf("tools unmarshal failed: %v", err) + } + if tools[0]["type"] != "image_generation" || tools[0]["action"] != "generate" || tools[0]["model"] != CodexImageModel { + t.Fatalf("unexpected image tool: %#v", tools[0]) + } +} + +func TestBuildCodexImageEditResponsesRequestMultipart(t *testing.T) { + var body bytes.Buffer + writer := multipart.NewWriter(&body) + _ = writer.WriteField("model", CodexImageModel) + _ = writer.WriteField("prompt", "把图片改成中文海报") + _ = writer.WriteField("response_format", "url") + _ = writer.WriteField("size", "1024x1024") + _ = writer.WriteField("input_fidelity", "high") + header := textproto.MIMEHeader{} + header.Set("Content-Disposition", `form-data; name="image"; filename="source.png"`) + header.Set("Content-Type", "image/png") + part, err := writer.CreatePart(header) + if err != nil { + t.Fatalf("CreatePart failed: %v", err) + } + _, _ = part.Write([]byte("pngdata")) + _ = writer.Close() + + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/images/edits", &body) + c.Request.Header.Set("Content-Type", writer.FormDataContentType()) + + request, err := buildCodexImageResponsesRequest(c, &relaycommon.RelayInfo{ + RelayMode: relayconstant.RelayModeImagesEdits, + }, dto.ImageRequest{ + Model: CodexImageModel, + Prompt: "把图片改成中文海报", + }) + if err != nil { + t.Fatalf("buildCodexImageResponsesRequest returned error: %v", err) + } + if !strings.Contains(string(request.Input), "data:image/png;base64,") { + t.Fatalf("input does not include multipart image data URL: %s", request.Input) + } + + var tools []map[string]any + if err := common.Unmarshal(request.Tools, &tools); err != nil { + t.Fatalf("tools unmarshal failed: %v", err) + } + if tools[0]["action"] != "edit" || tools[0]["input_fidelity"] != "high" { + t.Fatalf("unexpected edit tool: %#v", tools[0]) + } + if c.GetString(ginKeyCodexImageResponseFormat) != "url" { + t.Fatalf("response_format was not captured") + } +} + +func TestBuildCodexImageEditResponsesRequestDetectsOctetStreamImage(t *testing.T) { + var body bytes.Buffer + writer := multipart.NewWriter(&body) + _ = writer.WriteField("model", CodexImageModel) + _ = writer.WriteField("prompt", "把图片改成中文海报") + header := textproto.MIMEHeader{} + header.Set("Content-Disposition", `form-data; name="image"; filename="source.png"`) + header.Set("Content-Type", "application/octet-stream") + part, err := writer.CreatePart(header) + if err != nil { + t.Fatalf("CreatePart failed: %v", err) + } + _, _ = part.Write([]byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n', 0, 0, 0, 0}) + _ = writer.Close() + + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/images/edits", &body) + c.Request.Header.Set("Content-Type", writer.FormDataContentType()) + + request, err := buildCodexImageResponsesRequest(c, &relaycommon.RelayInfo{ + RelayMode: relayconstant.RelayModeImagesEdits, + }, dto.ImageRequest{ + Model: CodexImageModel, + Prompt: "把图片改成中文海报", + }) + if err != nil { + t.Fatalf("buildCodexImageResponsesRequest returned error: %v", err) + } + if !strings.Contains(string(request.Input), "data:image/png;base64,") { + t.Fatalf("octet-stream upload was not detected as png: %s", request.Input) + } +} + +func TestHandleImageResponseSSE(t *testing.T) { + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/images/generations", nil) + c.Set(ginKeyCodexImageResponseFormat, "b64_json") + + body := `data: {"type":"response.completed","response":{"created_at":123,"output":[{"type":"image_generation_call","result":"aGVsbG8=","output_format":"png","revised_prompt":"poster"}],"usage":{"input_tokens":1,"output_tokens":2,"total_tokens":3}}}` + "\n\n" + resp := &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"text/event-stream"}}, + Body: io.NopCloser(strings.NewReader(body)), + } + + usage, err := handleImageResponse(c, resp, &relaycommon.RelayInfo{RelayMode: relayconstant.RelayModeImagesGenerations}) + if err != nil { + t.Fatalf("handleImageResponse returned error: %v", err) + } + if usage.TotalTokens != 3 { + t.Fatalf("unexpected usage: %#v", usage) + } + if !strings.Contains(w.Body.String(), `"b64_json":"aGVsbG8="`) { + t.Fatalf("unexpected image response body: %s", w.Body.String()) + } +} + +func TestHandleImageResponseSSEUsesOutputItemDoneWhenCompletedOutputEmpty(t *testing.T) { + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/images/generations", nil) + c.Set(ginKeyCodexImageResponseFormat, "b64_json") + + body := `data: {"type":"response.image_generation_call.partial_image","partial_image_b64":"cGFydGlhbA==","output_format":"png","partial_image_index":0}` + "\n\n" + + `data: {"type":"response.output_item.done","item":{"type":"image_generation_call","status":"generating","result":"aGVsbG8=","output_format":"png","size":"1024x1024","revised_prompt":"poster"}}` + "\n\n" + + `data: {"type":"response.completed","response":{"created_at":123,"output":[],"tool_usage":{"image_gen":{"input_tokens":4,"output_tokens":5,"total_tokens":9}}}}` + "\n\n" + resp := &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"text/event-stream"}}, + Body: io.NopCloser(strings.NewReader(body)), + } + + usage, err := handleImageResponse(c, resp, &relaycommon.RelayInfo{RelayMode: relayconstant.RelayModeImagesGenerations}) + if err != nil { + t.Fatalf("handleImageResponse returned error: %v", err) + } + if usage.PromptTokens != 4 || usage.CompletionTokens != 5 || usage.TotalTokens != 9 { + t.Fatalf("unexpected usage from tool_usage.image_gen: %#v", usage) + } + if !strings.Contains(w.Body.String(), `"b64_json":"aGVsbG8="`) { + t.Fatalf("unexpected image response body: %s", w.Body.String()) + } + var imageResponse dto.ImageResponse + if err := common.Unmarshal(w.Body.Bytes(), &imageResponse); err != nil { + t.Fatalf("image response unmarshal failed: %v", err) + } + if len(imageResponse.Data) != 1 { + t.Fatalf("expected only final output_item.done image, got %d items: %s", len(imageResponse.Data), w.Body.String()) + } +} diff --git a/relay/channel/codex/constants.go b/relay/channel/codex/constants.go index 5233393eaeec..7cf5ad23bd9c 100644 --- a/relay/channel/codex/constants.go +++ b/relay/channel/codex/constants.go @@ -12,7 +12,17 @@ var baseModelList = []string{ "gpt-5.4", } -var ModelList = withCompactModelSuffix(baseModelList) +const ( + CodexImageModel = "gpt-image-2" + defaultImagesMainModel = "gpt-5.4-mini" + imageGenerationTool = "image_generation" +) + +var builtinModelList = []string{ + CodexImageModel, +} + +var ModelList = lo.Uniq(append(withCompactModelSuffix(baseModelList), builtinModelList...)) const ChannelName = "codex" diff --git a/relay/channel/codex/error.go b/relay/channel/codex/error.go new file mode 100644 index 000000000000..710b385dc7a1 --- /dev/null +++ b/relay/channel/codex/error.go @@ -0,0 +1,94 @@ +package codex + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + + "github.com/QuantumNous/new-api/service" + "github.com/QuantumNous/new-api/types" + "github.com/tidwall/gjson" +) + +const maxErrorBodyPreview = 2048 + +func RelayErrorHandler(ctx context.Context, resp *http.Response) *types.NewAPIError { + statusCode := http.StatusInternalServerError + statusText := http.StatusText(statusCode) + if resp != nil { + statusCode = resp.StatusCode + statusText = resp.Status + if strings.TrimSpace(statusText) == "" { + statusText = http.StatusText(statusCode) + } + } + + var responseBody []byte + if resp != nil && resp.Body != nil { + body, err := io.ReadAll(resp.Body) + if err == nil { + responseBody = body + } + service.CloseResponseBodyGracefully(resp) + } + + message := extractCodexErrorMessage(responseBody) + if message == "" { + message = strings.TrimSpace(string(responseBody)) + } + message = truncateErrorMessage(message) + + if message == "" { + message = fmt.Sprintf("codex upstream error: status %d %s", statusCode, statusText) + } else { + message = fmt.Sprintf("codex upstream error: status %d %s: %s", statusCode, statusText, message) + } + + return types.NewOpenAIError(fmt.Errorf("%s", message), types.ErrorCodeBadResponseStatusCode, statusCode) +} + +func extractCodexErrorMessage(body []byte) string { + trimmed := strings.TrimSpace(string(body)) + if trimmed == "" || (trimmed[0] != '{' && trimmed[0] != '[') { + return "" + } + + paths := []string{ + "error.message", + "message", + "msg", + "err", + "error_msg", + "detail", + "header.message", + "response.error.message", + "error", + } + for _, path := range paths { + result := gjson.GetBytes(body, path) + if !result.Exists() || result.Type == gjson.Null { + continue + } + switch result.Type { + case gjson.String, gjson.Number, gjson.True, gjson.False: + if msg := strings.TrimSpace(result.String()); msg != "" { + return msg + } + default: + if msg := strings.TrimSpace(result.Raw); msg != "" { + return msg + } + } + } + return "" +} + +func truncateErrorMessage(message string) string { + message = strings.TrimSpace(message) + if len(message) <= maxErrorBodyPreview { + return message + } + return message[:maxErrorBodyPreview] + "...(truncated)" +} diff --git a/relay/channel/codex/image.go b/relay/channel/codex/image.go new file mode 100644 index 000000000000..b27c16b62486 --- /dev/null +++ b/relay/channel/codex/image.go @@ -0,0 +1,572 @@ +package codex + +import ( + "bufio" + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "strconv" + "strings" + "time" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/dto" + relaycommon "github.com/QuantumNous/new-api/relay/common" + relayconstant "github.com/QuantumNous/new-api/relay/constant" + "github.com/QuantumNous/new-api/service" + "github.com/QuantumNous/new-api/types" + "github.com/gin-gonic/gin" + "github.com/tidwall/gjson" +) + +const ginKeyCodexImageResponseFormat = "codex_image_response_format" + +type imageCallResult struct { + Result string + RevisedPrompt string + OutputFormat string + Size string + Background string + Quality string +} + +func isCodexImageRelayMode(info *relaycommon.RelayInfo) bool { + if info == nil { + return false + } + return info.RelayMode == relayconstant.RelayModeImagesGenerations || + info.RelayMode == relayconstant.RelayModeImagesEdits +} + +func buildCodexImageResponsesRequest(c *gin.Context, info *relaycommon.RelayInfo, request dto.ImageRequest) (dto.OpenAIResponsesRequest, error) { + action := "generate" + var images []string + var mask string + var form *multipart.Form + + if info != nil && info.RelayMode == relayconstant.RelayModeImagesEdits { + action = "edit" + var err error + images, mask, form, err = collectImageEditInputs(c, request) + if err != nil { + return dto.OpenAIResponsesRequest{}, err + } + if len(images) == 0 { + return dto.OpenAIResponsesRequest{}, fmt.Errorf("image is required") + } + } + + imageModel := strings.TrimSpace(request.Model) + if imageModel == "" { + imageModel = CodexImageModel + } + + responseFormat := imageResponseFormat(request, form) + if responseFormat == "" { + responseFormat = "b64_json" + } + c.Set(ginKeyCodexImageResponseFormat, responseFormat) + + tool := map[string]any{ + "type": "image_generation", + "action": action, + "model": imageModel, + } + applyImageToolOptions(tool, request, form) + if mask != "" { + tool["input_image_mask"] = map[string]any{ + "image_url": mask, + } + } + + content := make([]map[string]any, 0, len(images)+1) + content = append(content, map[string]any{ + "type": "input_text", + "text": request.Prompt, + }) + for _, image := range images { + image = strings.TrimSpace(image) + if image == "" { + continue + } + content = append(content, map[string]any{ + "type": "input_image", + "image_url": image, + }) + } + + input := []map[string]any{{ + "type": "message", + "role": "user", + "content": content, + }} + inputRaw, err := common.Marshal(input) + if err != nil { + return dto.OpenAIResponsesRequest{}, err + } + toolsRaw, err := common.Marshal([]map[string]any{tool}) + if err != nil { + return dto.OpenAIResponsesRequest{}, err + } + includeRaw, err := common.Marshal([]string{"reasoning.encrypted_content"}) + if err != nil { + return dto.OpenAIResponsesRequest{}, err + } + toolChoiceRaw, err := common.Marshal(map[string]string{"type": "image_generation"}) + if err != nil { + return dto.OpenAIResponsesRequest{}, err + } + + return dto.OpenAIResponsesRequest{ + Model: defaultImagesMainModel, + Input: inputRaw, + Instructions: json.RawMessage(`""`), + Include: includeRaw, + ParallelToolCalls: json.RawMessage(`true`), + Reasoning: &dto.Reasoning{ + Effort: "medium", + Summary: "auto", + }, + Store: json.RawMessage(`false`), + Stream: common.GetPointer(true), + ToolChoice: toolChoiceRaw, + Tools: toolsRaw, + }, nil +} + +func collectImageEditInputs(c *gin.Context, request dto.ImageRequest) ([]string, string, *multipart.Form, error) { + contentType := "" + if c != nil && c.Request != nil { + contentType = strings.ToLower(c.Request.Header.Get("Content-Type")) + } + if strings.Contains(contentType, "multipart/form-data") || contentType == "" { + form, err := common.ParseMultipartFormReusable(c) + if err != nil { + return nil, "", nil, fmt.Errorf("failed to parse image edit form request: %w", err) + } + images, err := multipartFilesToDataURLs(collectMultipartImageFiles(form)) + if err != nil { + return nil, "", nil, err + } + mask := "" + if maskFiles := form.File["mask"]; len(maskFiles) > 0 { + mask, err = multipartFileToDataURL(maskFiles[0]) + if err != nil { + return nil, "", nil, err + } + } + return images, mask, form, nil + } + + images, mask, err := jsonImageEditInputs(request) + return images, mask, nil, err +} + +func collectMultipartImageFiles(form *multipart.Form) []*multipart.FileHeader { + if form == nil || form.File == nil { + return nil + } + if files := form.File["image[]"]; len(files) > 0 { + return files + } + if files := form.File["image"]; len(files) > 0 { + return files + } + + var imageFiles []*multipart.FileHeader + for fieldName, files := range form.File { + if strings.HasPrefix(fieldName, "image[") && len(files) > 0 { + imageFiles = append(imageFiles, files...) + } + } + return imageFiles +} + +func multipartFilesToDataURLs(fileHeaders []*multipart.FileHeader) ([]string, error) { + images := make([]string, 0, len(fileHeaders)) + for _, fileHeader := range fileHeaders { + dataURL, err := multipartFileToDataURL(fileHeader) + if err != nil { + return nil, err + } + images = append(images, dataURL) + } + return images, nil +} + +func multipartFileToDataURL(fileHeader *multipart.FileHeader) (string, error) { + if fileHeader == nil { + return "", fmt.Errorf("upload file is nil") + } + file, err := fileHeader.Open() + if err != nil { + return "", fmt.Errorf("open upload file failed: %w", err) + } + defer file.Close() + + data, err := io.ReadAll(file) + if err != nil { + return "", fmt.Errorf("read upload file failed: %w", err) + } + mediaType := strings.TrimSpace(fileHeader.Header.Get("Content-Type")) + if mediaType == "" || strings.EqualFold(mediaType, "application/octet-stream") || !strings.HasPrefix(strings.ToLower(mediaType), "image/") { + detected := http.DetectContentType(data) + if strings.HasPrefix(strings.ToLower(detected), "image/") { + mediaType = detected + } + } + if mediaType == "" { + mediaType = "application/octet-stream" + } + return "data:" + mediaType + ";base64," + base64.StdEncoding.EncodeToString(data), nil +} + +func jsonImageEditInputs(request dto.ImageRequest) ([]string, string, error) { + var images []string + if raw, ok := request.Extra["images"]; ok && len(raw) > 0 { + var items []struct { + ImageURL string `json:"image_url"` + FileID string `json:"file_id"` + } + if err := common.Unmarshal(raw, &items); err != nil { + return nil, "", err + } + for _, item := range items { + if strings.TrimSpace(item.ImageURL) != "" { + images = append(images, strings.TrimSpace(item.ImageURL)) + } + } + } + if len(images) == 0 && len(request.Image) > 0 { + var image string + if err := common.Unmarshal(request.Image, &image); err == nil && strings.TrimSpace(image) != "" { + images = append(images, strings.TrimSpace(image)) + } + } + + mask := "" + if raw, ok := request.Extra["mask"]; ok && len(raw) > 0 { + var maskObj struct { + ImageURL string `json:"image_url"` + FileID string `json:"file_id"` + } + if err := common.Unmarshal(raw, &maskObj); err != nil { + return nil, "", err + } + mask = strings.TrimSpace(maskObj.ImageURL) + } + return images, mask, nil +} + +func imageResponseFormat(request dto.ImageRequest, form *multipart.Form) string { + if form != nil { + if value := firstFormValue(form, "response_format"); value != "" { + return value + } + } + return strings.TrimSpace(request.ResponseFormat) +} + +func applyImageToolOptions(tool map[string]any, request dto.ImageRequest, form *multipart.Form) { + setStringToolOption(tool, "size", firstNonEmpty(request.Size, firstFormValue(form, "size"))) + setStringToolOption(tool, "quality", firstNonEmpty(request.Quality, firstFormValue(form, "quality"))) + setRawOrFormStringToolOption(tool, "background", request.Background, form) + setRawOrFormStringToolOption(tool, "output_format", request.OutputFormat, form) + setRawOrFormStringToolOption(tool, "moderation", request.Moderation, form) + setRawOrFormIntToolOption(tool, "output_compression", request.OutputCompression, form) + setRawOrFormIntToolOption(tool, "partial_images", request.PartialImages, form) + setStringToolOption(tool, "input_fidelity", firstFormValue(form, "input_fidelity")) +} + +func setStringToolOption(tool map[string]any, key string, value string) { + value = strings.TrimSpace(value) + if value != "" { + tool[key] = value + } +} + +func setRawOrFormStringToolOption(tool map[string]any, key string, raw json.RawMessage, form *multipart.Form) { + if value := firstFormValue(form, key); value != "" { + tool[key] = value + return + } + setRawToolOption(tool, key, raw) +} + +func setRawOrFormIntToolOption(tool map[string]any, key string, raw json.RawMessage, form *multipart.Form) { + if value := firstFormValue(form, key); value != "" { + parsed, err := strconv.ParseInt(value, 10, 64) + if err == nil { + tool[key] = parsed + } + return + } + setRawToolOption(tool, key, raw) +} + +func setRawToolOption(tool map[string]any, key string, raw json.RawMessage) { + raw = bytes.TrimSpace(raw) + if len(raw) == 0 { + return + } + var value any + if err := common.Unmarshal(raw, &value); err == nil { + tool[key] = value + } +} + +func firstFormValue(form *multipart.Form, key string) string { + if form == nil || form.Value == nil { + return "" + } + values := form.Value[key] + if len(values) == 0 { + return "" + } + return strings.TrimSpace(values[0]) +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return strings.TrimSpace(value) + } + } + return "" +} + +func handleImageResponse(c *gin.Context, resp *http.Response, info *relaycommon.RelayInfo) (*dto.Usage, *types.NewAPIError) { + if resp == nil || resp.Body == nil { + return nil, types.NewOpenAIError(fmt.Errorf("invalid response"), types.ErrorCodeBadResponse, http.StatusInternalServerError) + } + defer service.CloseResponseBodyGracefully(resp) + + responseBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, types.NewOpenAIError(err, types.ErrorCodeReadResponseBodyFailed, http.StatusInternalServerError) + } + + results, createdAt, usage, firstMeta, newAPIError := collectImagesFromResponseBody(responseBody) + if newAPIError != nil { + return nil, newAPIError + } + if len(results) == 0 { + return nil, types.NewOpenAIError(fmt.Errorf("upstream did not return image output"), types.ErrorCodeBadResponseBody, http.StatusBadGateway) + } + + output, err := buildImageAPIResponse(results, createdAt, usage, firstMeta, c.GetString(ginKeyCodexImageResponseFormat)) + if err != nil { + return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError) + } + jsonResp := cloneResponseWithContentType(resp, "application/json") + service.IOCopyBytesGracefully(c, jsonResp, output) + return usage, nil +} + +func collectImagesFromResponseBody(body []byte) ([]imageCallResult, int64, *dto.Usage, imageCallResult, *types.NewAPIError) { + if looksLikeSSE(body) { + return collectImagesFromSSE(body) + } + return extractImagesFromCompletedJSON(body) +} + +func collectImagesFromSSE(body []byte) ([]imageCallResult, int64, *dto.Usage, imageCallResult, *types.NewAPIError) { + scanner := bufio.NewScanner(bytes.NewReader(body)) + scanner.Buffer(make([]byte, 64<<10), 64<<20) + + var doneResults []imageCallResult + var firstDone imageCallResult + var partialResults []imageCallResult + var firstPartial imageCallResult + for scanner.Scan() { + payload := ssePayloadFromLine(scanner.Text()) + if len(payload) == 0 || bytes.Equal(payload, []byte("[DONE]")) { + continue + } + + switch gjson.GetBytes(payload, "type").String() { + case "response.completed": + results, createdAt, usage, firstMeta, newAPIError := extractImagesFromCompletedJSON(payload) + if newAPIError != nil { + return nil, 0, nil, imageCallResult{}, newAPIError + } + if len(results) > 0 { + return results, createdAt, usage, firstMeta, nil + } + if len(doneResults) > 0 { + return doneResults, createdAt, usage, firstDone, nil + } + if len(partialResults) > 0 { + return partialResults, createdAt, usage, firstPartial, nil + } + return results, createdAt, usage, firstMeta, nil + case "response.output_item.done": + item := gjson.GetBytes(payload, "item") + if item.Get("type").String() == dto.ResponsesOutputTypeImageGenerationCall { + result := imageCallResultFromGJSON(item) + if result.Result != "" { + if len(doneResults) == 0 { + firstDone = result + } + doneResults = append(doneResults, result) + } + } + case "response.image_generation_call.partial_image": + b64 := strings.TrimSpace(gjson.GetBytes(payload, "partial_image_b64").String()) + if b64 != "" { + result := imageCallResult{ + Result: b64, + OutputFormat: strings.TrimSpace(gjson.GetBytes(payload, "output_format").String()), + } + if len(partialResults) == 0 { + firstPartial = result + } + partialResults = append(partialResults, result) + } + case "response.error", "response.failed": + message := extractCodexErrorMessage(payload) + if message == "" { + message = strings.TrimSpace(string(payload)) + } + return nil, 0, nil, imageCallResult{}, types.NewOpenAIError(fmt.Errorf("codex upstream error: %s", truncateErrorMessage(message)), types.ErrorCodeBadResponseBody, http.StatusBadGateway) + } + } + if err := scanner.Err(); err != nil { + return nil, 0, nil, imageCallResult{}, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusBadGateway) + } + if len(doneResults) > 0 { + return doneResults, time.Now().Unix(), &dto.Usage{}, firstDone, nil + } + if len(partialResults) > 0 { + return partialResults, time.Now().Unix(), &dto.Usage{}, firstPartial, nil + } + return nil, 0, nil, imageCallResult{}, types.NewOpenAIError(fmt.Errorf("stream disconnected before image completion"), types.ErrorCodeBadResponseBody, http.StatusBadGateway) +} + +func extractImagesFromCompletedJSON(payload []byte) ([]imageCallResult, int64, *dto.Usage, imageCallResult, *types.NewAPIError) { + root := gjson.ParseBytes(payload) + response := root.Get("response") + if !response.Exists() { + response = root + } + + createdAt := response.Get("created_at").Int() + if createdAt <= 0 { + createdAt = time.Now().Unix() + } + + var results []imageCallResult + var firstMeta imageCallResult + output := response.Get("output") + if output.IsArray() { + for _, item := range output.Array() { + if item.Get("type").String() != dto.ResponsesOutputTypeImageGenerationCall { + continue + } + result := imageCallResultFromGJSON(item) + if result.Result == "" { + continue + } + if len(results) == 0 { + firstMeta = result + } + results = append(results, result) + } + } + + usage := &dto.Usage{} + if usageRaw := response.Get("usage"); usageRaw.Exists() && usageRaw.IsObject() { + var responseUsage dto.Usage + if err := common.Unmarshal([]byte(usageRaw.Raw), &responseUsage); err == nil { + usage = usageFromResponseUsage(&responseUsage) + } + } else if usageRaw := response.Get("tool_usage.image_gen"); usageRaw.Exists() && usageRaw.IsObject() { + var responseUsage dto.Usage + if err := common.Unmarshal([]byte(usageRaw.Raw), &responseUsage); err == nil { + usage = usageFromResponseUsage(&responseUsage) + } + } + return results, createdAt, usage, firstMeta, nil +} + +func imageCallResultFromGJSON(item gjson.Result) imageCallResult { + return imageCallResult{ + Result: strings.TrimSpace(item.Get("result").String()), + RevisedPrompt: strings.TrimSpace(item.Get("revised_prompt").String()), + OutputFormat: strings.TrimSpace(item.Get("output_format").String()), + Size: strings.TrimSpace(item.Get("size").String()), + Background: strings.TrimSpace(item.Get("background").String()), + Quality: strings.TrimSpace(item.Get("quality").String()), + } +} + +func usageFromResponseUsage(responseUsage *dto.Usage) *dto.Usage { + if responseUsage == nil { + return &dto.Usage{} + } + usage := *responseUsage + if usage.PromptTokens == 0 && usage.InputTokens > 0 { + usage.PromptTokens = usage.InputTokens + } + if usage.CompletionTokens == 0 && usage.OutputTokens > 0 { + usage.CompletionTokens = usage.OutputTokens + } + if usage.TotalTokens == 0 { + usage.TotalTokens = usage.PromptTokens + usage.CompletionTokens + } + return &usage +} + +func buildImageAPIResponse(results []imageCallResult, createdAt int64, usage *dto.Usage, firstMeta imageCallResult, responseFormat string) ([]byte, error) { + responseFormat = strings.ToLower(strings.TrimSpace(responseFormat)) + if responseFormat == "" { + responseFormat = "b64_json" + } + + imageResponse := dto.ImageResponse{ + Created: createdAt, + Data: make([]dto.ImageData, 0, len(results)), + Background: firstMeta.Background, + OutputFormat: firstMeta.OutputFormat, + Quality: firstMeta.Quality, + Size: firstMeta.Size, + } + if usage != nil && usage.TotalTokens > 0 { + imageResponse.Usage = usage + } + + for _, result := range results { + item := dto.ImageData{ + RevisedPrompt: result.RevisedPrompt, + } + if responseFormat == "url" { + item.Url = "data:" + mimeTypeFromOutputFormat(result.OutputFormat) + ";base64," + result.Result + } else { + item.B64Json = result.Result + } + imageResponse.Data = append(imageResponse.Data, item) + } + return common.Marshal(imageResponse) +} + +func mimeTypeFromOutputFormat(outputFormat string) string { + outputFormat = strings.ToLower(strings.TrimSpace(outputFormat)) + if outputFormat == "" { + return "image/png" + } + if strings.Contains(outputFormat, "/") { + return outputFormat + } + switch outputFormat { + case "jpg", "jpeg": + return "image/jpeg" + case "webp": + return "image/webp" + default: + return "image/png" + } +} diff --git a/relay/channel/codex/responses.go b/relay/channel/codex/responses.go new file mode 100644 index 000000000000..326c0dadb72c --- /dev/null +++ b/relay/channel/codex/responses.go @@ -0,0 +1,390 @@ +package codex + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/dto" + relaycommon "github.com/QuantumNous/new-api/relay/common" + relayhelper "github.com/QuantumNous/new-api/relay/helper" + "github.com/QuantumNous/new-api/service" + "github.com/QuantumNous/new-api/types" + "github.com/gin-gonic/gin" + "github.com/tidwall/gjson" +) + +func handleResponsesNonStream(c *gin.Context, resp *http.Response, info *relaycommon.RelayInfo) (*dto.Usage, *types.NewAPIError) { + if resp == nil || resp.Body == nil { + return nil, types.NewOpenAIError(fmt.Errorf("invalid response"), types.ErrorCodeBadResponse, http.StatusInternalServerError) + } + defer service.CloseResponseBodyGracefully(resp) + + responseBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, types.NewOpenAIError(err, types.ErrorCodeReadResponseBodyFailed, http.StatusInternalServerError) + } + + if !looksLikeSSE(responseBody) { + return writeResponsesJSONBody(c, resp, info, responseBody) + } + + responseJSON, usage, newAPIError := collectCompletedResponseFromSSE(responseBody) + if newAPIError != nil { + return nil, newAPIError + } + response := gjson.ParseBytes(responseJSON) + if responseHasImageGenerationOutput(response) { + markImageGenerationCall(c, responseImageGenerationQuality(response), responseImageGenerationSize(response)) + } + recordResponsesBuiltInToolUsageFromGJSON(info, response) + + jsonResp := cloneResponseWithContentType(resp, "application/json") + service.IOCopyBytesGracefully(c, jsonResp, responseJSON) + return usage, nil +} + +func handleResponsesStream(c *gin.Context, resp *http.Response, info *relaycommon.RelayInfo) (*dto.Usage, *types.NewAPIError) { + if resp == nil || resp.Body == nil { + return nil, types.NewOpenAIError(fmt.Errorf("invalid response"), types.ErrorCodeBadResponse, http.StatusInternalServerError) + } + defer service.CloseResponseBodyGracefully(resp) + + usage := &dto.Usage{} + var responseTextBuilder strings.Builder + + relayhelper.StreamScannerHandler(c, resp, info, func(data string, sr *relayhelper.StreamResult) { + payload := gjson.Parse(data) + eventType := payload.Get("type").String() + writeResponsesStreamPayload(c, eventType, data) + + switch eventType { + case "response.completed": + response := payload.Get("response") + if response.Exists() && response.IsObject() { + mergeResponseUsage(usage, usageFromResponseGJSON(response)) + if responseHasImageGenerationOutput(response) { + markImageGenerationCall(c, responseImageGenerationQuality(response), responseImageGenerationSize(response)) + } + recordResponsesBuiltInToolUsageFromGJSON(info, response) + } + mergeResponseUsage(usage, usageFromCodexToolUsageImageGen([]byte(data))) + case "response.output_text.delta": + responseTextBuilder.WriteString(payload.Get("delta").String()) + case dto.ResponsesOutputTypeItemDone: + item := payload.Get("item") + switch item.Get("type").String() { + case dto.ResponsesOutputTypeImageGenerationCall: + markImageGenerationCall(c, item.Get("quality").String(), item.Get("size").String()) + recordResponsesBuiltInToolCall(info, imageGenerationTool) + case dto.BuildInCallWebSearchCall: + recordResponsesBuiltInToolCall(info, dto.BuildInToolWebSearchPreview) + } + } + }) + + if usage.CompletionTokens == 0 { + tempStr := responseTextBuilder.String() + if len(tempStr) > 0 { + modelName := "" + if info != nil && info.ChannelMeta != nil { + modelName = info.UpstreamModelName + } + usage.CompletionTokens = service.CountTextToken(tempStr, modelName) + } + } + if info != nil && usage.PromptTokens == 0 && usage.CompletionTokens != 0 { + usage.PromptTokens = info.GetEstimatePromptTokens() + } + if usage.TotalTokens == 0 { + usage.TotalTokens = usage.PromptTokens + usage.CompletionTokens + } + return usage, nil +} + +func writeResponsesStreamPayload(c *gin.Context, eventType string, data string) { + if eventType != "" { + c.Render(-1, common.CustomEvent{Data: fmt.Sprintf("event: %s\n", eventType)}) + } + c.Render(-1, common.CustomEvent{Data: fmt.Sprintf("data: %s", data)}) + _ = relayhelper.FlushWriter(c) +} + +func looksLikeSSE(body []byte) bool { + scanner := bufio.NewScanner(bytes.NewReader(body)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + return strings.HasPrefix(line, "data:") || strings.HasPrefix(line, "event:") + } + return false +} + +func writeResponsesJSONBody(c *gin.Context, resp *http.Response, info *relaycommon.RelayInfo, body []byte) (*dto.Usage, *types.NewAPIError) { + response := gjson.ParseBytes(body) + if !response.Exists() || !response.IsObject() { + return nil, types.NewOpenAIError(fmt.Errorf("codex upstream returned invalid JSON response"), types.ErrorCodeBadResponseBody, http.StatusInternalServerError) + } + if errResult := response.Get("error"); errResult.Exists() && errResult.Type != gjson.Null { + message := extractCodexErrorMessage(body) + if message == "" { + message = strings.TrimSpace(errResult.String()) + } + if message == "" { + message = strings.TrimSpace(errResult.Raw) + } + return nil, types.WithOpenAIError(types.OpenAIError{ + Message: message, + Type: "upstream_error", + Code: types.ErrorCodeBadResponseBody, + }, resp.StatusCode) + } + + if responseHasImageGenerationOutput(response) { + markImageGenerationCall(c, responseImageGenerationQuality(response), responseImageGenerationSize(response)) + } + + jsonResp := cloneResponseWithContentType(resp, "application/json") + service.IOCopyBytesGracefully(c, jsonResp, body) + + usage := usageFromResponseGJSON(response) + recordResponsesBuiltInToolUsageFromGJSON(info, response) + return usage, nil +} + +func collectCompletedResponseFromSSE(body []byte) ([]byte, *dto.Usage, *types.NewAPIError) { + scanner := bufio.NewScanner(bytes.NewReader(body)) + scanner.Buffer(make([]byte, 64<<10), 64<<20) + + var outputItems []json.RawMessage + var imageOutputItems []json.RawMessage + for scanner.Scan() { + payload := ssePayloadFromLine(scanner.Text()) + if len(payload) == 0 || bytes.Equal(payload, []byte("[DONE]")) { + continue + } + + eventType := gjson.GetBytes(payload, "type").String() + switch eventType { + case dto.ResponsesOutputTypeItemDone: + item := gjson.GetBytes(payload, "item") + if item.Exists() && item.IsObject() { + outputItems = append(outputItems, json.RawMessage(item.Raw)) + if item.Get("type").String() == dto.ResponsesOutputTypeImageGenerationCall && strings.TrimSpace(item.Get("result").String()) != "" { + imageOutputItems = append(imageOutputItems, json.RawMessage(item.Raw)) + } + } + case "response.completed": + response := gjson.GetBytes(payload, "response") + if !response.Exists() || !response.IsObject() { + return nil, nil, types.NewOpenAIError(fmt.Errorf("codex response.completed missing response object"), types.ErrorCodeBadResponseBody, http.StatusBadGateway) + } + responseRaw := []byte(response.Raw) + if len(outputItems) > 0 && responseOutputLen(response) == 0 { + merged, err := appendResponseOutputItems(responseRaw, outputItems) + if err != nil { + return nil, nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusBadGateway) + } + responseRaw = merged + } else if len(imageOutputItems) > 0 && !responseHasImageGenerationOutput(response) { + merged, err := appendResponseOutputItems(responseRaw, imageOutputItems) + if err != nil { + return nil, nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusBadGateway) + } + responseRaw = merged + } + usage := usageFromResponseGJSON(gjson.ParseBytes(responseRaw)) + mergeResponseUsage(usage, usageFromCodexToolUsageImageGen(payload)) + return responseRaw, usage, nil + case "response.error", "response.failed": + message := extractCodexErrorMessage(payload) + if message == "" { + message = strings.TrimSpace(string(payload)) + } + return nil, nil, types.NewOpenAIError(fmt.Errorf("codex upstream error: %s", truncateErrorMessage(message)), types.ErrorCodeBadResponseBody, http.StatusBadGateway) + } + } + if err := scanner.Err(); err != nil { + return nil, nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusBadGateway) + } + return nil, nil, types.NewOpenAIError(fmt.Errorf("codex stream ended before response.completed"), types.ErrorCodeBadResponseBody, http.StatusBadGateway) +} + +func responseOutputLen(response gjson.Result) int { + output := response.Get("output") + if !output.IsArray() { + return 0 + } + return len(output.Array()) +} + +func responseHasImageGenerationOutput(response gjson.Result) bool { + output := response.Get("output") + if !output.IsArray() { + return false + } + for _, item := range output.Array() { + if item.Get("type").String() == dto.ResponsesOutputTypeImageGenerationCall { + return true + } + } + return false +} + +func responseImageGenerationQuality(response gjson.Result) string { + return firstImageGenerationOutputField(response, "quality") +} + +func responseImageGenerationSize(response gjson.Result) string { + return firstImageGenerationOutputField(response, "size") +} + +func firstImageGenerationOutputField(response gjson.Result, field string) string { + output := response.Get("output") + if !output.IsArray() { + return "" + } + for _, item := range output.Array() { + if item.Get("type").String() == dto.ResponsesOutputTypeImageGenerationCall { + return item.Get(field).String() + } + } + return "" +} + +func appendResponseOutputItems(responseRaw []byte, items []json.RawMessage) ([]byte, error) { + var response map[string]any + if err := common.Unmarshal(responseRaw, &response); err != nil { + return nil, err + } + output, _ := response["output"].([]any) + for _, rawItem := range items { + var item any + if err := common.Unmarshal(rawItem, &item); err != nil { + return nil, err + } + output = append(output, item) + } + response["output"] = output + return common.Marshal(response) +} + +func ssePayloadFromLine(line string) []byte { + line = strings.TrimSpace(strings.TrimRight(line, "\r")) + if !strings.HasPrefix(line, "data:") { + return nil + } + payload := strings.TrimSpace(strings.TrimPrefix(line, "data:")) + if payload == "" { + return nil + } + return []byte(payload) +} + +func usageFromResponseGJSON(response gjson.Result) *dto.Usage { + usage := &dto.Usage{} + usageRaw := response.Get("usage") + if !usageRaw.Exists() || !usageRaw.IsObject() { + return usage + } + usage.PromptTokens = int(usageRaw.Get("input_tokens").Int()) + usage.CompletionTokens = int(usageRaw.Get("output_tokens").Int()) + usage.TotalTokens = int(usageRaw.Get("total_tokens").Int()) + usage.InputTokens = usage.PromptTokens + usage.OutputTokens = usage.CompletionTokens + usage.PromptTokensDetails.CachedTokens = int(usageRaw.Get("input_tokens_details.cached_tokens").Int()) + return usage +} + +func usageFromCodexToolUsageImageGen(payload []byte) *dto.Usage { + usageRaw := gjson.GetBytes(payload, "response.tool_usage.image_gen") + if !usageRaw.Exists() || !usageRaw.IsObject() { + return &dto.Usage{} + } + var responseUsage dto.Usage + if err := common.Unmarshal([]byte(usageRaw.Raw), &responseUsage); err != nil { + return &dto.Usage{} + } + return usageFromResponseUsage(&responseUsage) +} + +func mergeResponseUsage(dst *dto.Usage, src *dto.Usage) { + if dst == nil || src == nil { + return + } + if src.PromptTokens != 0 { + dst.PromptTokens = src.PromptTokens + } + if src.CompletionTokens != 0 { + dst.CompletionTokens = src.CompletionTokens + } + if src.TotalTokens != 0 { + dst.TotalTokens = src.TotalTokens + } + if src.InputTokens != 0 { + dst.InputTokens = src.InputTokens + } + if src.OutputTokens != 0 { + dst.OutputTokens = src.OutputTokens + } + if src.InputTokensDetails != nil { + dst.InputTokensDetails = src.InputTokensDetails + } + if src.PromptTokensDetails.CachedTokens != 0 { + dst.PromptTokensDetails.CachedTokens = src.PromptTokensDetails.CachedTokens + } +} + +func markImageGenerationCall(c *gin.Context, quality string, size string) { + c.Set("image_generation_call", true) + if strings.TrimSpace(quality) != "" { + c.Set("image_generation_call_quality", quality) + } + if strings.TrimSpace(size) != "" { + c.Set("image_generation_call_size", size) + } +} + +func recordResponsesBuiltInToolCall(info *relaycommon.RelayInfo, toolType string) { + if info == nil || info.ResponsesUsageInfo == nil || info.ResponsesUsageInfo.BuiltInTools == nil { + return + } + buildToolInfo, ok := info.ResponsesUsageInfo.BuiltInTools[toolType] + if ok && buildToolInfo != nil { + buildToolInfo.CallCount++ + } +} + +func recordResponsesBuiltInToolUsageFromGJSON(info *relaycommon.RelayInfo, response gjson.Result) { + if info == nil || info.ResponsesUsageInfo == nil || info.ResponsesUsageInfo.BuiltInTools == nil { + return + } + tools := response.Get("tools") + if !tools.IsArray() { + return + } + for _, tool := range tools.Array() { + buildToolInfo, ok := info.ResponsesUsageInfo.BuiltInTools[tool.Get("type").String()] + if ok && buildToolInfo != nil { + buildToolInfo.CallCount++ + } + } +} + +func cloneResponseWithContentType(resp *http.Response, contentType string) *http.Response { + if resp == nil { + return nil + } + cloned := *resp + cloned.Header = resp.Header.Clone() + cloned.Header.Set("Content-Type", contentType) + cloned.Header.Del("Transfer-Encoding") + return &cloned +} diff --git a/relay/constant/relay_mode.go b/relay/constant/relay_mode.go index 256715679213..1e2cce6fa0bb 100644 --- a/relay/constant/relay_mode.go +++ b/relay/constant/relay_mode.go @@ -76,6 +76,10 @@ func Path2RelayMode(path string) int { relayMode = RelayModeResponsesCompact } else if strings.HasPrefix(path, "/v1/responses") { relayMode = RelayModeResponses + } else if strings.HasPrefix(path, "/backend-api/codex/responses/compact") { + relayMode = RelayModeResponsesCompact + } else if strings.HasPrefix(path, "/backend-api/codex/responses") { + relayMode = RelayModeResponses } else if strings.HasPrefix(path, "/v1/audio/speech") { relayMode = RelayModeAudioSpeech } else if strings.HasPrefix(path, "/v1/audio/transcriptions") { diff --git a/relay/constant/relay_mode_test.go b/relay/constant/relay_mode_test.go new file mode 100644 index 000000000000..7fb3ffc93e86 --- /dev/null +++ b/relay/constant/relay_mode_test.go @@ -0,0 +1,12 @@ +package constant + +import "testing" + +func TestPath2RelayModeCodexBackendResponses(t *testing.T) { + if got := Path2RelayMode("/backend-api/codex/responses"); got != RelayModeResponses { + t.Fatalf("unexpected relay mode for codex responses: got %d want %d", got, RelayModeResponses) + } + if got := Path2RelayMode("/backend-api/codex/responses/compact"); got != RelayModeResponsesCompact { + t.Fatalf("unexpected relay mode for codex compact responses: got %d want %d", got, RelayModeResponsesCompact) + } +} diff --git a/relay/image_handler.go b/relay/image_handler.go index e986dd897e65..e3fd74bcd7d9 100644 --- a/relay/image_handler.go +++ b/relay/image_handler.go @@ -11,6 +11,7 @@ import ( "github.com/QuantumNous/new-api/constant" "github.com/QuantumNous/new-api/dto" "github.com/QuantumNous/new-api/logger" + codexchannel "github.com/QuantumNous/new-api/relay/channel/codex" relaycommon "github.com/QuantumNous/new-api/relay/common" "github.com/QuantumNous/new-api/relay/helper" "github.com/QuantumNous/new-api/service" @@ -46,7 +47,8 @@ func ImageHelper(c *gin.Context, info *relaycommon.RelayInfo) (newAPIError *type var requestBody io.Reader - if model_setting.GetGlobalSettings().PassThroughRequestEnabled || info.ChannelSetting.PassThroughBodyEnabled { + passThroughRequest := model_setting.GetGlobalSettings().PassThroughRequestEnabled || info.ChannelSetting.PassThroughBodyEnabled + if passThroughRequest && info.ApiType != constant.APITypeCodex { storage, err := common.GetBodyStorage(c) if err != nil { return types.NewErrorWithStatusCode(err, types.ErrorCodeReadRequestBodyFailed, http.StatusBadRequest, types.ErrOptionWithSkipRetry()) @@ -97,6 +99,11 @@ func ImageHelper(c *gin.Context, info *relaycommon.RelayInfo) (newAPIError *type if httpResp.StatusCode == http.StatusCreated && info.ApiType == constant.APITypeReplicate { // replicate channel returns 201 Created when using Prefer: wait, treat it as success. httpResp.StatusCode = http.StatusOK + } else if info.ApiType == constant.APITypeCodex { + newAPIError = codexchannel.RelayErrorHandler(c.Request.Context(), httpResp) + // reset status code 重置状态码 + service.ResetStatusCode(newAPIError, statusCodeMappingStr) + return newAPIError } else { newAPIError = service.RelayErrorHandler(c.Request.Context(), httpResp, false) // reset status code 重置状态码 diff --git a/relay/responses_handler.go b/relay/responses_handler.go index 58324aa7cec9..ffd99b376727 100644 --- a/relay/responses_handler.go +++ b/relay/responses_handler.go @@ -10,6 +10,7 @@ import ( "github.com/QuantumNous/new-api/common" appconstant "github.com/QuantumNous/new-api/constant" "github.com/QuantumNous/new-api/dto" + codexchannel "github.com/QuantumNous/new-api/relay/channel/codex" relaycommon "github.com/QuantumNous/new-api/relay/common" relayconstant "github.com/QuantumNous/new-api/relay/constant" "github.com/QuantumNous/new-api/relay/helper" @@ -71,7 +72,8 @@ func ResponsesHelper(c *gin.Context, info *relaycommon.RelayInfo) (newAPIError * } adaptor.Init(info) var requestBody io.Reader - if model_setting.GetGlobalSettings().PassThroughRequestEnabled || info.ChannelSetting.PassThroughBodyEnabled { + passThroughRequest := model_setting.GetGlobalSettings().PassThroughRequestEnabled || info.ChannelSetting.PassThroughBodyEnabled + if passThroughRequest && info.ApiType != appconstant.APITypeCodex { storage, err := common.GetBodyStorage(c) if err != nil { return types.NewError(err, types.ErrorCodeReadRequestBodyFailed, types.ErrOptionWithSkipRetry()) @@ -89,7 +91,11 @@ func ResponsesHelper(c *gin.Context, info *relaycommon.RelayInfo) (newAPIError * } // remove disabled fields for OpenAI Responses API - jsonData, err = relaycommon.RemoveDisabledFields(jsonData, info.ChannelOtherSettings, info.ChannelSetting.PassThroughBodyEnabled) + skipFieldFiltering := info.ChannelSetting.PassThroughBodyEnabled + if info.ApiType == appconstant.APITypeCodex { + skipFieldFiltering = skipFieldFiltering || codexchannel.IsRawResponsesRequest(convertedRequest) + } + jsonData, err = relaycommon.RemoveDisabledFields(jsonData, info.ChannelOtherSettings, skipFieldFiltering) if err != nil { return types.NewError(err, types.ErrorCodeConvertRequestFailed, types.ErrOptionWithSkipRetry()) } @@ -120,7 +126,11 @@ func ResponsesHelper(c *gin.Context, info *relaycommon.RelayInfo) (newAPIError * httpResp = resp.(*http.Response) if httpResp.StatusCode != http.StatusOK { - newAPIError = service.RelayErrorHandler(c.Request.Context(), httpResp, false) + if info.ApiType == appconstant.APITypeCodex { + newAPIError = codexchannel.RelayErrorHandler(c.Request.Context(), httpResp) + } else { + newAPIError = service.RelayErrorHandler(c.Request.Context(), httpResp, false) + } // reset status code 重置状态码 service.ResetStatusCode(newAPIError, statusCodeMappingStr) return newAPIError diff --git a/router/relay-router.go b/router/relay-router.go index 17a13cad7fd6..a28b52162617 100644 --- a/router/relay-router.go +++ b/router/relay-router.go @@ -165,6 +165,21 @@ func SetRelayRouter(router *gin.Engine) { httpRouter.DELETE("/models/:model", controller.RelayNotImplemented) } + codexBackendRouter := router.Group("/backend-api/codex") + codexBackendRouter.Use(middleware.RouteTag("relay")) + codexBackendRouter.Use(middleware.SystemPerformanceCheck()) + codexBackendRouter.Use(middleware.TokenAuth()) + codexBackendRouter.Use(middleware.ModelRequestRateLimit()) + codexBackendRouter.Use(middleware.Distribute()) + { + codexBackendRouter.POST("/responses", func(c *gin.Context) { + controller.Relay(c, types.RelayFormatOpenAIResponses) + }) + codexBackendRouter.POST("/responses/compact", func(c *gin.Context) { + controller.Relay(c, types.RelayFormatOpenAIResponsesCompaction) + }) + } + relayMjRouter := router.Group("/mj") relayMjRouter.Use(middleware.RouteTag("relay")) relayMjRouter.Use(middleware.SystemPerformanceCheck())