Repository navigation
feat: openai response /v1/response/compact #2644
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 5 commits
85d3a4d
02405fb
f9d1aa1
bb3dfd8
3c98268
960fae0
d44fe0b
5eed8aa
1b0f11c
c631184
7f8f9bc
e235850
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| package dto | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
|
|
||
| "github.com/QuantumNous/new-api/types" | ||
| ) | ||
|
|
||
| type OpenAIResponsesCompactionResponse struct { | ||
| ID string `json:"id"` | ||
| Object string `json:"object"` | ||
| CreatedAt int `json:"created_at"` | ||
| Output json.RawMessage `json:"output"` | ||
| Usage *Usage `json:"usage"` | ||
| Error any `json:"error,omitempty"` | ||
| } | ||
|
|
||
| func (o *OpenAIResponsesCompactionResponse) GetOpenAIError() *types.OpenAIError { | ||
| return GetOpenAIError(o.Error) | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| package dto | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "strings" | ||
|
|
||
| "github.com/QuantumNous/new-api/types" | ||
|
|
||
| "github.com/gin-gonic/gin" | ||
| ) | ||
|
|
||
| type OpenAIResponsesCompactionRequest struct { | ||
| Model string `json:"model"` | ||
| Input json.RawMessage `json:"input,omitempty"` | ||
| Instructions json.RawMessage `json:"instructions,omitempty"` | ||
| PreviousResponseID string `json:"previous_response_id,omitempty"` | ||
| } | ||
|
|
||
| func (r *OpenAIResponsesCompactionRequest) GetTokenCountMeta() *types.TokenCountMeta { | ||
| var parts []string | ||
| if len(r.Instructions) > 0 { | ||
| parts = append(parts, string(r.Instructions)) | ||
| } | ||
| if len(r.Input) > 0 { | ||
| parts = append(parts, string(r.Input)) | ||
| } | ||
| return &types.TokenCountMeta{ | ||
| CombineText: strings.Join(parts, "\n"), | ||
| } | ||
| } | ||
|
|
||
| func (r *OpenAIResponsesCompactionRequest) IsStream(c *gin.Context) bool { | ||
| return false | ||
| } | ||
|
|
||
| func (r *OpenAIResponsesCompactionRequest) SetModelName(modelName string) { | ||
| if modelName != "" { | ||
| r.Model = modelName | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| package openai | ||
|
|
||
| import ( | ||
| "io" | ||
| "net/http" | ||
|
|
||
| "github.com/QuantumNous/new-api/common" | ||
| "github.com/QuantumNous/new-api/dto" | ||
| "github.com/QuantumNous/new-api/service" | ||
| "github.com/QuantumNous/new-api/types" | ||
|
|
||
| "github.com/gin-gonic/gin" | ||
| ) | ||
|
|
||
| func OaiResponsesCompactionHandler(c *gin.Context, resp *http.Response) (*dto.Usage, *types.NewAPIError) { | ||
| defer service.CloseResponseBodyGracefully(resp) | ||
|
|
||
| responseBody, err := io.ReadAll(resp.Body) | ||
| if err != nil { | ||
| return nil, types.NewOpenAIError(err, types.ErrorCodeReadResponseBodyFailed, http.StatusInternalServerError) | ||
| } | ||
|
|
||
| var compactResp dto.OpenAIResponsesCompactionResponse | ||
| if err := common.Unmarshal(responseBody, &compactResp); err != nil { | ||
| return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError) | ||
| } | ||
| if oaiError := compactResp.GetOpenAIError(); oaiError != nil && oaiError.Type != "" { | ||
| return nil, types.WithOpenAIError(*oaiError, resp.StatusCode) | ||
| } | ||
|
|
||
| service.IOCopyBytesGracefully(c, resp, responseBody) | ||
|
|
||
| usage := dto.Usage{} | ||
| if compactResp.Usage != nil { | ||
| usage.PromptTokens = compactResp.Usage.InputTokens | ||
| usage.CompletionTokens = compactResp.Usage.OutputTokens | ||
| usage.TotalTokens = compactResp.Usage.TotalTokens | ||
| if compactResp.Usage.InputTokensDetails != nil { | ||
| usage.PromptTokensDetails.CachedTokens = compactResp.Usage.InputTokensDetails.CachedTokens | ||
| } | ||
| } | ||
|
|
||
| return &usage, nil | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -4,13 +4,27 @@ import ( | |
| "encoding/json" | ||
| "errors" | ||
| "fmt" | ||
| "strings" | ||
|
|
||
| "github.com/QuantumNous/new-api/dto" | ||
| "github.com/QuantumNous/new-api/relay/common" | ||
| relayconstant "github.com/QuantumNous/new-api/relay/constant" | ||
| "github.com/QuantumNous/new-api/setting/ratio_setting" | ||
| "github.com/gin-gonic/gin" | ||
| ) | ||
|
|
||
| func ModelMappedHelper(c *gin.Context, info *common.RelayInfo, request dto.Request) error { | ||
| if info.ChannelMeta == nil { | ||
| info.ChannelMeta = &common.ChannelMeta{} | ||
| } | ||
|
|
||
| isResponsesCompact := info.RelayMode == relayconstant.RelayModeResponsesCompact | ||
| originModelName := info.OriginModelName | ||
| mappingModelName := originModelName | ||
| if isResponsesCompact && strings.HasSuffix(originModelName, ratio_setting.CompactModelSuffix) { | ||
| mappingModelName = strings.TrimSuffix(originModelName, ratio_setting.CompactModelSuffix) | ||
| } | ||
|
|
||
| // map model name | ||
| modelMapping := c.GetString("model_mapping") | ||
| if modelMapping != "" && modelMapping != "{}" { | ||
|
|
@@ -21,7 +35,7 @@ func ModelMappedHelper(c *gin.Context, info *common.RelayInfo, request dto.Reque | |
| } | ||
|
|
||
| // 支持链式模型重定向,最终使用链尾的模型 | ||
| currentModel := info.OriginModelName | ||
| currentModel := mappingModelName | ||
| visitedModels := map[string]bool{ | ||
| currentModel: true, | ||
| } | ||
|
|
@@ -51,6 +65,15 @@ func ModelMappedHelper(c *gin.Context, info *common.RelayInfo, request dto.Reque | |
| info.UpstreamModelName = currentModel | ||
| } | ||
| } | ||
|
|
||
| if isResponsesCompact { | ||
| finalUpstreamModelName := mappingModelName | ||
| if info.IsModelMapped && info.UpstreamModelName != "" { | ||
| finalUpstreamModelName = info.UpstreamModelName | ||
| } | ||
| info.UpstreamModelName = finalUpstreamModelName | ||
| info.OriginModelName = ratio_setting.WithCompactModelSuffix(finalUpstreamModelName) | ||
| } | ||
| if request != nil { | ||
| request.SetModelName(info.UpstreamModelName) | ||
| } | ||
|
Comment on lines
77
to
79
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🧩 Analysis chain🏁 Script executed: # Find all SetModelName calls and their context
ast-grep --pattern 'SetModelName($_)'Repository: QuantumNous/new-api Length of output: 45 🏁 Script executed: # Read the relevant section of model_mapped.go to understand the complete logic flow
wc -l relay/helper/model_mapped.goRepository: QuantumNous/new-api Length of output: 94 🏁 Script executed: # Read the entire model_mapped.go file to understand the logic
cat -n relay/helper/model_mapped.goRepository: QuantumNous/new-api Length of output: 2903 🏁 Script executed: # Search for SetModelName implementation and where request comes from
rg -B 5 -A 5 "SetModelName" relay/Repository: QuantumNous/new-api Length of output: 1778 🏁 Script executed: # Check where UpstreamModelName is initialized and used
rg "UpstreamModelName\s*=" relay/ -B 2 -A 2Repository: QuantumNous/new-api Length of output: 6974 🏁 Script executed: # Find the Request type and SetModelName definition
fd "request\.go|dto\.go" relay/ | head -5Repository: QuantumNous/new-api Length of output: 196 🏁 Script executed: # Find RelayInfo struct definition
rg "type RelayInfo struct" relay/ -A 20Repository: QuantumNous/new-api Length of output: 1263 🏁 Script executed: # Check api_request.go to understand SetModelName
cat -n relay/channel/api_request.go | head -60Repository: QuantumNous/new-api Length of output: 2127 🏁 Script executed: # Search for SetModelName implementation
rg "func.*SetModelName" relay/ -A 5Repository: QuantumNous/new-api Length of output: 45 🏁 Script executed: # Look for the Request interface/struct definition and SetModelName
rg "SetModelName" dto/ -B 2 -A 3Repository: QuantumNous/new-api Length of output: 3113 Use When 🤖 Prompt for AI Agents |
||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Initialize RequestConversionChain for compaction paths.
This branch returns early, so
InitRequestConversionChain()never runs and the original relay format won’t be recorded. Align with the other formats by settinginfoand falling through to the common init.💡 Suggested fix
case types.RelayFormatOpenAIResponsesCompaction: if request, ok := request.(*dto.OpenAIResponsesCompactionRequest); ok { - return GenRelayInfoResponsesCompaction(c, request), nil + info = GenRelayInfoResponsesCompaction(c, request) + break } - return nil, errors.New("request is not a OpenAIResponsesCompactionRequest") + err = errors.New("request is not a OpenAIResponsesCompactionRequest")🤖 Prompt for AI Agents