From 36f795d830c5b9cdecd1d9c87217b4a67accbafc Mon Sep 17 00:00:00 2001 From: akshaydeo Date: Wed, 27 May 2026 04:49:21 +0530 Subject: [PATCH 1/3] release pipeline changes --- .github/workflows/release-pipeline.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release-pipeline.yml b/.github/workflows/release-pipeline.yml index 283b0c89fb5..014c5b8f358 100644 --- a/.github/workflows/release-pipeline.yml +++ b/.github/workflows/release-pipeline.yml @@ -2070,6 +2070,8 @@ jobs: egress-policy: block allowed-endpoints: > _http._tcp.azure.archive.ubuntu.com:443 + _https._tcp.dl.google.com:443 + motd.ubuntu.com:443 _https._tcp.esm.ubuntu.com:443 _https._tcp.motd.ubuntu.com:443 _https._tcp.packages.microsoft.com:443 From 92123d8a698d9768cd80cb6aeb24543ada02d012 Mon Sep 17 00:00:00 2001 From: Metbcy Date: Wed, 27 May 2026 05:05:12 +0000 Subject: [PATCH 2/3] feat(sgl): add SGLang as base provider with rerank + error translation Implements #3131 (part 1 of 2). - Add SGL to SupportedBaseProviders, wire UI dropdowns and key-URL field - core/providers/sgl/rerank.go: POST /v1/rerank, omit `model` field (sglang's V1RerankReqInput rejects unknowns), parse bare-array response with `score` (not `relevance_score`), guard against duplicate and out-of-range indices - core/providers/sgl/errors.go: ParseSGLError delegates to OpenAI parser first (handles wrapped envelope + gzip + status fallbacks), falls back to sglang's flat `{object:error,message,type,code}` envelope. Substring mappings: context_length_exceeded, out_of_memory, model_not_found - Wire ParseSGLError into chat/text completion sync + stream paths - Rerank opts into BifrostContextKeyPassthroughExtraParams=true so caller-supplied ExtraParams are merged into the outgoing body (mirrors vLLM pattern) - ParseSGLError reads from bifrostErr.ExtraFields.RawResponse (the already-decoded body) instead of re-snapshotting resp.Body(), so gzipped 4xx/5xx from sglang parse correctly - Tests: errors_test.go (6 funcs incl. gzip flat envelope), rerank_test.go (9 funcs), rerank_live_test.go (full Rerank() path through httptest server: ExtraParams forwarded, gzipped 400 decoded) --- .github/workflows/release-pipeline.yml | 2 - core/providers/sgl/errors.go | 137 ++++++++++ core/providers/sgl/errors_test.go | 201 +++++++++++++++ core/providers/sgl/rerank.go | 239 ++++++++++++++++++ core/providers/sgl/rerank_live_test.go | 220 ++++++++++++++++ core/providers/sgl/rerank_test.go | 191 ++++++++++++++ core/providers/sgl/sgl.go | 13 +- core/schemas/bifrost.go | 1 + .../dialogs/addNewCustomProviderSheet.tsx | 1 + .../fragments/allowedRequestsFields.tsx | 1 + .../fragments/apiKeysFormFragment.tsx | 7 +- .../fragments/apiStructureFormFragment.tsx | 1 + .../providers/views/providerKeyForm.tsx | 7 +- ui/lib/constants/config.ts | 11 + ui/lib/types/config.ts | 2 +- 15 files changed, 1020 insertions(+), 14 deletions(-) create mode 100644 core/providers/sgl/errors.go create mode 100644 core/providers/sgl/errors_test.go create mode 100644 core/providers/sgl/rerank.go create mode 100644 core/providers/sgl/rerank_live_test.go create mode 100644 core/providers/sgl/rerank_test.go diff --git a/.github/workflows/release-pipeline.yml b/.github/workflows/release-pipeline.yml index 014c5b8f358..283b0c89fb5 100644 --- a/.github/workflows/release-pipeline.yml +++ b/.github/workflows/release-pipeline.yml @@ -2070,8 +2070,6 @@ jobs: egress-policy: block allowed-endpoints: > _http._tcp.azure.archive.ubuntu.com:443 - _https._tcp.dl.google.com:443 - motd.ubuntu.com:443 _https._tcp.esm.ubuntu.com:443 _https._tcp.motd.ubuntu.com:443 _https._tcp.packages.microsoft.com:443 diff --git a/core/providers/sgl/errors.go b/core/providers/sgl/errors.go new file mode 100644 index 00000000000..d2cab408792 --- /dev/null +++ b/core/providers/sgl/errors.go @@ -0,0 +1,137 @@ +package sgl + +import ( + "fmt" + "strings" + + "github.com/bytedance/sonic" + "github.com/maximhq/bifrost/core/providers/openai" + "github.com/maximhq/bifrost/core/schemas" + "github.com/valyala/fasthttp" +) + +// sglFlatError matches sglang's flat error envelope: +// +// {"object":"error","message":"...","type":"BadRequestError","code":400} +// +// See python/sglang/srt/entrypoints/openai/serving_base.py upstream +// (function create_error_response, lines ~209-225). +type sglFlatError struct { + Object string `json:"object"` + Message string `json:"message"` + Type string `json:"type"` + Code interface{} `json:"code"` +} + +// ParseSGLError parses sglang error responses. +// +// It handles both error envelope shapes used by sglang and its forks: +// - Flat: {"object":"error","message":"...","type":"BadRequestError","code":400} +// - Wrapped: {"error":{"message":"...","type":"...","code":"..."}} +// +// Well-known message substrings are mapped to OpenAI-style codes: +// - "longer than the model's context length" -> context_length_exceeded / invalid_request_error +// - "out of memory" -> out_of_memory / server_error +// - "model is not loaded" -> model_not_found / invalid_request_error +// +// The raw sglang `message` is always preserved on the returned BifrostError so +// callers see the actual server explanation rather than a generic "status N". +func ParseSGLError(resp *fasthttp.Response) *schemas.BifrostError { + // Delegate to the shared OpenAI-shape parser first. This handles the + // wrapped {"error":{...}} envelope, decodes gzip bodies, fills in + // StatusCode / ExtraFields, and applies sane HTTP-status fallbacks. + // The decoded body is stashed on ExtraFields.RawResponse, so we read + // from there rather than re-snapshotting resp.Body() (which would be + // the still-compressed bytes if Content-Encoding was gzip). + bifrostErr := openai.ParseOpenAIError(resp) + if bifrostErr.Error == nil { + bifrostErr.Error = &schemas.ErrorField{} + } + + // If the wrapped parser did not pick up a useful message, try sglang's + // flat envelope. We treat the generic "provider API error (status N)" / + // "provider API error" fallback as "no useful message" for this purpose. + currentMsg := strings.TrimSpace(bifrostErr.Error.Message) + wrappedHadMessage := currentMsg != "" && !strings.HasPrefix(currentMsg, "provider API error") + if !wrappedHadMessage { + if flat, ok := extractFlatSGLErrorFromRaw(bifrostErr.ExtraFields.RawResponse); ok && flat.Message != "" { + bifrostErr.Error.Message = flat.Message + if flat.Type != "" { + t := flat.Type + bifrostErr.Error.Type = &t + } + if flat.Code != nil { + if codeStr := codeToString(flat.Code); codeStr != "" { + bifrostErr.Error.Code = &codeStr + } + } + } + } + + msg := bifrostErr.Error.Message + switch { + case strings.Contains(msg, "longer than the model's context length"): + setSGLErrorCode(bifrostErr.Error, "context_length_exceeded", "invalid_request_error") + case strings.Contains(msg, "out of memory"): + setSGLErrorCode(bifrostErr.Error, "out_of_memory", "server_error") + case strings.Contains(msg, "model is not loaded"): + setSGLErrorCode(bifrostErr.Error, "model_not_found", "invalid_request_error") + } + + return bifrostErr +} + +func setSGLErrorCode(field *schemas.ErrorField, code, typ string) { + c := code + t := typ + field.Code = &c + field.Type = &t +} + +// extractFlatSGLErrorFromRaw pulls a flat sglang error envelope out of the +// already-decoded body that openai.ParseOpenAIError stashed on +// ExtraFields.RawResponse. RawResponse may be a string (when JSON parsing +// failed upstream) or a map[string]interface{} (the parsed body). We handle +// both so a gzipped 4xx still yields the sglang message/type/code rather +// than just the generic HTTP-status fallback. +func extractFlatSGLErrorFromRaw(raw interface{}) (sglFlatError, bool) { + switch v := raw.(type) { + case string: + var flat sglFlatError + if err := sonic.Unmarshal([]byte(v), &flat); err == nil { + return flat, true + } + case map[string]interface{}: + flat := sglFlatError{} + if s, ok := v["object"].(string); ok { + flat.Object = s + } + if s, ok := v["message"].(string); ok { + flat.Message = s + } + if s, ok := v["type"].(string); ok { + flat.Type = s + } + if c, ok := v["code"]; ok { + flat.Code = c + } + return flat, true + } + return sglFlatError{}, false +} + +func codeToString(v interface{}) string { + switch x := v.(type) { + case string: + return x + case float64: + return fmt.Sprintf("%d", int(x)) + case float32: + return fmt.Sprintf("%d", int(x)) + case int: + return fmt.Sprintf("%d", x) + case int64: + return fmt.Sprintf("%d", x) + } + return "" +} diff --git a/core/providers/sgl/errors_test.go b/core/providers/sgl/errors_test.go new file mode 100644 index 00000000000..fe387f9219a --- /dev/null +++ b/core/providers/sgl/errors_test.go @@ -0,0 +1,201 @@ +package sgl + +import ( + "bytes" + "compress/gzip" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/valyala/fasthttp" +) + +// buildSGLErrorResponse creates a fasthttp.Response with the given status code +// and body. Returned as a helper to keep tests focused on error parsing logic. +func buildSGLErrorResponse(status int, body string) *fasthttp.Response { + resp := fasthttp.AcquireResponse() + resp.SetStatusCode(status) + resp.Header.SetContentType("application/json") + resp.SetBodyString(body) + return resp +} + +// strDeref returns the dereferenced value or empty string for nil. +func strDeref(p *string) string { + if p == nil { + return "" + } + return *p +} + +func TestParseSGLError_FlatEnvelope(t *testing.T) { + t.Parallel() + + resp := buildSGLErrorResponse(400, `{"object":"error","message":"bad request","type":"BadRequestError","code":400}`) + defer fasthttp.ReleaseResponse(resp) + + bifrostErr := ParseSGLError(resp) + require.NotNil(t, bifrostErr) + require.NotNil(t, bifrostErr.Error) + assert.Equal(t, "bad request", bifrostErr.Error.Message) + assert.Equal(t, "BadRequestError", strDeref(bifrostErr.Error.Type)) + assert.Equal(t, "400", strDeref(bifrostErr.Error.Code)) +} + +func TestParseSGLError_WrappedEnvelope(t *testing.T) { + t.Parallel() + + resp := buildSGLErrorResponse(400, `{"error":{"message":"wrapped boom","type":"invalid_request_error","code":"some_code"}}`) + defer fasthttp.ReleaseResponse(resp) + + bifrostErr := ParseSGLError(resp) + require.NotNil(t, bifrostErr) + require.NotNil(t, bifrostErr.Error) + assert.Equal(t, "wrapped boom", bifrostErr.Error.Message) + assert.Equal(t, "invalid_request_error", strDeref(bifrostErr.Error.Type)) + assert.Equal(t, "some_code", strDeref(bifrostErr.Error.Code)) +} + +func TestParseSGLError_SubstringMappings(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + message string + wantCode string + wantType string + }{ + { + name: "context length exceeded", + message: "This model's maximum context length is 4096 tokens. However, you requested 5000 tokens (..). Please reduce the length of the messages or completion. Input is longer than the model's context length.", + wantCode: "context_length_exceeded", + wantType: "invalid_request_error", + }, + { + name: "out of memory", + message: "CUDA out of memory while attempting to allocate buffer", + wantCode: "out_of_memory", + wantType: "server_error", + }, + { + name: "model not loaded", + message: "requested model is not loaded on this server", + wantCode: "model_not_found", + wantType: "invalid_request_error", + }, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + body := `{"object":"error","message":` + jsonString(tc.message) + `,"type":"BadRequestError","code":400}` + resp := buildSGLErrorResponse(400, body) + defer fasthttp.ReleaseResponse(resp) + + bifrostErr := ParseSGLError(resp) + require.NotNil(t, bifrostErr) + require.NotNil(t, bifrostErr.Error) + + // Message is always preserved verbatim — never replaced by a generic. + assert.Equal(t, tc.message, bifrostErr.Error.Message) + + assert.Equal(t, tc.wantCode, strDeref(bifrostErr.Error.Code), "code mapping") + assert.Equal(t, tc.wantType, strDeref(bifrostErr.Error.Type), "type mapping") + }) + } +} + +func TestParseSGLError_FallthroughPreservesMessage(t *testing.T) { + t.Parallel() + + // An unrecognized sglang error message should be preserved as-is, with + // no substring-derived code/type overrides applied. + const msg = "some sglang-specific error nobody has mapped yet" + resp := buildSGLErrorResponse(503, `{"object":"error","message":"`+msg+`","type":"InternalServerError","code":503}`) + defer fasthttp.ReleaseResponse(resp) + + bifrostErr := ParseSGLError(resp) + require.NotNil(t, bifrostErr) + require.NotNil(t, bifrostErr.Error) + assert.Equal(t, msg, bifrostErr.Error.Message) + // Type/code come from the flat envelope, not from a substring mapping. + assert.Equal(t, "InternalServerError", strDeref(bifrostErr.Error.Type)) + assert.Equal(t, "503", strDeref(bifrostErr.Error.Code)) + require.NotNil(t, bifrostErr.StatusCode) + assert.Equal(t, 503, *bifrostErr.StatusCode) +} + +func TestParseSGLError_EmptyBodyDelegatesToFallback(t *testing.T) { + t.Parallel() + + resp := buildSGLErrorResponse(429, "") + defer fasthttp.ReleaseResponse(resp) + + bifrostErr := ParseSGLError(resp) + require.NotNil(t, bifrostErr) + require.NotNil(t, bifrostErr.Error) + // We do not assert exact phrasing of the HTTP-status fallback message; + // only that we produced something non-empty so callers see a useful error. + assert.NotEmpty(t, bifrostErr.Error.Message) + require.NotNil(t, bifrostErr.StatusCode) + assert.Equal(t, 429, *bifrostErr.StatusCode) +} + +// TestParseSGLError_GzipFlatEnvelope verifies that a gzip-encoded sglang flat +// error envelope is still parsed correctly. ParseOpenAIError decodes the body +// upstream and stashes the decoded JSON on ExtraFields.RawResponse; our +// flat-envelope fallback must read from there, not from the still-compressed +// resp.Body(). +func TestParseSGLError_GzipFlatEnvelope(t *testing.T) { + t.Parallel() + + const msg = "out of memory while loading shard 0" + plain := `{"object":"error","message":"` + msg + `","type":"InternalServerError","code":500}` + + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + _, err := gw.Write([]byte(plain)) + require.NoError(t, err) + require.NoError(t, gw.Close()) + + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(resp) + resp.SetStatusCode(500) + resp.Header.SetContentType("application/json") + resp.Header.Set("Content-Encoding", "gzip") + resp.SetBody(buf.Bytes()) + + bifrostErr := ParseSGLError(resp) + require.NotNil(t, bifrostErr) + require.NotNil(t, bifrostErr.Error) + assert.Equal(t, msg, bifrostErr.Error.Message) + // Substring mapping should still fire on the decoded message. + assert.Equal(t, "out_of_memory", strDeref(bifrostErr.Error.Code)) + assert.Equal(t, "server_error", strDeref(bifrostErr.Error.Type)) +} + +// jsonString minimally escapes a Go string for embedding in a JSON literal. +// Only handles characters used by the test fixtures. +func jsonString(s string) string { + out := make([]byte, 0, len(s)+2) + out = append(out, '"') + for i := 0; i < len(s); i++ { + c := s[i] + switch c { + case '"', '\\': + out = append(out, '\\', c) + case '\n': + out = append(out, '\\', 'n') + case '\r': + out = append(out, '\\', 'r') + case '\t': + out = append(out, '\\', 't') + default: + out = append(out, c) + } + } + out = append(out, '"') + return string(out) +} diff --git a/core/providers/sgl/rerank.go b/core/providers/sgl/rerank.go new file mode 100644 index 00000000000..84eec3dc603 --- /dev/null +++ b/core/providers/sgl/rerank.go @@ -0,0 +1,239 @@ +package sgl + +import ( + "fmt" + "net/http" + "sort" + "strings" + "time" + + "github.com/bytedance/sonic" + providerUtils "github.com/maximhq/bifrost/core/providers/utils" + schemas "github.com/maximhq/bifrost/core/schemas" + "github.com/valyala/fasthttp" +) + +// sglRerankRequest is the wire body for sglang's /v1/rerank endpoint +// (V1RerankReqInput upstream). +// +// IMPORTANT: sglang's V1RerankReqInput rejects unknown fields, including +// `model`, so we intentionally omit it here. Adding a model field will cause +// sglang to return a 400 with "extra fields not permitted". +type sglRerankRequest struct { + Query string `json:"query"` + Documents []string `json:"documents"` + TopN *int `json:"top_n,omitempty"` + ReturnDocuments *bool `json:"return_documents,omitempty"` + ExtraParams map[string]interface{} `json:"-"` +} + +// GetExtraParams returns passthrough parameters for providerUtils.CheckContextAndGetRequestBody. +func (r *sglRerankRequest) GetExtraParams() map[string]interface{} { + return r.ExtraParams +} + +// ToSGLRerankRequest converts a Bifrost rerank request into sglang's wire format. +// +// The outgoing body intentionally omits `model`; sglang's /v1/rerank +// (V1RerankReqInput) does not accept it. +func ToSGLRerankRequest(bifrostReq *schemas.BifrostRerankRequest) *sglRerankRequest { + if bifrostReq == nil { + return nil + } + + sglReq := &sglRerankRequest{ + Query: bifrostReq.Query, + Documents: make([]string, len(bifrostReq.Documents)), + } + for i, doc := range bifrostReq.Documents { + sglReq.Documents[i] = doc.Text + } + + if bifrostReq.Params != nil { + sglReq.TopN = bifrostReq.Params.TopN + sglReq.ReturnDocuments = bifrostReq.Params.ReturnDocuments + sglReq.ExtraParams = bifrostReq.Params.ExtraParams + } + + return sglReq +} + +// ToBifrostRerankResponse converts sglang's bare-array rerank response payload +// to Bifrost format. +// +// sglang returns a bare JSON array: [{score, document, index}, ...] — NOT +// wrapped in {"results": [...]} like vLLM/Cohere. The score field is "score" +// (not "relevance_score"). +func ToBifrostRerankResponse(items []interface{}, documents []schemas.RerankDocument, returnDocuments bool) (*schemas.BifrostRerankResponse, error) { + if items == nil { + return nil, fmt.Errorf("sgl rerank response is nil") + } + + response := &schemas.BifrostRerankResponse{} + seenIndices := make(map[int]struct{}, len(items)) + response.Results = make([]schemas.RerankResult, 0, len(items)) + + for _, item := range items { + itemMap, ok := item.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("invalid sgl rerank response: result item must be an object") + } + + index, ok := schemas.SafeExtractInt(itemMap["index"]) + if !ok { + return nil, fmt.Errorf("invalid sgl rerank response: result index is required") + } + if index < 0 || index >= len(documents) { + return nil, fmt.Errorf("invalid sgl rerank response: result index %d out of range", index) + } + if _, exists := seenIndices[index]; exists { + return nil, fmt.Errorf("invalid sgl rerank response: duplicate index %d", index) + } + seenIndices[index] = struct{}{} + + score, ok := schemas.SafeExtractFloat64(itemMap["score"]) + if !ok { + return nil, fmt.Errorf("invalid sgl rerank response: score is required") + } + + result := schemas.RerankResult{ + Index: index, + RelevanceScore: score, + } + + if returnDocuments { + doc := documents[index] + result.Document = &doc + } + + response.Results = append(response.Results, result) + } + + sort.SliceStable(response.Results, func(i, j int) bool { + if response.Results[i].RelevanceScore == response.Results[j].RelevanceScore { + return response.Results[i].Index < response.Results[j].Index + } + return response.Results[i].RelevanceScore > response.Results[j].RelevanceScore + }) + + return response, nil +} + +// callSGLRerankEndpoint POSTs to sglang's /v1/rerank and decodes the bare-array +// response. sglang only serves /v1/rerank, so unlike vLLM there is no +// /rerank fallback path. +func (provider *SGLProvider) callSGLRerankEndpoint( + ctx *schemas.BifrostContext, + key schemas.Key, + endpointPath string, + jsonData []byte, +) ([]interface{}, []byte, time.Duration, *schemas.BifrostError) { + baseURL, bifrostErr := provider.baseURLOrError(key) + if bifrostErr != nil { + return nil, nil, 0, bifrostErr + } + + req := fasthttp.AcquireRequest() + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseRequest(req) + defer fasthttp.ReleaseResponse(resp) + + providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) + + req.SetRequestURI(baseURL + endpointPath) + req.Header.SetMethod(http.MethodPost) + req.Header.SetContentType("application/json") + + if key.Value.GetValue() != "" { + req.Header.Set("Authorization", "Bearer "+key.Value.GetValue()) + } + if !providerUtils.ApplyLargePayloadRequestBodyWithModelNormalization(ctx, req, schemas.SGL) { + req.SetBody(jsonData) + } + + latency, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, provider.client, req, resp) + defer wait() + if bifrostErr != nil { + return nil, nil, latency, bifrostErr + } + + if resp.StatusCode() != fasthttp.StatusOK { + rawErrBody := append([]byte(nil), resp.Body()...) + return nil, rawErrBody, latency, ParseSGLError(resp) + } + + body, err := providerUtils.CheckAndDecodeBody(resp) + if err != nil { + rawErrBody := append([]byte(nil), resp.Body()...) + return nil, rawErrBody, latency, providerUtils.NewBifrostOperationError(schemas.ErrProviderResponseDecode, err) + } + + var items []interface{} + if err := sonic.Unmarshal(body, &items); err != nil { + return nil, body, latency, providerUtils.NewBifrostOperationError(schemas.ErrProviderResponseUnmarshal, err) + } + + return items, body, latency, nil +} + +// Rerank performs a rerank request to sglang's /v1/rerank endpoint. +func (provider *SGLProvider) Rerank(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostRerankRequest) (*schemas.BifrostRerankResponse, *schemas.BifrostError) { + // Opt into ExtraParams pass-through so caller-supplied request.Params.ExtraParams + // are merged into the outgoing JSON body. Mirrors vLLM's Rerank wiring. + ctx.SetValue(schemas.BifrostContextKeyPassthroughExtraParams, true) + + jsonData, bifrostErr := providerUtils.CheckContextAndGetRequestBody( + ctx, + request, + func() (providerUtils.RequestBodyWithExtraParams, error) { + return ToSGLRerankRequest(request), nil + }) + if bifrostErr != nil { + return nil, bifrostErr + } + + resolvedPath := providerUtils.GetPathFromContext(ctx, "") + if resolvedPath == "" { + resolvedPath = "/v1/rerank" + } else if !strings.HasPrefix(resolvedPath, "/") { + resolvedPath = "/" + resolvedPath + } + + sendBackRawRequest := providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest) + sendBackRawResponse := providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse) + + items, responseBody, latency, bifrostErr := provider.callSGLRerankEndpoint(ctx, key, resolvedPath, jsonData) + if bifrostErr != nil { + return nil, providerUtils.EnrichError(ctx, bifrostErr, jsonData, responseBody, sendBackRawRequest, sendBackRawResponse) + } + + returnDocuments := request.Params != nil && request.Params.ReturnDocuments != nil && *request.Params.ReturnDocuments + bifrostResponse, err := ToBifrostRerankResponse(items, request.Documents, returnDocuments) + if err != nil { + return nil, providerUtils.EnrichError( + ctx, + providerUtils.NewBifrostOperationError("error converting rerank response", err), + jsonData, + responseBody, + sendBackRawRequest, + sendBackRawResponse, + ) + } + + // Keep requested model as the canonical model in Bifrost response, + // since sglang's bare-array response does not include one. + bifrostResponse.Model = request.Model + bifrostResponse.ExtraFields.Latency = latency.Milliseconds() + + if sendBackRawRequest { + var rawReq interface{} + if err := sonic.Unmarshal(jsonData, &rawReq); err == nil { + bifrostResponse.ExtraFields.RawRequest = rawReq + } + } + if sendBackRawResponse { + bifrostResponse.ExtraFields.RawResponse = items + } + + return bifrostResponse, nil +} diff --git a/core/providers/sgl/rerank_live_test.go b/core/providers/sgl/rerank_live_test.go new file mode 100644 index 00000000000..5dd3fc39c9b --- /dev/null +++ b/core/providers/sgl/rerank_live_test.go @@ -0,0 +1,220 @@ +package sgl + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "testing" + + schemas "github.com/maximhq/bifrost/core/schemas" +) + +// TestRerankLive_HappyPath_ExtraParamsForwarded exercises the FULL Rerank() code +// path against an httptest server that mimics sglang's /v1/rerank wire shape. +// +// This is the live functional check for Fix 1: provider must set +// BifrostContextKeyPassthroughExtraParams=true so caller-supplied +// request.Params.ExtraParams are merged into the outgoing JSON body. +func TestRerankLive_HappyPath_ExtraParamsForwarded(t *testing.T) { + t.Parallel() + + var ( + gotPath string + gotMethod string + gotAuth string + gotCT string + gotBody map[string]interface{} + ) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotMethod = r.Method + gotAuth = r.Header.Get("Authorization") + gotCT = r.Header.Get("Content-Type") + + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "read err", http.StatusInternalServerError) + return + } + if err := json.Unmarshal(body, &gotBody); err != nil { + http.Error(w, "json err: "+err.Error(), http.StatusInternalServerError) + return + } + + // sglang returns a bare JSON array, score (not relevance_score), no model field. + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `[ + {"index": 0, "score": 0.91, "document": {"text": "alpha"}}, + {"index": 1, "score": 0.42, "document": {"text": "beta"}}, + {"index": 2, "score": 0.77, "document": {"text": "gamma"}} + ]`) + })) + defer server.Close() + + provider := newTestSGLProvider() + key := schemas.Key{ + ID: "live-key", + Value: schemas.EnvVar{Val: "live-api-key"}, + SGLKeyConfig: &schemas.SGLKeyConfig{ + URL: schemas.EnvVar{Val: server.URL}, + }, + } + + // Caller does NOT set BifrostContextKeyPassthroughExtraParams — Rerank() must set it itself. + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + + topN := 3 + returnDocs := true + req := &schemas.BifrostRerankRequest{ + Provider: schemas.SGL, + Model: "bge-reranker-v2-m3", + Query: "what is bifrost", + Documents: []schemas.RerankDocument{{Text: "alpha"}, {Text: "beta"}, {Text: "gamma"}}, + Params: &schemas.RerankParameters{ + TopN: &topN, + ReturnDocuments: &returnDocs, + ExtraParams: map[string]interface{}{ + "custom_sgl_flag": "should-make-it-through", + "numeric_knob": float64(7), + }, + }, + } + + resp, bifrostErr := provider.Rerank(ctx, key, req) + if bifrostErr != nil { + t.Fatalf("Rerank returned error: %v", bifrostErr.Error.Message) + } + + // --- Request-side assertions (Fix 1: ExtraParams forwarded) --- + if gotMethod != http.MethodPost { + t.Fatalf("expected POST, got %s", gotMethod) + } + if gotPath != "/v1/rerank" { + t.Fatalf("expected /v1/rerank, got %s", gotPath) + } + if gotAuth != "Bearer live-api-key" { + t.Fatalf("expected Authorization header set, got %q", gotAuth) + } + if gotCT != "application/json" { + t.Fatalf("expected JSON content-type, got %q", gotCT) + } + if _, hasModel := gotBody["model"]; hasModel { + t.Fatalf("outgoing body must NOT include `model` (sglang rejects unknown fields); got: %v", gotBody) + } + if gotBody["query"] != "what is bifrost" { + t.Fatalf("query missing/wrong in outgoing body: %v", gotBody["query"]) + } + if gotBody["custom_sgl_flag"] != "should-make-it-through" { + t.Fatalf("Fix 1 regression: ExtraParams.custom_sgl_flag missing from outgoing body. Got keys: %v", keysOf(gotBody)) + } + if gotBody["numeric_knob"] != float64(7) { + t.Fatalf("Fix 1 regression: ExtraParams.numeric_knob missing/wrong: %v", gotBody["numeric_knob"]) + } + + // --- Response-side assertions --- + if resp == nil || len(resp.Results) != 3 { + t.Fatalf("expected 3 results, got %+v", resp) + } + // Sorted descending by score: 0.91, 0.77, 0.42 → indices 0, 2, 1 + wantOrder := []int{0, 2, 1} + for i, want := range wantOrder { + if resp.Results[i].Index != want { + t.Fatalf("result[%d] index = %d, want %d (results not sorted by score desc)", i, resp.Results[i].Index, want) + } + } + if resp.Results[0].RelevanceScore != 0.91 { + t.Fatalf("top score = %v, want 0.91", resp.Results[0].RelevanceScore) + } + if resp.Results[0].Document == nil || resp.Results[0].Document.Text != "alpha" { + t.Fatalf("expected returned document for top result, got %+v", resp.Results[0].Document) + } + if resp.Model != "bge-reranker-v2-m3" { + t.Fatalf("response model = %q, want canonical request model", resp.Model) + } +} + +// TestRerankLive_GzipErrorPath exercises the FULL Rerank() error path against +// an httptest server that returns a gzip-encoded sglang error envelope with a +// 4xx status. +// +// This is the live functional check for Fix 2: ParseSGLError must read from +// the already-decoded body (ExtraFields.RawResponse) rather than re-snapshotting +// resp.Body() (which is still gzip-compressed at error time). +func TestRerankLive_GzipErrorPath(t *testing.T) { + t.Parallel() + + // sglang flat error envelope, gzipped. + envelope := `{"object":"error","message":"the input is longer than the model's context length","type":"invalid_request_error","code":400}` + var gzbuf bytes.Buffer + gw := gzip.NewWriter(&gzbuf) + if _, err := gw.Write([]byte(envelope)); err != nil { + t.Fatalf("gzip write: %v", err) + } + if err := gw.Close(); err != nil { + t.Fatalf("gzip close: %v", err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Encoding", "gzip") + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write(gzbuf.Bytes()) + })) + defer server.Close() + + provider := newTestSGLProvider() + key := schemas.Key{ + ID: "live-key", + Value: schemas.EnvVar{Val: "live-api-key"}, + SGLKeyConfig: &schemas.SGLKeyConfig{ + URL: schemas.EnvVar{Val: server.URL}, + }, + } + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + + req := &schemas.BifrostRerankRequest{ + Provider: schemas.SGL, + Model: "bge-reranker-v2-m3", + Query: "x", + Documents: []schemas.RerankDocument{{Text: "a"}, {Text: "b"}}, + } + + resp, bifrostErr := provider.Rerank(ctx, key, req) + if bifrostErr == nil { + t.Fatalf("expected error from gzipped 400, got success: %+v", resp) + } + + msg := bifrostErr.Error.Message + if msg == "" { + t.Fatal("Fix 2 regression: error message empty — gzipped body likely not decoded") + } + // Substring mapping should resolve to context_length_exceeded. + if bifrostErr.Error.Code == nil || *bifrostErr.Error.Code != "context_length_exceeded" { + gotCode := "" + if bifrostErr.Error.Code != nil { + gotCode = *bifrostErr.Error.Code + } + t.Fatalf("expected code=context_length_exceeded from substring map, got code=%q msg=%q", gotCode, msg) + } + if !bytesContains(msg, "context length") { + t.Fatalf("Fix 2 regression: decoded error message lost. Got: %q", msg) + } +} + +func keysOf(m map[string]interface{}) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} + +func bytesContains(s, sub string) bool { + return bytes.Contains([]byte(s), []byte(sub)) +} diff --git a/core/providers/sgl/rerank_test.go b/core/providers/sgl/rerank_test.go new file mode 100644 index 00000000000..a2b1104ff34 --- /dev/null +++ b/core/providers/sgl/rerank_test.go @@ -0,0 +1,191 @@ +package sgl + +import ( + "strings" + "testing" + + "github.com/bytedance/sonic" + "github.com/maximhq/bifrost/core/schemas" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestToSGLRerankRequestNil(t *testing.T) { + t.Parallel() + req := ToSGLRerankRequest(nil) + assert.Nil(t, req) +} + +// TestToSGLRerankRequest_NoModelField is the load-bearing test for this PR: +// sglang's V1RerankReqInput rejects unknown fields including `model`, so the +// converter must never emit one even when the Bifrost request has Model set. +func TestToSGLRerankRequest_NoModelField(t *testing.T) { + t.Parallel() + + topN := 3 + returnDocs := true + + req := ToSGLRerankRequest(&schemas.BifrostRerankRequest{ + Model: "BAAI/bge-reranker-v2-m3", + Query: "what is machine learning", + Documents: []schemas.RerankDocument{ + {Text: "Machine learning is a subset of AI."}, + {Text: "The weather is sunny."}, + }, + Params: &schemas.RerankParameters{ + TopN: &topN, + ReturnDocuments: &returnDocs, + ExtraParams: map[string]interface{}{ + "user": "test-user", + }, + }, + }) + + require.NotNil(t, req) + assert.Equal(t, "what is machine learning", req.Query) + assert.Equal(t, []string{"Machine learning is a subset of AI.", "The weather is sunny."}, req.Documents) + require.NotNil(t, req.TopN) + assert.Equal(t, 3, *req.TopN) + require.NotNil(t, req.ReturnDocuments) + assert.True(t, *req.ReturnDocuments) + assert.Equal(t, "test-user", req.ExtraParams["user"]) + + // Serialize and verify no `model` field is present in the wire body. + body, err := sonic.Marshal(req) + require.NoError(t, err) + var asMap map[string]interface{} + require.NoError(t, sonic.Unmarshal(body, &asMap)) + _, hasModel := asMap["model"] + assert.False(t, hasModel, "sglang /v1/rerank rejects the `model` field; converter must not emit it. body=%s", string(body)) + + // Spot-check expected fields exist. + assert.Contains(t, asMap, "query") + assert.Contains(t, asMap, "documents") + assert.Contains(t, asMap, "top_n") + assert.Contains(t, asMap, "return_documents") +} + +func TestToSGLRerankRequest_OmitsOptionalFields(t *testing.T) { + t.Parallel() + + req := ToSGLRerankRequest(&schemas.BifrostRerankRequest{ + Model: "BAAI/bge-reranker-v2-m3", + Query: "q", + Documents: []schemas.RerankDocument{{Text: "d"}}, + }) + require.NotNil(t, req) + body, err := sonic.Marshal(req) + require.NoError(t, err) + assert.NotContains(t, string(body), "top_n") + assert.NotContains(t, string(body), "return_documents") +} + +// TestToBifrostRerankResponse_BareArray verifies parsing of sglang's +// distinctive bare-array response shape using the `score` field. +func TestToBifrostRerankResponse_BareArray(t *testing.T) { + t.Parallel() + + documents := []schemas.RerankDocument{ + {Text: "doc-0"}, + {Text: "doc-1"}, + {Text: "doc-2"}, + } + + // sglang returns a bare JSON array. Simulate decoding into []interface{}. + const raw = `[ + {"index": 1, "score": 0.1, "document": "doc-1"}, + {"index": 0, "score": 0.9, "document": "doc-0"}, + {"index": 2, "score": 0.5, "document": "doc-2"} + ]` + var items []interface{} + require.NoError(t, sonic.Unmarshal([]byte(raw), &items)) + + response, err := ToBifrostRerankResponse(items, documents, true) + require.NoError(t, err) + require.NotNil(t, response) + require.Len(t, response.Results, 3) + + // Sorted descending by score. + assert.Equal(t, 0, response.Results[0].Index) + assert.Equal(t, 0.9, response.Results[0].RelevanceScore) + require.NotNil(t, response.Results[0].Document) + assert.Equal(t, "doc-0", response.Results[0].Document.Text) + + assert.Equal(t, 2, response.Results[1].Index) + assert.Equal(t, 0.5, response.Results[1].RelevanceScore) + + assert.Equal(t, 1, response.Results[2].Index) + assert.Equal(t, 0.1, response.Results[2].RelevanceScore) +} + +func TestToBifrostRerankResponse_OmitsDocumentsWhenNotRequested(t *testing.T) { + t.Parallel() + + documents := []schemas.RerankDocument{{Text: "doc-0"}} + items := []interface{}{ + map[string]interface{}{"index": 0, "score": 0.42}, + } + + response, err := ToBifrostRerankResponse(items, documents, false) + require.NoError(t, err) + require.NotNil(t, response) + require.Len(t, response.Results, 1) + assert.Nil(t, response.Results[0].Document) + assert.Equal(t, 0.42, response.Results[0].RelevanceScore) +} + +func TestToBifrostRerankResponse_DuplicateIndices(t *testing.T) { + t.Parallel() + + documents := []schemas.RerankDocument{{Text: "doc-0"}, {Text: "doc-1"}} + items := []interface{}{ + map[string]interface{}{"index": 0, "score": 0.9}, + map[string]interface{}{"index": 0, "score": 0.8}, + } + + _, err := ToBifrostRerankResponse(items, documents, true) + require.Error(t, err) + assert.True(t, strings.Contains(err.Error(), "duplicate index")) +} + +func TestToBifrostRerankResponse_OutOfRangeIndex(t *testing.T) { + t.Parallel() + + documents := []schemas.RerankDocument{{Text: "doc-0"}} + items := []interface{}{ + map[string]interface{}{"index": 1, "score": 0.9}, + } + + _, err := ToBifrostRerankResponse(items, documents, true) + require.Error(t, err) + assert.True(t, strings.Contains(err.Error(), "out of range")) +} + +func TestToBifrostRerankResponse_MissingScore(t *testing.T) { + t.Parallel() + + documents := []schemas.RerankDocument{{Text: "doc-0"}} + items := []interface{}{ + map[string]interface{}{"index": 0}, + } + + _, err := ToBifrostRerankResponse(items, documents, false) + require.Error(t, err) + assert.True(t, strings.Contains(err.Error(), "score is required")) +} + +func TestToBifrostRerankResponse_NilItems(t *testing.T) { + t.Parallel() + + _, err := ToBifrostRerankResponse(nil, []schemas.RerankDocument{{Text: "d"}}, false) + require.Error(t, err) +} + +func TestToBifrostRerankResponse_EmptyResults(t *testing.T) { + t.Parallel() + + response, err := ToBifrostRerankResponse([]interface{}{}, []schemas.RerankDocument{{Text: "d"}}, false) + require.NoError(t, err) + require.NotNil(t, response) + assert.Len(t, response.Results, 0) +} diff --git a/core/providers/sgl/sgl.go b/core/providers/sgl/sgl.go index e35885983bb..d3af860bb85 100644 --- a/core/providers/sgl/sgl.go +++ b/core/providers/sgl/sgl.go @@ -142,7 +142,7 @@ func (provider *SGLProvider) TextCompletion(ctx *schemas.BifrostContext, key sch providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest), providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse), nil, - nil, + ParseSGLError, provider.logger, ) } @@ -167,7 +167,7 @@ func (provider *SGLProvider) TextCompletionStream(ctx *schemas.BifrostContext, p providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest), providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse), provider.GetProviderKey(), - nil, + ParseSGLError, postHookRunner, nil, nil, @@ -194,7 +194,7 @@ func (provider *SGLProvider) ChatCompletion(ctx *schemas.BifrostContext, key sch providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse), provider.GetProviderKey(), nil, - nil, + ParseSGLError, provider.logger, ) } @@ -224,7 +224,7 @@ func (provider *SGLProvider) ChatCompletionStream(ctx *schemas.BifrostContext, p postHookRunner, nil, nil, - nil, + ParseSGLError, nil, nil, provider.logger, @@ -282,10 +282,7 @@ func (provider *SGLProvider) Speech(ctx *schemas.BifrostContext, key schemas.Key return nil, providerUtils.NewUnsupportedOperationError(schemas.SpeechRequest, provider.GetProviderKey()) } -// Rerank is not supported by the SGL provider. -func (provider *SGLProvider) Rerank(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostRerankRequest) (*schemas.BifrostRerankResponse, *schemas.BifrostError) { - return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) -} +// Rerank is implemented in rerank.go (sglang /v1/rerank). // OCR is not supported by the Sgl provider. func (provider *SGLProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { diff --git a/core/schemas/bifrost.go b/core/schemas/bifrost.go index 7be0acd8101..6642af54c09 100644 --- a/core/schemas/bifrost.go +++ b/core/schemas/bifrost.go @@ -69,6 +69,7 @@ var SupportedBaseProviders = []ModelProvider{ OpenAI, HuggingFace, Replicate, + SGL, } // StandardProviders is the list of all built-in (non-custom) providers. diff --git a/ui/app/workspace/providers/dialogs/addNewCustomProviderSheet.tsx b/ui/app/workspace/providers/dialogs/addNewCustomProviderSheet.tsx index f3728bc32aa..423db25028d 100644 --- a/ui/app/workspace/providers/dialogs/addNewCustomProviderSheet.tsx +++ b/ui/app/workspace/providers/dialogs/addNewCustomProviderSheet.tsx @@ -165,6 +165,7 @@ export function AddCustomProviderSheetContent({ show = true, onClose, onSave }: Cohere AWS Bedrock Replicate + SGLang diff --git a/ui/app/workspace/providers/fragments/allowedRequestsFields.tsx b/ui/app/workspace/providers/fragments/allowedRequestsFields.tsx index 5fbd46302f0..2fb655bb025 100644 --- a/ui/app/workspace/providers/fragments/allowedRequestsFields.tsx +++ b/ui/app/workspace/providers/fragments/allowedRequestsFields.tsx @@ -71,6 +71,7 @@ const RequestTypes: Array<{ key: RequestType; label: string }> = [ { key: "responses", label: "Responses" }, { key: "responses_stream", label: "Responses Stream" }, { key: "embedding", label: "Embedding" }, + { key: "rerank", label: "Rerank" }, { key: "speech", label: "Speech" }, { key: "speech_stream", label: "Speech Stream" }, { key: "transcription", label: "Transcription" }, diff --git a/ui/app/workspace/providers/fragments/apiKeysFormFragment.tsx b/ui/app/workspace/providers/fragments/apiKeysFormFragment.tsx index d841c270715..7d6dbc44f35 100644 --- a/ui/app/workspace/providers/fragments/apiKeysFormFragment.tsx +++ b/ui/app/workspace/providers/fragments/apiKeysFormFragment.tsx @@ -46,6 +46,7 @@ interface Props { control: Control; providerName: string; form: UseFormReturn; + baseProviderType?: string; } // Batch API form field for all providers @@ -71,14 +72,16 @@ function BatchAPIFormField({ control }: { control: Control; form: UseFormRe ); } -export function ApiKeyFormFragment({ control, providerName, form }: Props) { +export function ApiKeyFormFragment({ control, providerName, form, baseProviderType }: Props) { const isBedrock = providerName === "bedrock"; const isVertex = providerName === "vertex"; const isAzure = providerName === "azure"; const isReplicate = providerName === "replicate"; const isVLLM = providerName === "vllm"; const isOllama = providerName === "ollama"; - const isSGL = providerName === "sgl"; + // SGL applies to both the built-in `sgl` provider and any custom provider + // whose base_provider_type is `sgl`. + const isSGL = providerName === "sgl" || baseProviderType === "sgl"; const isKeylessProvider = isOllama || isSGL; const supportsBatchAPI = BATCH_SUPPORTED_PROVIDERS.includes(providerName); diff --git a/ui/app/workspace/providers/fragments/apiStructureFormFragment.tsx b/ui/app/workspace/providers/fragments/apiStructureFormFragment.tsx index 7721c43eb8c..4981ddb1c92 100644 --- a/ui/app/workspace/providers/fragments/apiStructureFormFragment.tsx +++ b/ui/app/workspace/providers/fragments/apiStructureFormFragment.tsx @@ -117,6 +117,7 @@ export function ApiStructureFormFragment({ provider }: Props) { Cohere Gemini Replicate + SGLang The underlying provider this custom provider will use diff --git a/ui/app/workspace/providers/views/providerKeyForm.tsx b/ui/app/workspace/providers/views/providerKeyForm.tsx index 0d564a2bdc8..f55cce998c6 100644 --- a/ui/app/workspace/providers/views/providerKeyForm.tsx +++ b/ui/app/workspace/providers/views/providerKeyForm.tsx @@ -123,7 +123,12 @@ export default function ProviderKeyForm({ provider, keyId, onCancel, onSave }: P
- + {isEditing && currentKey?.config_hash && }
diff --git a/ui/lib/constants/config.ts b/ui/lib/constants/config.ts index a3778f093ae..b4fbea66212 100644 --- a/ui/lib/constants/config.ts +++ b/ui/lib/constants/config.ts @@ -194,6 +194,17 @@ export const PROVIDER_SUPPORTED_REQUESTS: Record = { "responses_stream", "embedding", ], + sgl: [ + "list_models", + "text_completion", + "text_completion_stream", + "chat_completion", + "chat_completion_stream", + "responses", + "responses_stream", + "embedding", + "rerank", + ], }; export const IS_ENTERPRISE = process.env.BIFROST_IS_ENTERPRISE === "true"; diff --git a/ui/lib/types/config.ts b/ui/lib/types/config.ts index 99fad2e877b..033640223bf 100644 --- a/ui/lib/types/config.ts +++ b/ui/lib/types/config.ts @@ -7,7 +7,7 @@ import { EnvVar } from "./schemas"; export type KnownProvider = (typeof KnownProvidersNames)[number]; // Base provider names - all supported base providers -export type BaseProvider = "openai" | "anthropic" | "cohere" | "gemini" | "bedrock" | "replicate" | "fireworks"; +export type BaseProvider = "openai" | "anthropic" | "cohere" | "gemini" | "bedrock" | "replicate" | "fireworks" | "sgl"; // Branded type for custom provider names to prevent collision with known providers export type CustomProviderName = string & { readonly __brand: "CustomProviderName" }; From 0240a9755c75b7984bc26b7ee61501e58ed1b780 Mon Sep 17 00:00:00 2001 From: Metbcy Date: Tue, 2 Jun 2026 21:47:53 +0000 Subject: [PATCH 3/3] Address CodeRabbit & Greptile bot feedback - Hide rerank from Add Custom Provider dialog via hideAdvancedTypes prop; keeps default in form state but removes from UI to satisfy the scoped-fields requirement (CodeRabbit) - Guard flat sglang error envelope detection on object=="error" so a non-sglang JSON proxy response with a top-level message can't hijack the error mapping (Greptile P2) + regression test - Add sgl entry to ProviderEndpoints with /v1/rerank so the path override placeholder matches the provider (Greptile P2) --- core/providers/sgl/errors.go | 8 ++--- core/providers/sgl/errors_test.go | 21 ++++++++++++ .../dialogs/addNewCustomProviderSheet.tsx | 1 + .../fragments/allowedRequestsFields.tsx | 32 +++++++++++++++++-- 4 files changed, 54 insertions(+), 8 deletions(-) diff --git a/core/providers/sgl/errors.go b/core/providers/sgl/errors.go index d2cab408792..8eeea023fe3 100644 --- a/core/providers/sgl/errors.go +++ b/core/providers/sgl/errors.go @@ -54,7 +54,7 @@ func ParseSGLError(resp *fasthttp.Response) *schemas.BifrostError { currentMsg := strings.TrimSpace(bifrostErr.Error.Message) wrappedHadMessage := currentMsg != "" && !strings.HasPrefix(currentMsg, "provider API error") if !wrappedHadMessage { - if flat, ok := extractFlatSGLErrorFromRaw(bifrostErr.ExtraFields.RawResponse); ok && flat.Message != "" { + if flat, ok := extractFlatSGLErrorFromRaw(bifrostErr.ExtraFields.RawResponse); ok && flat.Message != "" && flat.Object == "error" { bifrostErr.Error.Message = flat.Message if flat.Type != "" { t := flat.Type @@ -82,10 +82,8 @@ func ParseSGLError(resp *fasthttp.Response) *schemas.BifrostError { } func setSGLErrorCode(field *schemas.ErrorField, code, typ string) { - c := code - t := typ - field.Code = &c - field.Type = &t + field.Code = schemas.Ptr(code) + field.Type = schemas.Ptr(typ) } // extractFlatSGLErrorFromRaw pulls a flat sglang error envelope out of the diff --git a/core/providers/sgl/errors_test.go b/core/providers/sgl/errors_test.go index fe387f9219a..bb9d45fd7d7 100644 --- a/core/providers/sgl/errors_test.go +++ b/core/providers/sgl/errors_test.go @@ -176,6 +176,27 @@ func TestParseSGLError_GzipFlatEnvelope(t *testing.T) { assert.Equal(t, "server_error", strDeref(bifrostErr.Error.Type)) } +// TestParseSGLError_NonErrorObjectIgnored verifies that a JSON object response +// missing `"object":"error"` is NOT treated as a flat sglang error envelope, +// even if it happens to have a top-level "message" key. This guards against +// a sidecar/proxy in front of sglang accidentally hijacking the error mapping. +func TestParseSGLError_NonErrorObjectIgnored(t *testing.T) { + t.Parallel() + + // Object without `"object":"error"` — e.g. some proxy's own 4xx envelope. + body := `{"message":"proxy denied request","type":"ProxyError","code":403}` + resp := buildSGLErrorResponse(403, body) + defer fasthttp.ReleaseResponse(resp) + + bifrostErr := ParseSGLError(resp) + require.NotNil(t, bifrostErr) + require.NotNil(t, bifrostErr.Error) + // The flat-envelope path should be skipped, so the message should NOT be + // "proxy denied request" — the wrapper's default ("provider API error ...") + // stays in place. + assert.NotEqual(t, "proxy denied request", bifrostErr.Error.Message) +} + // jsonString minimally escapes a Go string for embedding in a JSON literal. // Only handles characters used by the test fixtures. func jsonString(s string) string { diff --git a/ui/app/workspace/providers/dialogs/addNewCustomProviderSheet.tsx b/ui/app/workspace/providers/dialogs/addNewCustomProviderSheet.tsx index 423db25028d..91d1c4d366b 100644 --- a/ui/app/workspace/providers/dialogs/addNewCustomProviderSheet.tsx +++ b/ui/app/workspace/providers/dialogs/addNewCustomProviderSheet.tsx @@ -226,6 +226,7 @@ export function AddCustomProviderSheetContent({ show = true, onClose, onSave }: control={form.control} providerType={form.watch("baseFormat") as BaseProvider} disabled={!hasProviderCreateAccess} + hideAdvancedTypes />
diff --git a/ui/app/workspace/providers/fragments/allowedRequestsFields.tsx b/ui/app/workspace/providers/fragments/allowedRequestsFields.tsx index 2fb655bb025..829a9d16966 100644 --- a/ui/app/workspace/providers/fragments/allowedRequestsFields.tsx +++ b/ui/app/workspace/providers/fragments/allowedRequestsFields.tsx @@ -15,8 +15,18 @@ interface AllowedRequestsFieldsProps { pathOverridesPrefix?: string; providerType?: BaseProvider; disabled?: boolean; + // When true, advanced request types (rerank and the other non-core types) + // are hidden from the UI but still kept in form defaults. Used by the + // Add Custom Provider dialog to keep the dialog scoped. + hideAdvancedTypes?: boolean; } +// Advanced request types that should be hidden from the Add Custom Provider +// dialog to avoid clutter. Defaults are still preserved in form state. +const AdvancedRequestTypes: ReadonlySet = new Set([ + "rerank", +]); + // Provider-specific endpoint paths const ProviderEndpoints: Partial>>> = { openai: { @@ -52,6 +62,15 @@ const ProviderEndpoints: Partial (hideAdvancedTypes ? RequestTypes.filter(({ key }) => !AdvancedRequestTypes.has(key)) : RequestTypes), + [hideAdvancedTypes], + ); + const leftColumn = visibleRequestTypes.slice(0, Math.ceil(visibleRequestTypes.length / 2)); + const rightColumn = visibleRequestTypes.slice(Math.ceil(visibleRequestTypes.length / 2)); const { getValues, setValue } = useFormContext(); - // Reset disabled fields when providerType changes + // Reset disabled fields when providerType changes. Iterates over the full + // RequestTypes list (not visibleRequestTypes) so defaults stay in form + // state even when an entry is hidden from the UI. useEffect(() => { RequestTypes.forEach(({ key }) => { const fieldName = `${namePrefix}.${key}`;