feat(sgl): SGLang as base provider with rerank + error translation (#3131 part 1)

core/providers/sgl/errors.go

@@ -0,0 +1,135 @@
+package sgl
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/bytedance/sonic"
+	"github.com/maximhq/bifrost/core/providers/openai"
+	"github.com/maximhq/bifrost/core/schemas"
+	"github.com/valyala/fasthttp"
+)
+
+// sglFlatError matches sglang's flat error envelope:
+//
+//	{"object":"error","message":"...","type":"BadRequestError","code":400}
+//
+// See python/sglang/srt/entrypoints/openai/serving_base.py upstream
+// (function create_error_response, lines ~209-225).
+type sglFlatError struct {
+	Object  string      `json:"object"`
+	Message string      `json:"message"`
+	Type    string      `json:"type"`
+	Code    interface{} `json:"code"`
+}
+
+// ParseSGLError parses sglang error responses.
+//
+// It handles both error envelope shapes used by sglang and its forks:
+//   - Flat:    {"object":"error","message":"...","type":"BadRequestError","code":400}
+//   - Wrapped: {"error":{"message":"...","type":"...","code":"..."}}
+//
+// Well-known message substrings are mapped to OpenAI-style codes:
+//   - "longer than the model's context length" -> context_length_exceeded / invalid_request_error
+//   - "out of memory"                          -> out_of_memory          / server_error
+//   - "model is not loaded"                    -> model_not_found        / invalid_request_error
+//
+// The raw sglang `message` is always preserved on the returned BifrostError so
+// callers see the actual server explanation rather than a generic "status N".
+func ParseSGLError(resp *fasthttp.Response) *schemas.BifrostError {
+	// Delegate to the shared OpenAI-shape parser first. This handles the
+	// wrapped {"error":{...}} envelope, decodes gzip bodies, fills in
+	// StatusCode / ExtraFields, and applies sane HTTP-status fallbacks.
+	// The decoded body is stashed on ExtraFields.RawResponse, so we read
+	// from there rather than re-snapshotting resp.Body() (which would be
+	// the still-compressed bytes if Content-Encoding was gzip).
+	bifrostErr := openai.ParseOpenAIError(resp)
+	if bifrostErr.Error == nil {
+		bifrostErr.Error = &schemas.ErrorField{}
+	}
+
+	// If the wrapped parser did not pick up a useful message, try sglang's
+	// flat envelope. We treat the generic "provider API error (status N)" /
+	// "provider API error" fallback as "no useful message" for this purpose.
+	currentMsg := strings.TrimSpace(bifrostErr.Error.Message)
+	wrappedHadMessage := currentMsg != "" && !strings.HasPrefix(currentMsg, "provider API error")
+	if !wrappedHadMessage {
+		if flat, ok := extractFlatSGLErrorFromRaw(bifrostErr.ExtraFields.RawResponse); ok && flat.Message != "" && flat.Object == "error" {
+			bifrostErr.Error.Message = flat.Message
+			if flat.Type != "" {
+				t := flat.Type
+				bifrostErr.Error.Type = &t
+			}
+			if flat.Code != nil {
+				if codeStr := codeToString(flat.Code); codeStr != "" {
+					bifrostErr.Error.Code = &codeStr
+				}
+			}
+		}
+	}
+
+	msg := bifrostErr.Error.Message
+	switch {
+	case strings.Contains(msg, "longer than the model's context length"):
+		setSGLErrorCode(bifrostErr.Error, "context_length_exceeded", "invalid_request_error")
+	case strings.Contains(msg, "out of memory"):
+		setSGLErrorCode(bifrostErr.Error, "out_of_memory", "server_error")
+	case strings.Contains(msg, "model is not loaded"):
+		setSGLErrorCode(bifrostErr.Error, "model_not_found", "invalid_request_error")
+	}
+
+	return bifrostErr
+}
+
+func setSGLErrorCode(field *schemas.ErrorField, code, typ string) {
+	field.Code = schemas.Ptr(code)
+	field.Type = schemas.Ptr(typ)
+}
+
+// extractFlatSGLErrorFromRaw pulls a flat sglang error envelope out of the
+// already-decoded body that openai.ParseOpenAIError stashed on
+// ExtraFields.RawResponse. RawResponse may be a string (when JSON parsing
+// failed upstream) or a map[string]interface{} (the parsed body). We handle
+// both so a gzipped 4xx still yields the sglang message/type/code rather
+// than just the generic HTTP-status fallback.
+func extractFlatSGLErrorFromRaw(raw interface{}) (sglFlatError, bool) {
+	switch v := raw.(type) {
+	case string:
+		var flat sglFlatError
+		if err := sonic.Unmarshal([]byte(v), &flat); err == nil {
+			return flat, true
+		}
+	case map[string]interface{}:
+		flat := sglFlatError{}
+		if s, ok := v["object"].(string); ok {
+			flat.Object = s
+		}
+		if s, ok := v["message"].(string); ok {
+			flat.Message = s
+		}
+		if s, ok := v["type"].(string); ok {
+			flat.Type = s
+		}
+		if c, ok := v["code"]; ok {
+			flat.Code = c
+		}
+		return flat, true
+	}
+	return sglFlatError{}, false
+}
+
+func codeToString(v interface{}) string {
+	switch x := v.(type) {
+	case string:
+		return x
+	case float64:
+		return fmt.Sprintf("%d", int(x))
+	case float32:
+		return fmt.Sprintf("%d", int(x))
+	case int:
+		return fmt.Sprintf("%d", x)
+	case int64:
+		return fmt.Sprintf("%d", x)
+	}
+	return ""
+}

core/providers/sgl/errors_test.go

@@ -0,0 +1,222 @@
+package sgl
+
+import (
+	"bytes"
+	"compress/gzip"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/valyala/fasthttp"
+)
+
+// buildSGLErrorResponse creates a fasthttp.Response with the given status code
+// and body. Returned as a helper to keep tests focused on error parsing logic.
+func buildSGLErrorResponse(status int, body string) *fasthttp.Response {
+	resp := fasthttp.AcquireResponse()
+	resp.SetStatusCode(status)
+	resp.Header.SetContentType("application/json")
+	resp.SetBodyString(body)
+	return resp
+}
+
+// strDeref returns the dereferenced value or empty string for nil.
+func strDeref(p *string) string {
+	if p == nil {
+		return ""
+	}
+	return *p
+}
+
+func TestParseSGLError_FlatEnvelope(t *testing.T) {
+	t.Parallel()
+
+	resp := buildSGLErrorResponse(400, `{"object":"error","message":"bad request","type":"BadRequestError","code":400}`)
+	defer fasthttp.ReleaseResponse(resp)
+
+	bifrostErr := ParseSGLError(resp)
+	require.NotNil(t, bifrostErr)
+	require.NotNil(t, bifrostErr.Error)
+	assert.Equal(t, "bad request", bifrostErr.Error.Message)
+	assert.Equal(t, "BadRequestError", strDeref(bifrostErr.Error.Type))
+	assert.Equal(t, "400", strDeref(bifrostErr.Error.Code))
+}
+
+func TestParseSGLError_WrappedEnvelope(t *testing.T) {
+	t.Parallel()
+
+	resp := buildSGLErrorResponse(400, `{"error":{"message":"wrapped boom","type":"invalid_request_error","code":"some_code"}}`)
+	defer fasthttp.ReleaseResponse(resp)
+
+	bifrostErr := ParseSGLError(resp)
+	require.NotNil(t, bifrostErr)
+	require.NotNil(t, bifrostErr.Error)
+	assert.Equal(t, "wrapped boom", bifrostErr.Error.Message)
+	assert.Equal(t, "invalid_request_error", strDeref(bifrostErr.Error.Type))
+	assert.Equal(t, "some_code", strDeref(bifrostErr.Error.Code))
+}
+
+func TestParseSGLError_SubstringMappings(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name     string
+		message  string
+		wantCode string
+		wantType string
+	}{
+		{
+			name:     "context length exceeded",
+			message:  "This model's maximum context length is 4096 tokens. However, you requested 5000 tokens (..). Please reduce the length of the messages or completion. Input is longer than the model's context length.",
+			wantCode: "context_length_exceeded",
+			wantType: "invalid_request_error",
+		},
+		{
+			name:     "out of memory",
+			message:  "CUDA out of memory while attempting to allocate buffer",
+			wantCode: "out_of_memory",
+			wantType: "server_error",
+		},
+		{
+			name:     "model not loaded",
+			message:  "requested model is not loaded on this server",
+			wantCode: "model_not_found",
+			wantType: "invalid_request_error",
+		},
+	}
+
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			body := `{"object":"error","message":` + jsonString(tc.message) + `,"type":"BadRequestError","code":400}`
+			resp := buildSGLErrorResponse(400, body)
+			defer fasthttp.ReleaseResponse(resp)
+
+			bifrostErr := ParseSGLError(resp)
+			require.NotNil(t, bifrostErr)
+			require.NotNil(t, bifrostErr.Error)
+
+			// Message is always preserved verbatim — never replaced by a generic.
+			assert.Equal(t, tc.message, bifrostErr.Error.Message)
+
+			assert.Equal(t, tc.wantCode, strDeref(bifrostErr.Error.Code), "code mapping")
+			assert.Equal(t, tc.wantType, strDeref(bifrostErr.Error.Type), "type mapping")
+		})
+	}
+}
+
+func TestParseSGLError_FallthroughPreservesMessage(t *testing.T) {
+	t.Parallel()
+
+	// An unrecognized sglang error message should be preserved as-is, with
+	// no substring-derived code/type overrides applied.
+	const msg = "some sglang-specific error nobody has mapped yet"
+	resp := buildSGLErrorResponse(503, `{"object":"error","message":"`+msg+`","type":"InternalServerError","code":503}`)
+	defer fasthttp.ReleaseResponse(resp)
+
+	bifrostErr := ParseSGLError(resp)
+	require.NotNil(t, bifrostErr)
+	require.NotNil(t, bifrostErr.Error)
+	assert.Equal(t, msg, bifrostErr.Error.Message)
+	// Type/code come from the flat envelope, not from a substring mapping.
+	assert.Equal(t, "InternalServerError", strDeref(bifrostErr.Error.Type))
+	assert.Equal(t, "503", strDeref(bifrostErr.Error.Code))
+	require.NotNil(t, bifrostErr.StatusCode)
+	assert.Equal(t, 503, *bifrostErr.StatusCode)
+}
+
+func TestParseSGLError_EmptyBodyDelegatesToFallback(t *testing.T) {
+	t.Parallel()
+
+	resp := buildSGLErrorResponse(429, "")
+	defer fasthttp.ReleaseResponse(resp)
+
+	bifrostErr := ParseSGLError(resp)
+	require.NotNil(t, bifrostErr)
+	require.NotNil(t, bifrostErr.Error)
+	// We do not assert exact phrasing of the HTTP-status fallback message;
+	// only that we produced something non-empty so callers see a useful error.
+	assert.NotEmpty(t, bifrostErr.Error.Message)
+	require.NotNil(t, bifrostErr.StatusCode)
+	assert.Equal(t, 429, *bifrostErr.StatusCode)
+}
+
+// TestParseSGLError_GzipFlatEnvelope verifies that a gzip-encoded sglang flat
+// error envelope is still parsed correctly. ParseOpenAIError decodes the body
+// upstream and stashes the decoded JSON on ExtraFields.RawResponse; our
+// flat-envelope fallback must read from there, not from the still-compressed
+// resp.Body().
+func TestParseSGLError_GzipFlatEnvelope(t *testing.T) {
+	t.Parallel()
+
+	const msg = "out of memory while loading shard 0"
+	plain := `{"object":"error","message":"` + msg + `","type":"InternalServerError","code":500}`
+
+	var buf bytes.Buffer
+	gw := gzip.NewWriter(&buf)
+	_, err := gw.Write([]byte(plain))
+	require.NoError(t, err)
+	require.NoError(t, gw.Close())
+
+	resp := fasthttp.AcquireResponse()
+	defer fasthttp.ReleaseResponse(resp)
+	resp.SetStatusCode(500)
+	resp.Header.SetContentType("application/json")
+	resp.Header.Set("Content-Encoding", "gzip")
+	resp.SetBody(buf.Bytes())
+
+	bifrostErr := ParseSGLError(resp)
+	require.NotNil(t, bifrostErr)
+	require.NotNil(t, bifrostErr.Error)
+	assert.Equal(t, msg, bifrostErr.Error.Message)
+	// Substring mapping should still fire on the decoded message.
+	assert.Equal(t, "out_of_memory", strDeref(bifrostErr.Error.Code))
+	assert.Equal(t, "server_error", strDeref(bifrostErr.Error.Type))
+}
+
+// TestParseSGLError_NonErrorObjectIgnored verifies that a JSON object response
+// missing `"object":"error"` is NOT treated as a flat sglang error envelope,
+// even if it happens to have a top-level "message" key. This guards against
+// a sidecar/proxy in front of sglang accidentally hijacking the error mapping.
+func TestParseSGLError_NonErrorObjectIgnored(t *testing.T) {
+	t.Parallel()
+
+	// Object without `"object":"error"` — e.g. some proxy's own 4xx envelope.
+	body := `{"message":"proxy denied request","type":"ProxyError","code":403}`
+	resp := buildSGLErrorResponse(403, body)
+	defer fasthttp.ReleaseResponse(resp)
+
+	bifrostErr := ParseSGLError(resp)
+	require.NotNil(t, bifrostErr)
+	require.NotNil(t, bifrostErr.Error)
+	// The flat-envelope path should be skipped, so the message should NOT be
+	// "proxy denied request" — the wrapper's default ("provider API error ...")
+	// stays in place.
+	assert.NotEqual(t, "proxy denied request", bifrostErr.Error.Message)
+}
+
+// jsonString minimally escapes a Go string for embedding in a JSON literal.
+// Only handles characters used by the test fixtures.
+func jsonString(s string) string {
+	out := make([]byte, 0, len(s)+2)
+	out = append(out, '"')
+	for i := 0; i < len(s); i++ {
+		c := s[i]
+		switch c {
+		case '"', '\\':
+			out = append(out, '\\', c)
+		case '\n':
+			out = append(out, '\\', 'n')
+		case '\r':
+			out = append(out, '\\', 'r')
+		case '\t':
+			out = append(out, '\\', 't')
+		default:
+			out = append(out, c)
+		}
+	}
+	out = append(out, '"')
+	return string(out)
+}