From 5c4701454bd8ca041c010104700001709fc7f746 Mon Sep 17 00:00:00 2001 From: tejas ghatte Date: Fri, 11 Sep 2026 14:15:45 +0530 Subject: [PATCH] fix: config flag to route to openai compat api in bedrock --- core/providers/bedrock/bedrock.go | 16 +++- core/providers/bedrock/runtimeopenai.go | 71 ++++++++++++++ core/providers/bedrock/surface.go | 40 ++++++-- core/providers/bedrock/surface_test.go | 95 ++++++++++++++++--- core/providers/openai/responses.go | 8 +- core/schemas/account.go | 3 + framework/configstore/clientconfig.go | 6 ++ framework/configstore/migrations.go | 29 ++++++ framework/configstore/rdb.go | 7 +- framework/configstore/tables/key.go | 13 +++ transports/config.schema.json | 8 ++ .../fragments/apiKeysFormFragment.tsx | 22 +++++ ui/lib/types/config.ts | 2 + ui/lib/types/schemas.ts | 2 + 14 files changed, 292 insertions(+), 30 deletions(-) diff --git a/core/providers/bedrock/bedrock.go b/core/providers/bedrock/bedrock.go index 38ab95b3b09..f4ca4132f6e 100644 --- a/core/providers/bedrock/bedrock.go +++ b/core/providers/bedrock/bedrock.go @@ -1276,9 +1276,13 @@ func (provider *BedrockProvider) ChatCompletion(ctx *schemas.BifrostContext, key return nil, err } - if provider.routesToMantle(ctx, key, request.Model) { + surface := provider.resolveSurface(ctx, key, request.Model) + if surface.isMantle() { return provider.mantleChatCompletions(ctx, key, request) } + if runtimeServesOpenAIAPI(ctx, key, surface, request.Model, schemas.BedrockAPIChatCompletions) { + return provider.runtimeChatCompletions(ctx, key, request) + } // Use Bedrock Converse API for all other models jsonData, bifrostErr := providerUtils.CheckContextAndGetRequestBody( @@ -1467,9 +1471,13 @@ func (provider *BedrockProvider) ChatCompletionStream(ctx *schemas.BifrostContex return nil, err } - if provider.routesToMantle(ctx, key, request.Model) { + surface := provider.resolveSurface(ctx, key, request.Model) + if surface.isMantle() { return provider.mantleChatCompletionsStream(ctx, postHookRunner, postHookSpanFinalizer, key, request) } + if runtimeServesOpenAIAPI(ctx, key, surface, request.Model, schemas.BedrockAPIChatCompletions) { + return provider.runtimeChatCompletionsStream(ctx, postHookRunner, postHookSpanFinalizer, key, request) + } // Use Bedrock Converse streaming API for all other models jsonData, bifrostErr := providerUtils.CheckContextAndGetRequestBody( @@ -1793,7 +1801,7 @@ func (provider *BedrockProvider) Responses(ctx *schemas.BifrostContext, key sche if surface.isMantle() { return provider.mantleResponses(ctx, key, request) } - if runtimeServesResponses(ctx, surface, request.Model) { + if runtimeServesOpenAIAPI(ctx, key, surface, request.Model, schemas.BedrockAPIResponses) { return provider.runtimeResponses(ctx, key, request) } @@ -1883,7 +1891,7 @@ func (provider *BedrockProvider) ResponsesStream(ctx *schemas.BifrostContext, po if surface.isMantle() { return provider.mantleResponsesStream(ctx, postHookRunner, postHookSpanFinalizer, key, request) } - if runtimeServesResponses(ctx, surface, request.Model) { + if runtimeServesOpenAIAPI(ctx, key, surface, request.Model, schemas.BedrockAPIResponses) { return provider.runtimeResponsesStream(ctx, postHookRunner, postHookSpanFinalizer, key, request) } diff --git a/core/providers/bedrock/runtimeopenai.go b/core/providers/bedrock/runtimeopenai.go index f16f1bc473c..cfd8f5c9878 100644 --- a/core/providers/bedrock/runtimeopenai.go +++ b/core/providers/bedrock/runtimeopenai.go @@ -90,3 +90,74 @@ func (provider *BedrockProvider) runtimeResponsesStream( postHookSpanFinalizer, ) } + +// runtimeChatCompletions handles non-streaming chat requests on bedrock-runtime's +// OpenAI-compatible surface. Reached only when the operator opts in. +func (provider *BedrockProvider) runtimeChatCompletions( + ctx *schemas.BifrostContext, + key schemas.Key, + request *schemas.BifrostChatRequest, +) (*schemas.BifrostChatResponse, *schemas.BifrostError) { + region := resolveBedrockRegion(ctx, key, request.Model) + url := runtimeOpenAIURL(bedrockEndpoints(key.BedrockKeyConfig), region, "chat/completions") + + var signer providerUtils.BodySigner + if key.Value.GetValue() == "" { + signer = func(body []byte) (map[string]string, *schemas.BifrostError) { + return signOpenAIV4Headers(ctx, body, url, "application/json", key, region, provider.networkConfig.ExtraHeaders, bedrockSigningService) + } + } + + return openai.HandleOpenAIChatCompletionRequest( + ctx, + provider.mantleClient, + url, + request, + openai.BearerAuthHeader(key), + provider.networkConfig.ExtraHeaders, + providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest), + providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse), + provider.GetProviderKey(), + nil, + nil, + signer, + provider.logger, + ) +} + +// runtimeChatCompletionsStream handles streaming chat requests on bedrock-runtime's +// OpenAI-compatible surface. +func (provider *BedrockProvider) runtimeChatCompletionsStream( + ctx *schemas.BifrostContext, + postHookRunner schemas.PostHookRunner, + postHookSpanFinalizer func(context.Context), + key schemas.Key, + request *schemas.BifrostChatRequest, +) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { + region := resolveBedrockRegion(ctx, key, request.Model) + url := runtimeOpenAIURL(bedrockEndpoints(key.BedrockKeyConfig), region, "chat/completions") + + var signer providerUtils.BodySigner + if key.Value.GetValue() == "" { + signer = func(body []byte) (map[string]string, *schemas.BifrostError) { + return signOpenAIV4Headers(ctx, body, url, "text/event-stream", key, region, provider.networkConfig.ExtraHeaders, bedrockSigningService) + } + } + + return openai.HandleOpenAIChatCompletionStreaming( + ctx, provider.mantleStreamingClient, url, request, + openai.BearerAuthHeader(key), provider.networkConfig.ExtraHeaders, + provider.networkConfig.StreamIdleTimeoutInSeconds, + providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest), + providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse), + provider.GetProviderKey(), postHookRunner, + nil, + nil, + nil, + nil, + nil, + signer, + provider.logger, + postHookSpanFinalizer, + ) +} diff --git a/core/providers/bedrock/surface.go b/core/providers/bedrock/surface.go index 0d18fc5b956..3e664357258 100644 --- a/core/providers/bedrock/surface.go +++ b/core/providers/bedrock/surface.go @@ -169,24 +169,44 @@ func resolveBedrockSurface(ctx *schemas.BifrostContext, key schemas.Key, model s return bedrockSurface{host: bedrockServiceRuntime, reason: reasonModelFamilyFallback} } -// runtimeServesResponses reports whether a runtime-bound Responses request should use -// bedrock-runtime's OpenAI-compatible /openai/v1/responses surface instead of Converse. +// ResolveUseOpenAIEndpoints reports whether this key or alias routes Bedrock inference +// through the OpenAI-compatible endpoints instead of Converse. Opt-in, and an alias value +// wins over the key, mirroring ResolveUseAnthropicEndpoints. // -// Converse holds no conversation state and has no previous_response_id, so it silently -// drops the reference: a stateful client sends only the new turn and the model never sees -// the rest. With tools that fails outright, since the tool result arrives with its toolUse -// left behind in state. The OpenAI surface both reads and mints response ids. +// Opt-in rather than automatic because the two surfaces are not interchangeable. Converse +// carries Bedrock Guardrails, performanceConfig and requestMetadata, all of which the +// OpenAI-compatible endpoints accept and silently ignore, so diverting on Bifrost's own +// initiative could stop a guardrail being enforced with no error anywhere. +func ResolveUseOpenAIEndpoints(ctx *schemas.BifrostContext, key schemas.Key) bool { + if ra := schemas.GetResolvedAlias(ctx); ra != nil && ra.Config != nil && ra.Config.UseOpenAIEndpoints != nil { + return *ra.Config.UseOpenAIEndpoints + } + return key.UseOpenAIEndpoints != nil && *key.UseOpenAIEndpoints +} + +// runtimeServesOpenAIAPI reports whether a runtime-bound request should use +// bedrock-runtime's OpenAI-compatible surface for the given wire API. // -// The datasheet decides when it publishes a runtime row; otherwise family detection, since -// AWS 404s every other family on this path ("doesn't support this API"). -func runtimeServesResponses(ctx *schemas.BifrostContext, surface bedrockSurface, model string) bool { +// What the opt-in buys: Converse holds no conversation state and has no +// previous_response_id, so it silently drops the reference. A stateful client sends only +// the new turn and the model never sees the rest; with tools that fails outright, since +// the tool result arrives with its toolUse left behind in state. +// +// The datasheet decides support when it publishes a runtime row; otherwise family +// detection, since AWS 404s every other family here ("doesn't support this API"). Support +// is a separate question from the flag: opting in never forces a surface the model cannot +// serve. +func runtimeServesOpenAIAPI(ctx *schemas.BifrostContext, key schemas.Key, surface bedrockSurface, model string, api schemas.BedrockAPI) bool { + if !ResolveUseOpenAIEndpoints(ctx, key) { + return false + } // An application inference profile is Converse-only, so it must never divert. if surface.isMantle() || surface.reason == reasonApplicationProfile { return false } canonical := schemas.ResolveCanonicalModel(ctx, model) if apis := schemas.ResolveModelCaps(schemas.Bedrock, canonical).BedrockAPIs(); len(apis) > 0 { - return slices.Contains(apis, schemas.BedrockAPIResponses) + return slices.Contains(apis, api) } return schemas.IsOpenAIModelFamily(ctx, canonical) || schemas.IsGrokModel(canonical) } diff --git a/core/providers/bedrock/surface_test.go b/core/providers/bedrock/surface_test.go index 5a000970ac2..7db318d2bcb 100644 --- a/core/providers/bedrock/surface_test.go +++ b/core/providers/bedrock/surface_test.go @@ -599,6 +599,7 @@ func TestIsAIPResourceID(t *testing.T) { // detection is what actually gates the OpenAI-compatible Responses surface // today. AWS 404s every other family there. func TestRuntimeServesResponsesFamilyFallback(t *testing.T) { + optedIn := schemas.Key{UseOpenAIEndpoints: schemas.Ptr(true)} cases := []struct { model string want bool @@ -613,9 +614,9 @@ func TestRuntimeServesResponsesFamilyFallback(t *testing.T) { for _, tc := range cases { t.Run(tc.model, func(t *testing.T) { ctx := surfaceTestCtx() - surface := resolveBedrockSurface(ctx, schemas.Key{}, tc.model) - if got := runtimeServesResponses(ctx, surface, tc.model); got != tc.want { - t.Errorf("runtimeServesResponses(%q) = %v, want %v", tc.model, got, tc.want) + surface := resolveBedrockSurface(ctx, optedIn, tc.model) + if got := runtimeServesOpenAIAPI(ctx, optedIn, surface, tc.model, schemas.BedrockAPIResponses); got != tc.want { + t.Errorf("runtimeServesOpenAIAPI(%q) = %v, want %v", tc.model, got, tc.want) } }) } @@ -624,6 +625,7 @@ func TestRuntimeServesResponsesFamilyFallback(t *testing.T) { // A published runtime row is authoritative in both directions: it can divert a // model family detection would not, and hold back one it would. func TestRuntimeServesResponsesDatasheetWinsOverFamily(t *testing.T) { + optedIn := schemas.Key{UseOpenAIEndpoints: schemas.Ptr(true)} installCaps(t, map[schemas.ModelProvider]map[string][]schemas.BedrockAPI{ schemas.Bedrock: { // Converse-only despite being OpenAI family. @@ -642,9 +644,9 @@ func TestRuntimeServesResponsesDatasheetWinsOverFamily(t *testing.T) { for _, tc := range cases { t.Run(tc.model, func(t *testing.T) { ctx := surfaceTestCtx() - surface := resolveBedrockSurface(ctx, schemas.Key{}, tc.model) - if got := runtimeServesResponses(ctx, surface, tc.model); got != tc.want { - t.Errorf("runtimeServesResponses(%q) = %v, want %v", tc.model, got, tc.want) + surface := resolveBedrockSurface(ctx, optedIn, tc.model) + if got := runtimeServesOpenAIAPI(ctx, optedIn, surface, tc.model, schemas.BedrockAPIResponses); got != tc.want { + t.Errorf("runtimeServesOpenAIAPI(%q) = %v, want %v", tc.model, got, tc.want) } }) } @@ -654,27 +656,30 @@ func TestRuntimeServesResponsesDatasheetWinsOverFamily(t *testing.T) { // family, so without the guard the alias chain would resolve it to an OpenAI // name and divert a request AWS cannot serve. func TestRuntimeServesResponsesNeverDivertsApplicationProfile(t *testing.T) { + optedIn := keyWithARN(appProfileARN) + optedIn.UseOpenAIEndpoints = schemas.Ptr(true) ctx := withAlias("my-gpt", "3dnkdwuaalc7", appProfileARN) name := "gpt-5.6-luna" schemas.GetResolvedAlias(ctx).Config.ModelName = &name - surface := resolveBedrockSurface(ctx, keyWithARN(appProfileARN), "3dnkdwuaalc7") + surface := resolveBedrockSurface(ctx, optedIn, "3dnkdwuaalc7") if surface.reason != reasonApplicationProfile { t.Fatalf("precondition: reason = %q, want %q", surface.reason, reasonApplicationProfile) } - if runtimeServesResponses(ctx, surface, "3dnkdwuaalc7") { + if runtimeServesOpenAIAPI(ctx, optedIn, surface, "3dnkdwuaalc7", schemas.BedrockAPIResponses) { t.Error("an application inference profile must stay on Converse") } } // Mantle has its own Responses path; the runtime surface must never claim it. func TestRuntimeServesResponsesIgnoresMantleSurface(t *testing.T) { + optedIn := schemas.Key{UseOpenAIEndpoints: schemas.Ptr(true)} ctx := surfaceTestCtx() - surface := resolveBedrockSurface(ctx, schemas.Key{}, "openai.gpt-5.6-terra") + surface := resolveBedrockSurface(ctx, optedIn, "openai.gpt-5.6-terra") if !surface.isMantle() { t.Fatalf("precondition: bare id should route to mantle, got %q", surface.host) } - if runtimeServesResponses(ctx, surface, "openai.gpt-5.6-terra") { + if runtimeServesOpenAIAPI(ctx, optedIn, surface, "openai.gpt-5.6-terra", schemas.BedrockAPIResponses) { t.Error("a mantle-bound request must not divert to the runtime surface") } } @@ -682,12 +687,13 @@ func TestRuntimeServesResponsesIgnoresMantleSurface(t *testing.T) { // The gate reads the canonical name, so an alias whose wire id carries no family // still diverts. func TestRuntimeServesResponsesResolvesAliasedModel(t *testing.T) { + optedIn := schemas.Key{UseOpenAIEndpoints: schemas.Ptr(true)} ctx := withAlias("my-gpt", "us.openai.gpt-5.6-terra", "") name := "gpt-5.6-terra" schemas.GetResolvedAlias(ctx).Config.ModelName = &name - surface := resolveBedrockSurface(ctx, schemas.Key{}, "us.openai.gpt-5.6-terra") - if !runtimeServesResponses(ctx, surface, "us.openai.gpt-5.6-terra") { + surface := resolveBedrockSurface(ctx, optedIn, "us.openai.gpt-5.6-terra") + if !runtimeServesOpenAIAPI(ctx, optedIn, surface, "us.openai.gpt-5.6-terra", schemas.BedrockAPIResponses) { t.Error("aliased OpenAI model must divert to the runtime Responses surface") } } @@ -697,3 +703,68 @@ func TestRuntimeOpenAIURL(t *testing.T) { t.Errorf("runtimeOpenAIURL = %q", got) } } + +// Opt-in, two states, mirroring use_anthropic_endpoints: off keeps everything on +// Converse, on moves both request types to the OpenAI-compatible surface. +func TestUseOpenAIEndpointsFlag(t *testing.T) { + const model = "us.openai.gpt-5.6-terra" + cases := []struct { + name string + flag *bool + want bool + }{ + {"unset", nil, false}, + {"false", schemas.Ptr(false), false}, + {"true", schemas.Ptr(true), true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctx := surfaceTestCtx() + key := schemas.Key{UseOpenAIEndpoints: tc.flag} + surface := resolveBedrockSurface(ctx, key, model) + for _, api := range []schemas.BedrockAPI{schemas.BedrockAPIResponses, schemas.BedrockAPIChatCompletions} { + if got := runtimeServesOpenAIAPI(ctx, key, surface, model, api); got != tc.want { + t.Errorf("%s = %v, want %v", api, got, tc.want) + } + } + }) + } +} + +// An alias-level value wins over the key, matching use_anthropic_endpoints. +func TestUseOpenAIEndpointsAliasOverridesKey(t *testing.T) { + const model = "us.openai.gpt-5.6-terra" + ctx := withAlias("my-gpt", model, "") + schemas.GetResolvedAlias(ctx).Config.UseOpenAIEndpoints = schemas.Ptr(false) + + key := schemas.Key{UseOpenAIEndpoints: schemas.Ptr(true)} + surface := resolveBedrockSurface(ctx, key, model) + if runtimeServesOpenAIAPI(ctx, key, surface, model, schemas.BedrockAPIResponses) { + t.Error("alias false must override key true") + } +} + +// Opting in cannot force a surface the model does not serve: AWS 404s Claude there. +func TestUseOpenAIEndpointsCannotForceUnsupportedModel(t *testing.T) { + const model = "us.anthropic.claude-sonnet-4-6" + key := schemas.Key{UseOpenAIEndpoints: schemas.Ptr(true)} + ctx := surfaceTestCtx() + surface := resolveBedrockSurface(ctx, key, model) + if runtimeServesOpenAIAPI(ctx, key, surface, model, schemas.BedrockAPIResponses) { + t.Error("claude must stay on converse even when opted in") + } +} + +// Nor can it override the application-inference-profile guard. +func TestUseOpenAIEndpointsCannotForceApplicationProfile(t *testing.T) { + ctx := withAlias("my-gpt", "3dnkdwuaalc7", appProfileARN) + name := "gpt-5.6-luna" + schemas.GetResolvedAlias(ctx).Config.ModelName = &name + + key := keyWithARN(appProfileARN) + key.UseOpenAIEndpoints = schemas.Ptr(true) + surface := resolveBedrockSurface(ctx, key, "3dnkdwuaalc7") + if runtimeServesOpenAIAPI(ctx, key, surface, "3dnkdwuaalc7", schemas.BedrockAPIResponses) { + t.Error("an application inference profile must stay on Converse even when opted in") + } +} diff --git a/core/providers/openai/responses.go b/core/providers/openai/responses.go index 09834c0cb4d..e1446b42d15 100644 --- a/core/providers/openai/responses.go +++ b/core/providers/openai/responses.go @@ -153,7 +153,9 @@ const maxResponsesCacheBreakpoints = 4 // per-block cache_control through /v1/responses and converts a breakpoint back into // an Anthropic one (#6290). OpenAI defined the field for gpt-5.6, where it pairs with // request-level prompt_cache_options; Azure and Bedrock Mantle serve the same models -// through the same wire format, so they inherit it (#6180). +// through the same wire format, so they inherit it (#6180). Bedrock is listed for the +// same reason: its OpenAI-compatible surfaces on both hosts speak that wire format, and +// a request there reports the bedrock key rather than bedrock_mantle. // // Everything else either accepts cache_control directly or caches implicitly, and for // those the serializer's existing strip is the correct behaviour. @@ -161,7 +163,7 @@ func responsesUsesPromptCacheBreakpoints(provider schemas.ModelProvider, model s switch provider { case schemas.OpenRouter: return true - case schemas.OpenAI, schemas.Azure, schemas.BedrockMantle: + case schemas.OpenAI, schemas.Azure, schemas.BedrockMantle, schemas.Bedrock: return schemas.IsGPT56Model(model) default: return false @@ -195,7 +197,7 @@ func responsesHasPromptCacheBreakpoint(messages []schemas.ResponsesMessage) bool // that off; mode=explicit does. OpenRouter has no equivalent field and needs none. func responsesUsesPromptCacheOptions(provider schemas.ModelProvider, model string) bool { switch provider { - case schemas.OpenAI, schemas.Azure, schemas.BedrockMantle: + case schemas.OpenAI, schemas.Azure, schemas.BedrockMantle, schemas.Bedrock: return schemas.IsGPT56Model(model) default: return false diff --git a/core/schemas/account.go b/core/schemas/account.go index 02a58b17485..cdbf5f03134 100644 --- a/core/schemas/account.go +++ b/core/schemas/account.go @@ -144,6 +144,7 @@ type Key struct { Enabled *bool `json:"enabled,omitempty"` // Whether the key is active (default:true) UseForBatchAPI *bool `json:"use_for_batch_api,omitempty"` // Whether this key can be used for batch API operations (default:false for new keys, migrated keys default to true) UseAnthropicEndpoints *bool `json:"use_anthropic_endpoints,omitempty"` // Whether to use anthropic endpoints for this key + UseOpenAIEndpoints *bool `json:"use_openai_endpoints,omitempty"` // Whether to use OpenAI-compatible endpoints for this key ConfigHash string `json:"config_hash,omitempty"` // Hash of config.json version, used for change detection Status KeyStatusType `json:"status,omitempty"` // Status of key Description string `json:"description,omitempty"` // Description of key @@ -234,6 +235,7 @@ type AliasConfig struct { // a field name shared by multiple same-depth anonymous structs. ProjectID *SecretVar `json:"project_id,omitempty"` UseAnthropicEndpoints *bool `json:"use_anthropic_endpoints,omitempty"` // Whether to use anthropic endpoints for this alias + UseOpenAIEndpoints *bool `json:"use_openai_endpoints,omitempty"` // Whether to use OpenAI-compatible endpoints for this alias *AzureAliasCfg *VertexAliasCfg @@ -252,6 +254,7 @@ func (ac AliasConfig) isLegacyShape() bool { ac.Region == nil && ac.ProjectID == nil && ac.UseAnthropicEndpoints == nil && + ac.UseOpenAIEndpoints == nil && ac.AzureAliasCfg == nil && ac.VertexAliasCfg == nil && ac.BedrockAliasCfg == nil && diff --git a/framework/configstore/clientconfig.go b/framework/configstore/clientconfig.go index 4113a25dd71..8bb7397bf05 100644 --- a/framework/configstore/clientconfig.go +++ b/framework/configstore/clientconfig.go @@ -570,6 +570,12 @@ func (p *ProviderConfig) Redacted() *ProviderConfig { } else { redactedConfig.Keys[i].UseAnthropicEndpoints = new(false) } + // Add back use openai endpoints + if key.UseOpenAIEndpoints != nil { + redactedConfig.Keys[i].UseOpenAIEndpoints = key.UseOpenAIEndpoints + } else { + redactedConfig.Keys[i].UseOpenAIEndpoints = new(false) + } // Add model discovery status and error redactedConfig.Keys[i].Status = key.Status diff --git a/framework/configstore/migrations.go b/framework/configstore/migrations.go index e2697d41028..2d3c560f9ba 100644 --- a/framework/configstore/migrations.go +++ b/framework/configstore/migrations.go @@ -492,6 +492,7 @@ var configstoreMigrationSteps = []migrationStep{ {IDs: []string{"backfill_vk_allow_all_providers_hash"}, run: migrationBackfillVirtualKeyAllowAllProvidersHash}, {IDs: []string{"add_prompt_cache_json_column"}, run: migrationAddPromptCacheJSONColumn}, {IDs: []string{"add_hidden_request_types_json_column"}, run: migrationAddHiddenRequestTypesJSONColumn}, + {IDs: []string{"add_use_openai_endpoints_column"}, run: migrationAddUseOpenAIEndpointsColumn}, } // videoResolutionPricingColumns are the resolution-banded video output rate columns. @@ -13493,3 +13494,31 @@ func migrationAddHiddenRequestTypesJSONColumn(ctx context.Context, db *gorm.DB, } return nil } + +// migrationAddUseOpenAIEndpointsColumn adds the use_openai_endpoints column to the config_keys table. +func migrationAddUseOpenAIEndpointsColumn(ctx context.Context, db *gorm.DB, logger schemas.Logger) error { + migrationName := "add_use_openai_endpoints_column" + logger.Info("[configstore] starting migration %s", migrationName) + defer logger.Info("[configstore] finished migration %s", migrationName) + m := migrator.New(db, migrator.DefaultOptions, []*migrator.Migration{{ + ID: migrationName, + Migrate: func(tx *gorm.DB) error { + tx = tx.WithContext(ctx) + if err := addColumnIfNotExists(tx, logger, &tables.TableKey{}, "use_openai_endpoints"); err != nil { + return fmt.Errorf("failed to add use_openai_endpoints column: %w", err) + } + return nil + }, + Rollback: func(tx *gorm.DB) error { + tx = tx.WithContext(ctx) + if err := dropColumnIfExists(tx, logger, &tables.TableKey{}, "use_openai_endpoints"); err != nil { + return fmt.Errorf("failed to drop use_openai_endpoints column: %w", err) + } + return nil + }, + }}) + if err := m.Migrate(); err != nil { + return fmt.Errorf("failed to run migration %s: %w", migrationName, err) + } + return nil +} diff --git a/framework/configstore/rdb.go b/framework/configstore/rdb.go index e7e02a47e49..14a3c2f4ff0 100644 --- a/framework/configstore/rdb.go +++ b/framework/configstore/rdb.go @@ -154,6 +154,7 @@ func schemaKeyFromTableKey(dbKey tables.TableKey) schemas.Key { Enabled: dbKey.Enabled, UseForBatchAPI: dbKey.UseForBatchAPI, UseAnthropicEndpoints: dbKey.UseAnthropicEndpoints, + UseOpenAIEndpoints: dbKey.UseOpenAIEndpoints, AzureKeyConfig: dbKey.AzureKeyConfig, VertexKeyConfig: dbKey.VertexKeyConfig, BedrockKeyConfig: dbKey.BedrockKeyConfig, @@ -185,6 +186,7 @@ func tableKeyFromSchemaKey(provider tables.TableProvider, key schemas.Key) (tabl Enabled: key.Enabled, UseForBatchAPI: key.UseForBatchAPI, UseAnthropicEndpoints: key.UseAnthropicEndpoints, + UseOpenAIEndpoints: key.UseOpenAIEndpoints, AzureKeyConfig: key.AzureKeyConfig, VertexKeyConfig: key.VertexKeyConfig, BedrockKeyConfig: key.BedrockKeyConfig, @@ -757,6 +759,7 @@ func (s *RDBConfigStore) UpdateProvidersConfig(ctx context.Context, providers ma Enabled: key.Enabled, UseForBatchAPI: key.UseForBatchAPI, UseAnthropicEndpoints: key.UseAnthropicEndpoints, + UseOpenAIEndpoints: key.UseOpenAIEndpoints, AzureKeyConfig: key.AzureKeyConfig, VertexKeyConfig: key.VertexKeyConfig, BedrockKeyConfig: key.BedrockKeyConfig, @@ -1001,6 +1004,7 @@ func (s *RDBConfigStore) UpdateProvider(ctx context.Context, provider schemas.Mo Enabled: key.Enabled, UseForBatchAPI: key.UseForBatchAPI, UseAnthropicEndpoints: key.UseAnthropicEndpoints, + UseOpenAIEndpoints: key.UseOpenAIEndpoints, AzureKeyConfig: key.AzureKeyConfig, VertexKeyConfig: key.VertexKeyConfig, BedrockKeyConfig: key.BedrockKeyConfig, @@ -1157,6 +1161,7 @@ func (s *RDBConfigStore) AddProvider(ctx context.Context, provider schemas.Model Enabled: key.Enabled, UseForBatchAPI: key.UseForBatchAPI, UseAnthropicEndpoints: key.UseAnthropicEndpoints, + UseOpenAIEndpoints: key.UseOpenAIEndpoints, AzureKeyConfig: key.AzureKeyConfig, VertexKeyConfig: key.VertexKeyConfig, BedrockKeyConfig: key.BedrockKeyConfig, @@ -9888,4 +9893,4 @@ func (s *RDBConfigStore) DeleteWebhookJob(ctx context.Context, id, runnerID stri return fmt.Errorf("webhook job not found or no longer owned by caller") } return nil -} \ No newline at end of file +} diff --git a/framework/configstore/tables/key.go b/framework/configstore/tables/key.go index 64cf527e6e7..9597ed83577 100644 --- a/framework/configstore/tables/key.go +++ b/framework/configstore/tables/key.go @@ -98,6 +98,11 @@ type TableKey struct { // endpoints instead of its OpenAI-compatible ones. UseAnthropicEndpoints *bool `gorm:"default:false" json:"use_anthropic_endpoints,omitempty"` + // UseOpenAIEndpoints routes Bedrock inference through the OpenAI-compatible endpoints + // instead of Converse. Column name is pinned: the default naming strategy does not + // split OpenAI the way the JSON tag does. + UseOpenAIEndpoints *bool `gorm:"column:use_openai_endpoints;default:false" json:"use_openai_endpoints,omitempty"` + Status string `gorm:"type:varchar(50);default:'unknown'" json:"status"` Description string `gorm:"type:text" json:"description,omitempty"` @@ -163,6 +168,10 @@ func (k *TableKey) BeforeSave(tx *gorm.DB) error { useAnthropicEndpoints := false // DB default k.UseAnthropicEndpoints = &useAnthropicEndpoints } + if k.UseOpenAIEndpoints == nil { + useOpenAIEndpoints := false // DB default + k.UseOpenAIEndpoints = &useOpenAIEndpoints + } // IMPORTANT: All *SecretVar fields assigned from provider config structs (AzureKeyConfig, // VertexKeyConfig, BedrockKeyConfig) MUST be value-copied before assignment. The caller // may retain the config struct pointer; if BeforeSave (or future encryption) mutates a @@ -865,6 +874,10 @@ func (k *TableKey) AfterFind(tx *gorm.DB) error { useAnthropicEndpoints := false // DB default k.UseAnthropicEndpoints = &useAnthropicEndpoints } + if k.UseOpenAIEndpoints == nil { + useOpenAIEndpoints := false // DB default + k.UseOpenAIEndpoints = &useOpenAIEndpoints + } // Reconstruct Azure config if fields are present if k.AzureEndpoint != nil || k.AzureClientID != nil || k.AzureClientSecret != nil || k.AzureTenantID != nil || (k.AzureScopesJSON != nil && *k.AzureScopesJSON != "") { var scopes []string diff --git a/transports/config.schema.json b/transports/config.schema.json index ead057ec2ba..1be6f200dfe 100644 --- a/transports/config.schema.json +++ b/transports/config.schema.json @@ -4597,6 +4597,10 @@ "use_anthropic_endpoints": { "type": "boolean", "description": "Routes chat completions and responses requests through Anthropic-compatible endpoints." + }, + "use_openai_endpoints": { + "type": "boolean", + "description": "Per-alias override of use_openai_endpoints." } }, "required": ["model_id"], @@ -4728,6 +4732,10 @@ }, "required": ["region"], "additionalProperties": false + }, + "use_openai_endpoints": { + "type": "boolean", + "description": "Routes chat completions and responses requests through Bedrock's OpenAI-compatible endpoints (/openai/v1) instead of Converse, for models that serve them. Converse is used for everything else. Bedrock Guardrails, performanceConfig and requestMetadata apply only on Converse: the OpenAI-compatible endpoints accept and ignore them." } } } diff --git a/ui/app/workspace/providers/fragments/apiKeysFormFragment.tsx b/ui/app/workspace/providers/fragments/apiKeysFormFragment.tsx index 63a2a97217f..81fb8f386d6 100644 --- a/ui/app/workspace/providers/fragments/apiKeysFormFragment.tsx +++ b/ui/app/workspace/providers/fragments/apiKeysFormFragment.tsx @@ -1244,6 +1244,28 @@ export function ApiKeyFormFragment({ control, providerName, baseProviderType, fo )} {isBedrock && (
+ ( + +
+ Use OpenAI Endpoints + + Routes requests through Bedrock's OpenAI-compatible endpoints instead of Converse. + +
+ + + + + )} + />
Authentication Method diff --git a/ui/lib/types/config.ts b/ui/lib/types/config.ts index 0578afcb603..085761c1f3a 100644 --- a/ui/lib/types/config.ts +++ b/ui/lib/types/config.ts @@ -74,6 +74,7 @@ export interface AliasConfig { // Replicate overrides use_deployments_endpoint?: boolean; use_anthropic_endpoints?: boolean; + use_openai_endpoints?: boolean; } // AzureKeyConfig matching Go's schemas.AzureKeyConfig @@ -272,6 +273,7 @@ export interface ModelProviderKey { enabled?: boolean; use_for_batch_api?: boolean; use_anthropic_endpoints?: boolean; + use_openai_endpoints?: boolean; aliases?: Record; azure_key_config?: AzureKeyConfig; vertex_key_config?: VertexKeyConfig; diff --git a/ui/lib/types/schemas.ts b/ui/lib/types/schemas.ts index 4948e762c6e..815aa236afe 100644 --- a/ui/lib/types/schemas.ts +++ b/ui/lib/types/schemas.ts @@ -421,6 +421,7 @@ const aliasConfigObjectSchema = z.object({ // Replicate overrides use_deployments_endpoint: z.boolean().optional(), use_anthropic_endpoints: z.boolean().optional(), + use_openai_endpoints: z.boolean().optional(), }); // The Go server emits the legacy string wire shape (`{"my-alias": "model-id"}`) @@ -470,6 +471,7 @@ export const modelProviderKeySchema = z github_copilot_key_config: githubCopilotKeyConfigSchema.optional(), use_for_batch_api: z.boolean().optional(), use_anthropic_endpoints: z.boolean().optional(), + use_openai_endpoints: z.boolean().optional(), enabled: z.boolean().optional(), }) .refine(