diff --git a/core/providers/vertex/utils.go b/core/providers/vertex/utils.go index 53b187390aa..e944e0102e9 100644 --- a/core/providers/vertex/utils.go +++ b/core/providers/vertex/utils.go @@ -58,6 +58,19 @@ func resolveVertexRegion(ctx *schemas.BifrostContext, key schemas.Key) string { return "" } +// resolveVertexForceSingleRegion reports whether Bifrost must use the configured +// region as-is and skip promoting multi-region-only models to a multi-region pool +// endpoint. Priority: per-alias VertexAliasCfg.ForceSingleRegion (when set) > +// key-level VertexKeyConfig.ForceSingleRegion. Used by provisioned-throughput +// customers who can serve multi-region-only models (e.g. Opus 4.7/4.8) from a +// single region. +func resolveVertexForceSingleRegion(ctx *schemas.BifrostContext, key schemas.Key) bool { + if ra := schemas.GetResolvedAlias(ctx); ra != nil && ra.Config != nil && ra.Config.VertexAliasCfg != nil && ra.Config.VertexAliasCfg.ForceSingleRegion != nil { + return *ra.Config.VertexAliasCfg.ForceSingleRegion + } + return key.VertexKeyConfig != nil && key.VertexKeyConfig.ForceSingleRegion +} + // isVertexMultiRegionEndpoint reports whether the Vertex location uses Google's // partner-model multi-region pool endpoint host instead of the single-region host. func isVertexMultiRegionEndpoint(region string) bool { @@ -88,7 +101,10 @@ func getVertexAPIHost(region string) string { // the corresponding multi-region pool endpoint — but only for US (us-*) and // Europe (europe-*) regions that have multi-region pools. Other regions // (asia-*, me-*, etc.) stay on the single-region host. -func getVertexModelAwareAPIHost(region string, model string) string { +// +// When forceSingleRegion is set the promotion is skipped and the configured +// single-region host is used as-is (e.g. provisioned-throughput deployments). +func getVertexModelAwareAPIHost(region string, model string, forceSingleRegion bool, logger schemas.Logger) string { if region == "global" { return "aiplatform.googleapis.com" } @@ -99,7 +115,16 @@ func getVertexModelAwareAPIHost(region string, model string) string { // Single-region: promote to multi-region pool if the model requires it // and the region belongs to a pool that supports multi-region. if providerUtils.IsVertexMultiRegionOnlyModel(model) { + if forceSingleRegion { + if logger != nil { + logger.Debug("[vertex] force_single_region set: keeping requested region %q for multi-region-only model %q; skipping multi-region pool promotion", region, model) + } + return fmt.Sprintf("%s-aiplatform.googleapis.com", region) + } if pool, ok := vertexRegionToPool(region); ok { + if logger != nil { + logger.Debug("[vertex] promoting multi-region-only model %q from region %q to multi-region pool %q (host aiplatform.%s.rep.googleapis.com)", model, region, pool, pool) + } return fmt.Sprintf("aiplatform.%s.rep.googleapis.com", pool) } } @@ -134,8 +159,8 @@ func getVertexAPIBaseURL(region string, apiVersion string) string { // getVertexModelAwareAPIBaseURL is like getVertexAPIBaseURL but uses model-aware // host selection for multi-region endpoints. -func getVertexModelAwareAPIBaseURL(region string, apiVersion string, model string) string { - return fmt.Sprintf("https://%s/%s", getVertexModelAwareAPIHost(region, model), apiVersion) +func getVertexModelAwareAPIBaseURL(region string, apiVersion string, model string, forceSingleRegion bool, logger schemas.Logger) string { + return fmt.Sprintf("https://%s/%s", getVertexModelAwareAPIHost(region, model, forceSingleRegion, logger), apiVersion) } func getVertexProjectLocationURL(region string, apiVersion string, projectID string) string { @@ -151,21 +176,22 @@ func getVertexPublisherModelURL(region string, apiVersion string, projectID stri // inference endpoints that may need multi-region pool hosts. // When a single-region is promoted to multi-region, both the host AND the // locations/ path segment are updated to the pool region. -func getVertexModelAwarePublisherModelURL(region string, apiVersion string, projectID string, publisher string, model string, method string) string { - effectiveRegion := getVertexEffectiveRegion(region, model) - baseURL := fmt.Sprintf("https://%s/%s", getVertexModelAwareAPIHost(region, model), apiVersion) +func getVertexModelAwarePublisherModelURL(region string, apiVersion string, projectID string, publisher string, model string, method string, forceSingleRegion bool, logger schemas.Logger) string { + effectiveRegion := getVertexEffectiveRegion(region, model, forceSingleRegion) + baseURL := fmt.Sprintf("https://%s/%s", getVertexModelAwareAPIHost(region, model, forceSingleRegion, logger), apiVersion) return fmt.Sprintf("%s/projects/%s/locations/%s/publishers/%s/models/%s%s", baseURL, projectID, effectiveRegion, publisher, model, method) } // getVertexEffectiveRegion returns the region to use in URL path segments. // For multi-region locations it returns the region as-is. For single-region // locations it returns the multi-region pool if the model is flagged, otherwise -// the original region. -func getVertexEffectiveRegion(region string, model string) string { +// the original region. When forceSingleRegion is set the region is returned +// as-is (no pool promotion). +func getVertexEffectiveRegion(region string, model string, forceSingleRegion bool) string { if isVertexMultiRegionEndpoint(region) || region == "global" { return region } - if providerUtils.IsVertexMultiRegionOnlyModel(model) { + if !forceSingleRegion && providerUtils.IsVertexMultiRegionOnlyModel(model) { if pool, ok := vertexRegionToPool(region); ok { return pool } diff --git a/core/providers/vertex/utils_test.go b/core/providers/vertex/utils_test.go index 60dc344cc26..f7984acee10 100644 --- a/core/providers/vertex/utils_test.go +++ b/core/providers/vertex/utils_test.go @@ -210,10 +210,11 @@ func TestGetVertexModelAwareAPIHost(t *testing.T) { }) tests := []struct { - name string - region string - model string - expected string + name string + region string + model string + forceSingleRegion bool + expected string }{ { name: "global endpoint ignores model flag", @@ -275,13 +276,27 @@ func TestGetVertexModelAwareAPIHost(t *testing.T) { model: "claude-sonnet-4-5", expected: "us-central1-aiplatform.googleapis.com", }, + { + name: "force single region skips us pool promotion for flagged model", + region: "us-east5", + model: "claude-opus-4-7", + forceSingleRegion: true, + expected: "us-east5-aiplatform.googleapis.com", + }, + { + name: "force single region skips eu pool promotion for flagged model", + region: "europe-west1", + model: "claude-opus-4-7", + forceSingleRegion: true, + expected: "europe-west1-aiplatform.googleapis.com", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - actual := getVertexModelAwareAPIHost(tt.region, tt.model) + actual := getVertexModelAwareAPIHost(tt.region, tt.model, tt.forceSingleRegion, nil) if actual != tt.expected { t.Fatalf("expected %q, got %q", tt.expected, actual) } @@ -299,10 +314,11 @@ func TestGetVertexModelAwarePublisherModelURL(t *testing.T) { }) tests := []struct { - name string - region string - model string - expected string + name string + region string + model string + forceSingleRegion bool + expected string }{ { name: "us multi-region flagged model gets rep host URL", @@ -340,13 +356,20 @@ func TestGetVertexModelAwarePublisherModelURL(t *testing.T) { model: "claude-3-5-sonnet", expected: "https://us-central1-aiplatform.googleapis.com/v1/projects/project-123/locations/us-central1/publishers/anthropic/models/claude-3-5-sonnet:rawPredict", }, + { + name: "force single region keeps flagged model on requested region host and path", + region: "us-east5", + model: "claude-opus-4-7", + forceSingleRegion: true, + expected: "https://us-east5-aiplatform.googleapis.com/v1/projects/project-123/locations/us-east5/publishers/anthropic/models/claude-opus-4-7:rawPredict", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - actual := getVertexModelAwarePublisherModelURL(tt.region, "v1", "project-123", "anthropic", tt.model, ":rawPredict") + actual := getVertexModelAwarePublisherModelURL(tt.region, "v1", "project-123", "anthropic", tt.model, ":rawPredict", tt.forceSingleRegion, nil) if actual != tt.expected { t.Fatalf("expected %q, got %q", tt.expected, actual) } @@ -530,6 +553,69 @@ func TestResolveVertexProjectNumber_AliasOverride(t *testing.T) { } } +// TestResolveVertexForceSingleRegion_AliasOverride verifies the per-alias +// VertexAliasCfg.ForceSingleRegion (*bool) override: an explicit alias-level +// value wins over the key-level bool — including an alias false overriding a key +// true — while a nil alias value falls through to the key-level bool. +func TestResolveVertexForceSingleRegion_AliasOverride(t *testing.T) { + keyForceTrue := schemas.Key{ + VertexKeyConfig: &schemas.VertexKeyConfig{ForceSingleRegion: true}, + } + keyForceFalse := schemas.Key{ + VertexKeyConfig: &schemas.VertexKeyConfig{ForceSingleRegion: false}, + } + + // nil ctx uses the key-level value. + if got := resolveVertexForceSingleRegion(nil, keyForceTrue); !got { + t.Errorf("nil ctx: got %v, want key-level true", got) + } + + // empty ctx (no resolved alias) uses the key-level value. + ctx0 := schemas.NewBifrostContext(nil, schemas.NoDeadline) + if got := resolveVertexForceSingleRegion(ctx0, keyForceFalse); got { + t.Errorf("empty ctx: got %v, want key-level false", got) + } + + // Alias-level true overrides key-level false. + ctxTrue := schemas.NewBifrostContext(nil, schemas.NoDeadline) + ctxTrue.SetValue(schemas.BifrostContextKeyResolvedAlias, &schemas.ResolvedAlias{ + Key: "best-claude", + Config: &schemas.AliasConfig{ + ModelID: "claude-opus-4-7", + VertexAliasCfg: &schemas.VertexAliasCfg{ForceSingleRegion: schemas.Ptr(true)}, + }, + }) + if got := resolveVertexForceSingleRegion(ctxTrue, keyForceFalse); !got { + t.Errorf("alias true should win over key false: got %v, want true", got) + } + + // Alias-level false overrides key-level true (explicit set beats nil). + ctxFalse := schemas.NewBifrostContext(nil, schemas.NoDeadline) + ctxFalse.SetValue(schemas.BifrostContextKeyResolvedAlias, &schemas.ResolvedAlias{ + Key: "best-claude", + Config: &schemas.AliasConfig{ + ModelID: "claude-opus-4-7", + VertexAliasCfg: &schemas.VertexAliasCfg{ForceSingleRegion: schemas.Ptr(false)}, + }, + }) + if got := resolveVertexForceSingleRegion(ctxFalse, keyForceTrue); got { + t.Errorf("alias false should win over key true: got %v, want false", got) + } + + // Alias present but ForceSingleRegion nil falls through to the key-level value. + ctxNil := schemas.NewBifrostContext(nil, schemas.NoDeadline) + ctxNil.SetValue(schemas.BifrostContextKeyResolvedAlias, &schemas.ResolvedAlias{ + Key: "best-claude", + Config: &schemas.AliasConfig{ + ModelID: "claude-opus-4-7", + VertexAliasCfg: &schemas.VertexAliasCfg{ForceSingleRegion: nil}, + }, + }) + if got := resolveVertexForceSingleRegion(ctxNil, keyForceTrue); !got { + t.Errorf("nil alias value should fall through to key true: got %v, want true", got) + } +} + // TestGeminiImageURLSchemesContract pins the exact allowlist that Vertex passes // into the Gemini converter. vertex.go reuses geminiImageURLSchemes across // ChatCompletion / ChatCompletionStream / Responses / ResponsesStream / CountTokens, diff --git a/core/providers/vertex/vertex.go b/core/providers/vertex/vertex.go index 18b1e027f95..de0dd5fcfc7 100644 --- a/core/providers/vertex/vertex.go +++ b/core/providers/vertex/vertex.go @@ -631,7 +631,7 @@ func (provider *VertexProvider) ChatCompletion(ctx *schemas.BifrostContext, key completeURL = getVertexEndpointURL(region, "v1beta1", projectNumber, request.Model, ":generateContent") } else if schemas.IsAnthropicModelFamily(ctx, request.Model) { // Claude models use Anthropic publisher — model-aware host for multi-region support - completeURL = getVertexModelAwarePublisherModelURL(region, "v1", projectID, "anthropic", request.Model, ":rawPredict") + completeURL = getVertexModelAwarePublisherModelURL(region, "v1", projectID, "anthropic", request.Model, ":rawPredict", resolveVertexForceSingleRegion(ctx, key), provider.logger) } else if schemas.IsMistralModelFamily(ctx, request.Model) { // Mistral models use mistralai publisher with rawPredict completeURL = getVertexPublisherModelURL(region, "v1", projectID, "mistralai", request.Model, ":rawPredict") @@ -854,7 +854,7 @@ func (provider *VertexProvider) ChatCompletionStream(ctx *schemas.BifrostContext } } - completeURL := getVertexModelAwarePublisherModelURL(region, "v1", projectID, "anthropic", request.Model, ":streamRawPredict") + completeURL := getVertexModelAwarePublisherModelURL(region, "v1", projectID, "anthropic", request.Model, ":streamRawPredict", resolveVertexForceSingleRegion(ctx, key), provider.logger) // Prepare headers for Vertex Anthropic headers := map[string]string{ @@ -1074,7 +1074,7 @@ func (provider *VertexProvider) Responses(ctx *schemas.BifrostContext, key schem } // Claude models use Anthropic publisher — model-aware host for multi-region support - url := getVertexModelAwarePublisherModelURL(region, "v1beta1", projectID, "anthropic", request.Model, ":rawPredict") + url := getVertexModelAwarePublisherModelURL(region, "v1beta1", projectID, "anthropic", request.Model, ":rawPredict", resolveVertexForceSingleRegion(ctx, key), provider.logger) // Create HTTP request for streaming req := fasthttp.AcquireRequest() @@ -1369,7 +1369,7 @@ func (provider *VertexProvider) ResponsesStream(ctx *schemas.BifrostContext, pos return nil, bifrostErr } - url := getVertexModelAwarePublisherModelURL(region, "v1", projectID, "anthropic", request.Model, ":streamRawPredict") + url := getVertexModelAwarePublisherModelURL(region, "v1", projectID, "anthropic", request.Model, ":streamRawPredict", resolveVertexForceSingleRegion(ctx, key), provider.logger) // Prepare headers for Vertex Anthropic headers := map[string]string{ @@ -4093,8 +4093,9 @@ func (provider *VertexProvider) CountTokens(ctx *schemas.BifrostContext, key sch if schemas.IsAnthropicModelFamily(ctx, request.Model) { // Use model-aware host based on request.Model, but URL path uses "count-tokens" - effectiveRegion := getVertexEffectiveRegion(region, request.Model) - baseURL := getVertexModelAwareAPIBaseURL(region, "v1", request.Model) + forceSingleRegion := resolveVertexForceSingleRegion(ctx, key) + effectiveRegion := getVertexEffectiveRegion(region, request.Model, forceSingleRegion) + baseURL := getVertexModelAwareAPIBaseURL(region, "v1", request.Model, forceSingleRegion, provider.logger) completeURL = fmt.Sprintf("%s/projects/%s/locations/%s/publishers/%s/models/%s%s", baseURL, projectID, effectiveRegion, "anthropic", "count-tokens", ":rawPredict") } else if schemas.IsGeminiModelFamily(ctx, request.Model) || schemas.IsAllDigitsASCII(request.Model) || schemas.IsGemmaModelFamily(ctx, request.Model) { if key.Value.GetValue() != "" { diff --git a/core/schemas/account.go b/core/schemas/account.go index 98383c99a5d..e8099aca038 100644 --- a/core/schemas/account.go +++ b/core/schemas/account.go @@ -190,8 +190,9 @@ type AzureAliasCfg struct { // VertexAliasCfg holds Vertex-specific overrides that apply to a single alias. type VertexAliasCfg struct { - ProjectID *SecretVar `json:"project_id,omitempty"` - ProjectNumber *SecretVar `json:"project_number,omitempty"` + ProjectID *SecretVar `json:"project_id,omitempty"` + ProjectNumber *SecretVar `json:"project_number,omitempty"` + ForceSingleRegion *bool `json:"force_single_region,omitempty"` } // BedrockAliasCfg holds Bedrock-specific overrides that apply to a single alias. @@ -628,6 +629,9 @@ type VertexKeyConfig struct { ProjectNumber SecretVar `json:"project_number"` Region SecretVar `json:"region"` AuthCredentials SecretVar `json:"auth_credentials"` + // ForceSingleRegion pins requests to the configured region and disables automatic promotion of + // multi-region-only models to a multi-region pool endpoint (e.g. for provisioned throughput). + ForceSingleRegion bool `json:"force_single_region,omitempty"` } // NOTE: To use Vertex IAM role authentication, set AuthCredentials to empty string. diff --git a/docs/providers/supported-providers/vertex.mdx b/docs/providers/supported-providers/vertex.mdx index 3fe7828ce41..993d1abc0b1 100644 --- a/docs/providers/supported-providers/vertex.mdx +++ b/docs/providers/supported-providers/vertex.mdx @@ -431,12 +431,13 @@ func (a *MyAccount) GetKeysForProvider(ctx *context.Context, provider schemas.Mo **`vertex_key_config` fields:** -| Field | Required | Description | -| ------------------ | -------- | ------------------------------------------------------ | -| `project_id` | Yes | Google Cloud project ID | -| `region` | Yes | GCP region (e.g., `us-central1`, `eu-west1`, `global`) | -| `auth_credentials` | No | Service account JSON string (leave empty for ADC) | -| `project_number` | No | GCP project number (required for fine-tuned models) | +| Field | Required | Description | +| --------------------- | -------- | -------------------------------------------------------------------------------------------------- | +| `project_id` | Yes | Google Cloud project ID | +| `region` | Yes | GCP region (e.g., `us-central1`, `eu-west1`, `global`) | +| `auth_credentials` | No | Service account JSON string (leave empty for ADC) | +| `project_number` | No | GCP project number (required for fine-tuned models) | +| `force_single_region` | No | When `true`, always call `region` and skip multi-region promotion of multi-region-only models | **Key-level fields:** @@ -694,6 +695,10 @@ The region determines the API endpoint: Availability varies by region. Check [GCP documentation](https://cloud.google.com/vertex-ai/docs/general/locations) for model availability. +### Multi-region-only models + +Some models (e.g. Claude Opus 4.7/4.8) are published only on Google's multi-region pool endpoints. When such a model is requested with a single US or Europe region, Bifrost automatically promotes the call to the matching pool endpoint (`aiplatform.us.rep.googleapis.com` / `aiplatform.eu.rep.googleapis.com`) and rewrites the `locations/` path segment to `us` / `eu`. Set `force_single_region: true` on the Vertex key (or per-deployment) to disable this and keep the exact region you configured — for example when you serve these models from a single region via provisioned throughput. Endpoint rewrites are emitted at `debug` log level. + ## Streaming Streaming format depends on model type: diff --git a/framework/configstore/clientconfig.go b/framework/configstore/clientconfig.go index 31766a3f32a..ccc16938dc2 100644 --- a/framework/configstore/clientconfig.go +++ b/framework/configstore/clientconfig.go @@ -544,6 +544,7 @@ func (p *ProviderConfig) Redacted() *ProviderConfig { vertexConfig.ProjectNumber = *key.VertexKeyConfig.ProjectNumber.Redacted() vertexConfig.Region = *key.VertexKeyConfig.Region.Redacted() vertexConfig.AuthCredentials = *key.VertexKeyConfig.AuthCredentials.Redacted() + vertexConfig.ForceSingleRegion = key.VertexKeyConfig.ForceSingleRegion redactedConfig.Keys[i].VertexKeyConfig = vertexConfig } diff --git a/framework/configstore/encryption_test.go b/framework/configstore/encryption_test.go index 60233f88b5a..ad175a97cea 100644 --- a/framework/configstore/encryption_test.go +++ b/framework/configstore/encryption_test.go @@ -858,10 +858,11 @@ func TestBeforeSave_DoesNotMutateSharedProviderConfigs(t *testing.T) { azureCfg.ClientSecret = schemas.NewSecretVar("azure-client-secret") vertexCfg := &schemas.VertexKeyConfig{ - ProjectID: *schemas.NewSecretVar("my-project"), - ProjectNumber: *schemas.NewSecretVar("123456789"), - Region: *schemas.NewSecretVar("us-central1"), - AuthCredentials: *schemas.NewSecretVar("vertex-creds"), + ProjectID: *schemas.NewSecretVar("my-project"), + ProjectNumber: *schemas.NewSecretVar("123456789"), + Region: *schemas.NewSecretVar("us-central1"), + AuthCredentials: *schemas.NewSecretVar("vertex-creds"), + ForceSingleRegion: true, } bedrockCfg := &schemas.BedrockKeyConfig{ @@ -908,6 +909,8 @@ func TestBeforeSave_DoesNotMutateSharedProviderConfigs(t *testing.T) { "BeforeSave must not mutate shared VertexKeyConfig.ProjectNumber") assert.Equal(t, "us-central1", vertexCfg.Region.GetValue(), "BeforeSave must not mutate shared VertexKeyConfig.Region") + assert.True(t, vertexCfg.ForceSingleRegion, + "BeforeSave must not mutate shared VertexKeyConfig.ForceSingleRegion") // Bedrock: encrypted fields assert.Equal(t, "AKIAEXAMPLE", bedrockCfg.AccessKey.GetValue(), @@ -935,6 +938,8 @@ func TestBeforeSave_DoesNotMutateSharedProviderConfigs(t *testing.T) { assert.Equal(t, "my-project", found.VertexKeyConfig.ProjectID.GetValue()) assert.Equal(t, "123456789", found.VertexKeyConfig.ProjectNumber.GetValue()) assert.Equal(t, "us-central1", found.VertexKeyConfig.Region.GetValue()) + assert.True(t, found.VertexKeyConfig.ForceSingleRegion, + "ForceSingleRegion must survive the save/reload round-trip") require.NotNil(t, found.BedrockKeyConfig) assert.Equal(t, "AKIAEXAMPLE", found.BedrockKeyConfig.AccessKey.GetValue()) assert.Equal(t, "secret-key", found.BedrockKeyConfig.SecretKey.GetValue()) diff --git a/framework/configstore/migrations.go b/framework/configstore/migrations.go index 561b6bdbf33..74a6c785db0 100644 --- a/framework/configstore/migrations.go +++ b/framework/configstore/migrations.go @@ -436,6 +436,7 @@ var configstoreMigrationSteps = []migrationStep{ {IDs: []string{"add_model_pricing_is_deprecated_column"}, run: migrationAddModelPricingIsDeprecatedColumn}, {IDs: []string{"add_mcp_client_tool_execution_timeout_column"}, run: migrationAddMCPClientToolExecutionTimeoutColumn}, {IDs: []string{"add_virtual_key_expires_at_column"}, run: migrationAddVirtualKeyExpiresAtColumn}, + {IDs: []string{"add_vertex_force_single_region_column"}, run: migrationAddVertexForceSingleRegionColumn}, } // quoteSQLiteIdentifier quotes a SQLite identifier, escaping any double quotes. @@ -10316,3 +10317,26 @@ func migrationAddVirtualKeyExpiresAtColumn(ctx context.Context, db *gorm.DB, log } return nil } + +// migrationAddVertexForceSingleRegionColumn adds the vertex_force_single_region column to the key table. +// Existing keys default to false (NULL), preserving the current multi-region promotion behaviour. +func migrationAddVertexForceSingleRegionColumn(ctx context.Context, db *gorm.DB, logger schemas.Logger) error { + migrationName := "add_vertex_force_single_region_column" + logger.Info("[configstore] starting migration %s", migrationName) + defer logger.Info("[configstore] finished migration %s", migrationName) + m := migrator.New(db, migrator.DefaultOptions, []*migrator.Migration{{ + ID: migrationName, + Migrate: func(tx *gorm.DB) error { + tx = tx.WithContext(ctx) + return addColumnIfNotExists(tx, logger, &tables.TableKey{}, "vertex_force_single_region") + }, + Rollback: func(tx *gorm.DB) error { + tx = tx.WithContext(ctx) + return dropColumnIfExists(tx, logger, &tables.TableKey{}, "vertex_force_single_region") + }, + }}) + if err := m.Migrate(); err != nil { + return fmt.Errorf("error running %s migration: %w", migrationName, err) + } + return nil +} diff --git a/framework/configstore/rdb.go b/framework/configstore/rdb.go index 524cf1b2cb5..a00f2357975 100644 --- a/framework/configstore/rdb.go +++ b/framework/configstore/rdb.go @@ -202,6 +202,7 @@ func tableKeyFromSchemaKey(provider tables.TableProvider, key schemas.Key) (tabl dbKey.VertexProjectNumber = &key.VertexKeyConfig.ProjectNumber dbKey.VertexRegion = &key.VertexKeyConfig.Region dbKey.VertexAuthCredentials = &key.VertexKeyConfig.AuthCredentials + dbKey.VertexForceSingleRegion = &key.VertexKeyConfig.ForceSingleRegion } if key.BedrockKeyConfig != nil { @@ -736,6 +737,7 @@ func (s *RDBConfigStore) UpdateProvidersConfig(ctx context.Context, providers ma dbKey.VertexProjectNumber = &key.VertexKeyConfig.ProjectNumber dbKey.VertexRegion = &key.VertexKeyConfig.Region dbKey.VertexAuthCredentials = &key.VertexKeyConfig.AuthCredentials + dbKey.VertexForceSingleRegion = &key.VertexKeyConfig.ForceSingleRegion } // Handle Bedrock config @@ -965,6 +967,7 @@ func (s *RDBConfigStore) UpdateProvider(ctx context.Context, provider schemas.Mo dbKey.VertexProjectNumber = &key.VertexKeyConfig.ProjectNumber dbKey.VertexRegion = &key.VertexKeyConfig.Region dbKey.VertexAuthCredentials = &key.VertexKeyConfig.AuthCredentials + dbKey.VertexForceSingleRegion = &key.VertexKeyConfig.ForceSingleRegion } // Handle Bedrock config @@ -1103,6 +1106,7 @@ func (s *RDBConfigStore) AddProvider(ctx context.Context, provider schemas.Model dbKey.VertexProjectNumber = &key.VertexKeyConfig.ProjectNumber dbKey.VertexRegion = &key.VertexKeyConfig.Region dbKey.VertexAuthCredentials = &key.VertexKeyConfig.AuthCredentials + dbKey.VertexForceSingleRegion = &key.VertexKeyConfig.ForceSingleRegion } // Handle Bedrock config if key.BedrockKeyConfig != nil { diff --git a/framework/configstore/tables/key.go b/framework/configstore/tables/key.go index 3eec1f6c3db..d80d38636b4 100644 --- a/framework/configstore/tables/key.go +++ b/framework/configstore/tables/key.go @@ -40,10 +40,11 @@ type TableKey struct { AzureScopesJSON *string `gorm:"column:azure_scopes;type:text" json:"-"` // JSON serialized []string // Vertex config fields (embedded) - VertexProjectID *schemas.SecretVar `gorm:"type:text" json:"vertex_project_id,omitempty"` - VertexProjectNumber *schemas.SecretVar `gorm:"type:text" json:"vertex_project_number,omitempty"` - VertexRegion *schemas.SecretVar `gorm:"type:text" json:"vertex_region,omitempty"` - VertexAuthCredentials *schemas.SecretVar `gorm:"type:text" json:"vertex_auth_credentials,omitempty"` + VertexProjectID *schemas.SecretVar `gorm:"type:text" json:"vertex_project_id,omitempty"` + VertexProjectNumber *schemas.SecretVar `gorm:"type:text" json:"vertex_project_number,omitempty"` + VertexRegion *schemas.SecretVar `gorm:"type:text" json:"vertex_region,omitempty"` + VertexAuthCredentials *schemas.SecretVar `gorm:"type:text" json:"vertex_auth_credentials,omitempty"` + VertexForceSingleRegion *bool `gorm:"column:vertex_force_single_region" json:"vertex_force_single_region,omitempty"` // Bedrock config fields (embedded) BedrockAccessKey *schemas.SecretVar `gorm:"type:text" json:"bedrock_access_key,omitempty"` @@ -205,11 +206,14 @@ func (k *TableKey) BeforeSave(tx *gorm.DB) error { } else { k.VertexAuthCredentials = nil } + fsr := k.VertexKeyConfig.ForceSingleRegion + k.VertexForceSingleRegion = &fsr } else { k.VertexProjectID = nil k.VertexProjectNumber = nil k.VertexRegion = nil k.VertexAuthCredentials = nil + k.VertexForceSingleRegion = nil } if k.BedrockKeyConfig != nil { if k.BedrockKeyConfig.AccessKey.IsSet() { @@ -647,7 +651,7 @@ func (k *TableKey) AfterFind(tx *gorm.DB) error { k.AzureKeyConfig = azureConfig } // Reconstruct Vertex config if fields are present - if k.VertexProjectID != nil || k.VertexProjectNumber != nil || k.VertexRegion != nil || k.VertexAuthCredentials != nil { + if k.VertexProjectID != nil || k.VertexProjectNumber != nil || k.VertexRegion != nil || k.VertexAuthCredentials != nil || k.VertexForceSingleRegion != nil { config := &schemas.VertexKeyConfig{} if k.VertexProjectID != nil { @@ -664,6 +668,9 @@ func (k *TableKey) AfterFind(tx *gorm.DB) error { if k.VertexAuthCredentials != nil { config.AuthCredentials = *k.VertexAuthCredentials } + if k.VertexForceSingleRegion != nil { + config.ForceSingleRegion = *k.VertexForceSingleRegion + } k.VertexKeyConfig = config } // Reconstruct Bedrock config if fields are present diff --git a/transports/config.schema.json b/transports/config.schema.json index 50c8f8a766e..e705346bd14 100644 --- a/transports/config.schema.json +++ b/transports/config.schema.json @@ -3345,6 +3345,10 @@ "type": "string", "description": "Per-alias Vertex project number override (can use env. prefix)." }, + "force_single_region": { + "type": "boolean", + "description": "Per-alias Vertex override: when true, call the alias region as-is and skip promoting multi-region-only models to a multi-region endpoint." + }, "inference_profile_arn": { "type": "string", "description": "Per-alias Bedrock inference profile ARN (can use env. prefix)." @@ -3680,6 +3684,11 @@ "auth_credentials": { "type": "string", "description": "Authentication credentials (can use env. prefix)" + }, + "force_single_region": { + "type": "boolean", + "description": "When true, always call the configured region and skip automatic promotion of multi-region-only models to a multi-region endpoint. Enable for provisioned throughput.", + "default": false } }, "required": ["project_id", "region"], diff --git a/ui/app/workspace/providers/fragments/apiKeysFormFragment.tsx b/ui/app/workspace/providers/fragments/apiKeysFormFragment.tsx index fe461720af0..cb8c3e09d8f 100644 --- a/ui/app/workspace/providers/fragments/apiKeysFormFragment.tsx +++ b/ui/app/workspace/providers/fragments/apiKeysFormFragment.tsx @@ -608,6 +608,10 @@ export function ApiKeyFormFragment({ control, providerName, baseProviderType, fo render={({ field }) => ( Region (Required) + + Multi-region-only models are automatically routed to Google's matching multi-region endpoint. Turn on{" "} + Force single region below to always use exactly this region. + @@ -661,6 +665,24 @@ export function ApiKeyFormFragment({ control, providerName, baseProviderType, fo )} /> )} + ( + +
+ Force single region + + Always call the region set above and skip automatic promotion of multi-region-only models to a + multi-region endpoint. Enable when serving these models from a single region via provisioned throughput. + +
+ + + +
+ )} + /> {supportsBatchAPI && } )} diff --git a/ui/app/workspace/providers/fragments/deploymentsTable.tsx b/ui/app/workspace/providers/fragments/deploymentsTable.tsx index 1ec7190d668..300ed34439c 100644 --- a/ui/app/workspace/providers/fragments/deploymentsTable.tsx +++ b/ui/app/workspace/providers/fragments/deploymentsTable.tsx @@ -183,7 +183,7 @@ function VertexSection({ config, onChange, disabled }: ProviderSectionProps) { disabled={disabled} /> - + onChange({ region: v })} @@ -191,6 +191,20 @@ function VertexSection({ config, onChange, disabled }: ProviderSectionProps) { disabled={disabled} /> +
+
+ +

+ Call the region above as-is and skip multi-region promotion of multi-region-only models. Use for provisioned + throughput. +

+
+ onChange({ force_single_region: checked })} + disabled={disabled} + /> +
); } diff --git a/ui/lib/schemas/providerForm.ts b/ui/lib/schemas/providerForm.ts index c47583cee63..595ed239d83 100644 --- a/ui/lib/schemas/providerForm.ts +++ b/ui/lib/schemas/providerForm.ts @@ -89,6 +89,7 @@ const VertexKeyConfigSchema = z.object({ .refine((value) => !value || isValidVertexAuthCredentials(value), { message: "Auth Credentials must be a valid JSON object or env.VAR format when provided", }), + force_single_region: z.boolean().optional(), }); // S3 bucket configuration for Bedrock batch operations diff --git a/ui/lib/types/config.ts b/ui/lib/types/config.ts index 291862a6514..95773452c68 100644 --- a/ui/lib/types/config.ts +++ b/ui/lib/types/config.ts @@ -65,6 +65,7 @@ export interface AliasConfig { // Vertex overrides project_id?: SecretVar; project_number?: SecretVar; + force_single_region?: boolean; // Bedrock overrides inference_profile_arn?: SecretVar; // Replicate overrides @@ -94,6 +95,7 @@ export interface VertexKeyConfig { project_number?: SecretVar; region: SecretVar; auth_credentials?: SecretVar; + force_single_region?: boolean; } export const DefaultVertexKeyConfig: VertexKeyConfig = { @@ -101,6 +103,7 @@ export const DefaultVertexKeyConfig: VertexKeyConfig = { project_number: { value: "", ref: "" }, region: { value: "", ref: "" }, auth_credentials: { value: "", ref: "" }, + force_single_region: false, } as const satisfies Required; export interface S3BucketConfig { diff --git a/ui/lib/types/schemas.ts b/ui/lib/types/schemas.ts index 2c2ec12e8e8..5cbda456d82 100644 --- a/ui/lib/types/schemas.ts +++ b/ui/lib/types/schemas.ts @@ -116,6 +116,7 @@ export const vertexKeyConfigSchema = z project_number: secretVarSchema.optional(), region: secretVarSchema.optional(), auth_credentials: secretVarSchema.optional(), + force_single_region: z.boolean().optional(), }) .refine((data) => isSecretVarSet(data.project_id), { message: "Project ID is required", @@ -293,6 +294,7 @@ const aliasConfigObjectSchema = z.object({ // Vertex overrides project_id: secretVarSchema.optional(), project_number: secretVarSchema.optional(), + force_single_region: z.boolean().optional(), // Bedrock overrides inference_profile_arn: secretVarSchema.optional(), // Replicate overrides