diff --git a/.github/ISSUE_TEMPLATE/mcp_library_submission.yml b/.github/ISSUE_TEMPLATE/mcp_library_submission.yml new file mode 100644 index 0000000000..e3d6284d90 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/mcp_library_submission.yml @@ -0,0 +1,135 @@ +name: "Add MCP Server to Library" +description: "Propose a new MCP server for the Bifrost community library" +title: "[MCP Library] Add: " +labels: ["community", "mcp-library"] +body: + - type: markdown + attributes: + value: | + Thanks for contributing to the Bifrost MCP Library. Please fill out the details below, then open a PR editing `community/mcp-library/servers.json`. + + Do not include secrets, tokens, API keys, private URLs, or personal data. + + - type: input + id: name + attributes: + label: Server name + description: "Human-readable display name. Bifrost derives the internal slug from this name." + placeholder: "My Server" + validations: + required: true + + - type: textarea + id: description + attributes: + label: Description + description: "What does this MCP server do? Keep it concise and factual." + validations: + required: true + + - type: input + id: category + attributes: + label: Category + description: "Reuse an existing category from community/mcp-library/README.md when possible." + placeholder: "Developer Tools" + validations: + required: true + + - type: dropdown + id: connection_type + attributes: + label: Connection type + options: + - http + - stdio + - sse + - inprocess + validations: + required: true + + - type: input + id: connection_url + attributes: + label: Connection URL + description: "Required for http or sse servers. Leave blank for stdio servers." + placeholder: "https://api.example.com/mcp" + + - type: textarea + id: stdio_config + attributes: + label: STDIO config + description: "Required for stdio servers. Include command, args, and environment variable names only." + value: | + command: + args: + envs: + + - type: dropdown + id: auth_type + attributes: + label: Auth type + options: + - none + - headers + - oauth + - per_user_oauth + - per_user_headers + validations: + required: true + + - type: input + id: required_header_keys + attributes: + label: Required header keys + description: "Header names only, comma-separated. Never include values." + placeholder: "Authorization, x-api-key" + + - type: input + id: docs_url + attributes: + label: Documentation URL + description: "Link to the server's docs, repo, or homepage." + placeholder: "https://github.com/example/mcp-server" + validations: + required: true + + - type: input + id: publisher + attributes: + label: Publisher + description: "Person or organization that maintains this server." + validations: + required: true + + - type: input + id: icon_url + attributes: + label: Icon URL or path + description: "Optional square PNG or SVG icon URL/path." + placeholder: "/images/mcp-servers/example.svg" + + - type: input + id: tags + attributes: + label: Tags + description: "Short tags, comma-separated." + placeholder: "api, docs, developer-tools" + + - type: checkboxes + id: checklist + attributes: + label: Checklist + options: + - label: "I searched `servers.json` and this is not a duplicate." + required: true + - label: "I did not include secrets, tokens, API keys, private URLs, or personal data." + required: true + - label: "I did not add `slug` or `version` fields." + required: true + + - type: textarea + id: additional + attributes: + label: Additional context + description: "Anything else maintainers should know?" diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ebc640b384..f6fb54b2e4 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -314,3 +314,8 @@ updates: directory: /tests/cmd/seedvks schedule: interval: weekly + + - package-ecosystem: gomod + directory: /plugins/modelcatalogresolver + schedule: + interval: weekly diff --git a/.github/workflows/scripts/run-migration-tests.sh b/.github/workflows/scripts/run-migration-tests.sh index 9be3b35fa0..c9853707d1 100755 --- a/.github/workflows/scripts/run-migration-tests.sh +++ b/.github/workflows/scripts/run-migration-tests.sh @@ -1801,6 +1801,62 @@ append_dynamic_columns_postgres() { echo "UPDATE logs SET inc_number = NULL WHERE id = 'log-migration-test-002';" >> "$output_file" echo "UPDATE logs SET inc_number = NULL WHERE id = 'log-migration-test-003';" >> "$output_file" fi + + # ------------------------------------------------------------------------- + # v1.5.9 columns - config store tables + # ------------------------------------------------------------------------- + + # governance_budgets.model_config_id (added in v1.5.9 via add_budget_model_config_id_column) + if column_exists_postgres "governance_budgets" "model_config_id"; then + echo "UPDATE governance_budgets SET model_config_id = NULL WHERE id = 'budget-migration-test-1';" >> "$output_file" + echo "UPDATE governance_budgets SET model_config_id = NULL WHERE id = 'budget-migration-test-2';" >> "$output_file" + echo "UPDATE governance_budgets SET model_config_id = NULL WHERE id = 'budget-migration-test-3';" >> "$output_file" + fi + + # governance_budgets.customer_id (added in v1.5.9 via add_customer_budgets_to_budgets_table) + if column_exists_postgres "governance_budgets" "customer_id"; then + echo "UPDATE governance_budgets SET customer_id = NULL WHERE id = 'budget-migration-test-1';" >> "$output_file" + echo "UPDATE governance_budgets SET customer_id = NULL WHERE id = 'budget-migration-test-2';" >> "$output_file" + echo "UPDATE governance_budgets SET customer_id = NULL WHERE id = 'budget-migration-test-3';" >> "$output_file" + fi + + # governance_customers.calendar_aligned (added in v1.5.9 via add_customer_calendar_aligned_column) + if column_exists_postgres "governance_customers" "calendar_aligned"; then + echo "UPDATE governance_customers SET calendar_aligned = false WHERE id = 'customer-migration-test-1';" >> "$output_file" + echo "UPDATE governance_customers SET calendar_aligned = false WHERE id = 'customer-migration-test-2';" >> "$output_file" + fi + + # governance_model_configs.scope / scope_id (added in v1.5.9 via add_model_config_scope_columns) + # scope is set to the column default 'global'; scope_id stays NULL so the seeded rows keep + # satisfying the idx_model_scope_provider unique index + if column_exists_postgres "governance_model_configs" "scope"; then + echo "UPDATE governance_model_configs SET scope = 'global' WHERE id = 'model-config-migration-test-1';" >> "$output_file" + echo "UPDATE governance_model_configs SET scope = 'global' WHERE id = 'model-config-migration-test-2';" >> "$output_file" + fi + if column_exists_postgres "governance_model_configs" "scope_id"; then + echo "UPDATE governance_model_configs SET scope_id = NULL WHERE id = 'model-config-migration-test-1';" >> "$output_file" + echo "UPDATE governance_model_configs SET scope_id = NULL WHERE id = 'model-config-migration-test-2';" >> "$output_file" + fi + + # governance_model_configs.calendar_aligned (added in v1.5.9 via add_model_config_calendar_aligned_column) + if column_exists_postgres "governance_model_configs" "calendar_aligned"; then + echo "UPDATE governance_model_configs SET calendar_aligned = false WHERE id = 'model-config-migration-test-1';" >> "$output_file" + echo "UPDATE governance_model_configs SET calendar_aligned = false WHERE id = 'model-config-migration-test-2';" >> "$output_file" + fi + + # ------------------------------------------------------------------------- + # v1.5.9 columns - log store tables + # ------------------------------------------------------------------------- + + # logs multi-team/BU/customer JSON-array columns (added in v1.5.9 via + # logs_add_multi_team_business_unit_columns and logs_add_customer_array_columns) + for arr_col in team_ids team_names customer_ids customer_names business_unit_ids business_unit_names; do + if column_exists_postgres "logs" "$arr_col"; then + echo "UPDATE logs SET $arr_col = NULL WHERE id = 'log-migration-test-001';" >> "$output_file" + echo "UPDATE logs SET $arr_col = NULL WHERE id = 'log-migration-test-002';" >> "$output_file" + echo "UPDATE logs SET $arr_col = NULL WHERE id = 'log-migration-test-003';" >> "$output_file" + fi + done } # Append dynamic column UPDATEs for columns that may not exist in older schemas (SQLite) @@ -2792,6 +2848,59 @@ append_dynamic_columns_sqlite() { echo "UPDATE logs SET inc_number = NULL WHERE id = 'log-migration-test-001';" >> "$output_file" echo "UPDATE logs SET inc_number = NULL WHERE id = 'log-migration-test-002';" >> "$output_file" echo "UPDATE logs SET inc_number = NULL WHERE id = 'log-migration-test-003';" >> "$output_file" + + # ------------------------------------------------------------------------- + # v1.5.9 columns + # ------------------------------------------------------------------------- + + if [ -f "$config_db" ]; then + # governance_budgets.model_config_id (added in v1.5.9 via add_budget_model_config_id_column) + if column_exists_sqlite "$config_db" "governance_budgets" "model_config_id"; then + echo "UPDATE governance_budgets SET model_config_id = NULL WHERE id = 'budget-migration-test-1';" >> "$output_file" + echo "UPDATE governance_budgets SET model_config_id = NULL WHERE id = 'budget-migration-test-2';" >> "$output_file" + echo "UPDATE governance_budgets SET model_config_id = NULL WHERE id = 'budget-migration-test-3';" >> "$output_file" + fi + + # governance_budgets.customer_id (added in v1.5.9 via add_customer_budgets_to_budgets_table) + if column_exists_sqlite "$config_db" "governance_budgets" "customer_id"; then + echo "UPDATE governance_budgets SET customer_id = NULL WHERE id = 'budget-migration-test-1';" >> "$output_file" + echo "UPDATE governance_budgets SET customer_id = NULL WHERE id = 'budget-migration-test-2';" >> "$output_file" + echo "UPDATE governance_budgets SET customer_id = NULL WHERE id = 'budget-migration-test-3';" >> "$output_file" + fi + + # governance_customers.calendar_aligned (added in v1.5.9 via add_customer_calendar_aligned_column) + if column_exists_sqlite "$config_db" "governance_customers" "calendar_aligned"; then + echo "UPDATE governance_customers SET calendar_aligned = 0 WHERE id = 'customer-migration-test-1';" >> "$output_file" + echo "UPDATE governance_customers SET calendar_aligned = 0 WHERE id = 'customer-migration-test-2';" >> "$output_file" + fi + + # governance_model_configs.scope / scope_id (added in v1.5.9 via add_model_config_scope_columns) + # scope is set to the column default 'global'; scope_id stays NULL so the seeded rows keep + # satisfying the idx_model_scope_provider unique index + if column_exists_sqlite "$config_db" "governance_model_configs" "scope"; then + echo "UPDATE governance_model_configs SET scope = 'global' WHERE id = 'model-config-migration-test-1';" >> "$output_file" + echo "UPDATE governance_model_configs SET scope = 'global' WHERE id = 'model-config-migration-test-2';" >> "$output_file" + fi + if column_exists_sqlite "$config_db" "governance_model_configs" "scope_id"; then + echo "UPDATE governance_model_configs SET scope_id = NULL WHERE id = 'model-config-migration-test-1';" >> "$output_file" + echo "UPDATE governance_model_configs SET scope_id = NULL WHERE id = 'model-config-migration-test-2';" >> "$output_file" + fi + + # governance_model_configs.calendar_aligned (added in v1.5.9 via add_model_config_calendar_aligned_column) + if column_exists_sqlite "$config_db" "governance_model_configs" "calendar_aligned"; then + echo "UPDATE governance_model_configs SET calendar_aligned = 0 WHERE id = 'model-config-migration-test-1';" >> "$output_file" + echo "UPDATE governance_model_configs SET calendar_aligned = 0 WHERE id = 'model-config-migration-test-2';" >> "$output_file" + fi + fi + + # logs multi-team/BU/customer JSON-array columns (added in v1.5.9 via + # logs_add_multi_team_business_unit_columns and logs_add_customer_array_columns) + # Emitted unconditionally - logs table is in logs_db; fails silently on config_db + for arr_col in team_ids team_names customer_ids customer_names business_unit_ids business_unit_names; do + echo "UPDATE logs SET $arr_col = NULL WHERE id = 'log-migration-test-001';" >> "$output_file" + echo "UPDATE logs SET $arr_col = NULL WHERE id = 'log-migration-test-002';" >> "$output_file" + echo "UPDATE logs SET $arr_col = NULL WHERE id = 'log-migration-test-003';" >> "$output_file" + done } # ============================================================================ diff --git a/.github/workflows/scripts/validate-helm-config-fields.sh b/.github/workflows/scripts/validate-helm-config-fields.sh index 4340413a90..40b61427d1 100755 --- a/.github/workflows/scripts/validate-helm-config-fields.sh +++ b/.github/workflows/scripts/validate-helm-config-fields.sh @@ -555,7 +555,6 @@ assert_field_value 'governance.providers[0].rate_limit_id' '.governance.provider assert_field_value 'governance.auth_config.admin_username' '.governance.auth_config.admin_username' '"admin"' assert_field_value 'governance.auth_config.admin_password' '.governance.auth_config.admin_password' '"secret"' assert_field_value 'governance.auth_config.is_enabled' '.governance.auth_config.is_enabled' 'true' -assert_field_value 'governance.auth_config.disable_auth_on_inference' '.governance.auth_config.disable_auth_on_inference' 'true' ############################################################################### # 5. Top-level Auth Config @@ -579,7 +578,6 @@ render_config "$TMPDIR/values-auth.yaml" assert_field_value 'auth_config.admin_username' '.auth_config.admin_username' '"root"' assert_field_value 'auth_config.admin_password' '.auth_config.admin_password' '"rootpass"' assert_field_value 'auth_config.is_enabled' '.auth_config.is_enabled' 'true' -assert_field_value 'auth_config.disable_auth_on_inference' '.auth_config.disable_auth_on_inference' 'false' ############################################################################### # 6. Plugins (telemetry, logging, governance, maxim, semantic_cache, otel, datadog, custom) @@ -656,12 +654,22 @@ bifrost: enabled: true config: service_name: "bifrost-dd" + ml_app: "bifrost-ml" agent_addr: "dd-agent:8126" + dogstatsd_addr: "dd-agent:8125" env: "staging" version: "1.0.0" custom_tags: team: "platform" + enable_metrics: true enable_traces: true + enable_llm_obs: true + disable_content_logging: false + agentless: true + api_key: "env.DD_API_KEY" + site: "datadoghq.com" + request_headers: + - "x-bf-session-id" custom: - name: "my-plugin" enabled: true @@ -727,11 +735,20 @@ assert_field_value 'plugins: otel insecure' '.plugins.[5].config.insecure' 'true # Datadog plugin assert_field_value 'plugins: datadog name' '.plugins.[6].name' '"datadog"' assert_field_value 'plugins: datadog service_name' '.plugins.[6].config.service_name' '"bifrost-dd"' +assert_field_value 'plugins: datadog ml_app' '.plugins.[6].config.ml_app' '"bifrost-ml"' assert_field_value 'plugins: datadog agent_addr' '.plugins.[6].config.agent_addr' '"dd-agent:8126"' +assert_field_value 'plugins: datadog dogstatsd_addr' '.plugins.[6].config.dogstatsd_addr' '"dd-agent:8125"' assert_field_value 'plugins: datadog env' '.plugins.[6].config.env' '"staging"' assert_field_value 'plugins: datadog version' '.plugins.[6].config.version' '"1.0.0"' assert_field 'plugins: datadog custom_tags' '.plugins.[6].config.custom_tags' +assert_field_value 'plugins: datadog enable_metrics' '.plugins.[6].config.enable_metrics' 'true' assert_field_value 'plugins: datadog enable_traces' '.plugins.[6].config.enable_traces' 'true' +assert_field_value 'plugins: datadog enable_llm_obs' '.plugins.[6].config.enable_llm_obs' 'true' +assert_field_value 'plugins: datadog disable_content_logging' '.plugins.[6].config.disable_content_logging' 'false' +assert_field_value 'plugins: datadog agentless' '.plugins.[6].config.agentless' 'true' +assert_field_value 'plugins: datadog api_key' '.plugins.[6].config.api_key' '"env.DD_API_KEY"' +assert_field_value 'plugins: datadog site' '.plugins.[6].config.site' '"datadoghq.com"' +assert_field 'plugins: datadog request_headers' '.plugins.[6].config.request_headers' # Custom plugin assert_field_value 'plugins: custom name' '.plugins.[7].name' '"my-plugin"' diff --git a/.gitignore b/.gitignore index 7733fb766b..63b8ebf7a0 100644 --- a/.gitignore +++ b/.gitignore @@ -183,4 +183,7 @@ examples/mcps/oauth-demo-server/oauth-demo-server # binaries tests/cmd/e2eseed/e2eseed -tests/cmd/seedvks/seedvks \ No newline at end of file +tests/cmd/seedvks/seedvks + +# routing harness ledgers (local run journals, never committed) +tests/e2e/api/routing/ledger-* diff --git a/Makefile b/Makefile index 7591a4f562..007c749fc3 100644 --- a/Makefile +++ b/Makefile @@ -1718,14 +1718,14 @@ install-newman: ## Install newman + htmlextra reporter if not already installed @$(USE_NODE); npm list -g newman-reporter-htmlextra > /dev/null 2>&1 || ($(ECHO) "$(YELLOW)Installing newman-reporter-htmlextra...$(NC)" && npm install -g newman-reporter-htmlextra) @$(ECHO) "$(GREEN)Newman + htmlextra are ready$(NC)" -run-provider-harness-test: $(if $(HELP),,install-newman) ## Run the Bifrost provider-harness Postman collection. HELP=1 prints full parameter docs. Filter via PROVIDER=openai|anthropic|bedrock|gemini|vertex|azure|passthrough, FEATURE="" or FEATURE="," (AND across substrings; matches request name/URL/body), RERUN_FAILED=1 (re-run only items that failed last run). INCLUDE_PREVIEW=1 to run [PREVIEW]-tagged account/region-scoped cases. SKIP_STREAM_CANCEL=1 skips stream cancellation probes. USE_INFISICAL=1 to source from Infisical (Usage: make run-provider-harness-test [HELP=1] [PROVIDER=anthropic] [FEATURE="web search"] [FEATURE="cross-cut,structured output"] [RERUN_FAILED=1] [INCLUDE_PREVIEW=1] [BASE_URL=...] [FOLDER="..."] [ENV_FILE=...] [VIEWER_PORT=8090] [CI=1]) +run-provider-harness-test: $(if $(HELP),,install-newman) ## Run the Bifrost provider-harness Postman collection. HELP=1 prints full parameter docs. Filter via PROVIDER=openai|anthropic|bedrock|gemini|vertex|azure|passthrough|openrouter, FEATURE="" or FEATURE="," (AND across substrings; matches request name/URL/body), RERUN_FAILED=1 (re-run only items that failed last run). INCLUDE_PREVIEW=1 to run [PREVIEW]-tagged account/region-scoped cases. SKIP_STREAM_CANCEL=1 skips stream cancellation probes. USE_INFISICAL=1 to source from Infisical (Usage: make run-provider-harness-test [HELP=1] [PROVIDER=anthropic] [FEATURE="web search"] [FEATURE="cross-cut,structured output"] [RERUN_FAILED=1] [INCLUDE_PREVIEW=1] [BASE_URL=...] [FOLDER="..."] [ENV_FILE=...] [VIEWER_PORT=8090] [CI=1]) @if [ -n "$(HELP)" ]; then \ printf '\n%s\n' "$(CYAN)run-provider-harness-test - Bifrost provider harness runner$(NC)"; \ printf '%s\n\n' "Runs the Bifrost provider-harness Postman collection through newman, with optional filtering."; \ printf '%s\n\n' "Includes §8 Criss-Cross: endpoint-shape × model-provider × modality matrix (chat, streaming, embeddings, audio, image gen, tools, vision, JSON, reasoning)."; \ printf '%s\n' "$(YELLOW)PARAMETERS$(NC)"; \ printf ' %-18s %s\n' "HELP=1" "Print this help and exit (no Bifrost or network activity)."; \ - printf ' %-18s %s\n' "PROVIDER=" "Filter requests by provider. One of: openai, anthropic, bedrock, gemini, vertex, azure, passthrough."; \ + printf ' %-18s %s\n' "PROVIDER=" "Filter requests by provider. One of: openai, anthropic, bedrock, gemini, vertex, azure, passthrough, openrouter."; \ printf ' %-18s %s\n' "" " Matches via PROVIDER_KEYWORDS in tests/e2e/api/runners/filter-collection.mjs (loose name/body substring)."; \ printf ' %-18s %s\n' "FEATURE=\"\"" "Filter by case-insensitive keyword(s) against the full request JSON (name + URL + body + ancestor folder names)."; \ printf ' %-18s %s\n' "" " Single: FEATURE=\"web search\". Multi-keyword AND (comma-separated): FEATURE=\"cross-cut,structured output\"."; \ @@ -1743,6 +1743,8 @@ run-provider-harness-test: $(if $(HELP),,install-newman) ## Run the Bifrost prov printf ' %-18s %s\n' "PARALLEL=0" "Disable per-provider parallelism (default: ON). When ON, forks one newman per provider (openai, anthropic, bedrock, gemini, vertex, azure) concurrently; reports merged into tmp/newman-report.json. The htmlextra report is only emitted in sequential mode (PARALLEL=0)."; \ printf ' %-18s %s\n' "SKIP_STREAM_CANCEL=1" "Skip the post-Newman stream-abort probes that verify server-side cancellation on client disconnect."; \ printf ' %-18s %s\n' "USE_INFISICAL=1" "Source secrets from Infisical CLI ('infisical export --path /local --format dotenv') instead of .env."; \ + printf ' %-18s %s\n' "VERTEX_GCS_BUCKET" "Env-sourced (.env/Infisical): GCS bucket for Vertex file ops (forwarded to Newman as vertexGcsBucket)."; \ + printf ' %-18s %s\n' "VERTEX_GCS_PREFIX" "Env-sourced: GCS object prefix for Vertex file ops (forwarded as vertexGcsPrefix)."; \ printf '\n%s\n' "$(YELLOW)EXAMPLES$(NC)"; \ printf ' %s\n' "make run-provider-harness-test HELP=1"; \ printf ' %s\n' "make run-provider-harness-test # full provider sweep"; \ @@ -1872,11 +1874,11 @@ run-provider-harness-test: $(if $(HELP),,install-newman) ## Run the Bifrost prov $(USE_NODE); \ PARALLEL_VAL="$(or $(PARALLEL),1)"; \ if [ "$$PARALLEL_VAL" != "0" ] && [ -n "$$PARALLEL_VAL" ]; then \ - $(ECHO) "$(CYAN)Parallel mode (default): forking one newman per provider (openai, anthropic, bedrock, gemini, vertex, azure, passthrough). Set PARALLEL=0 to disable.$(NC)"; \ + $(ECHO) "$(CYAN)Parallel mode (default): forking one newman per provider (openai, anthropic, bedrock, gemini, vertex, azure, passthrough, openrouter). Set PARALLEL=0 to disable.$(NC)"; \ rm -f tmp/newman-report-*.json tmp/newman-cli-*.log tmp/parallel-pids tmp/parallel-status; \ : > tmp/parallel-pids; \ : > tmp/parallel-status; \ - PROVIDERS="openai anthropic bedrock gemini vertex azure passthrough"; \ + PROVIDERS="openai anthropic bedrock gemini vertex azure passthrough openrouter"; \ if [ -n "$(PROVIDER)" ]; then PROVIDERS="$(PROVIDER)"; fi; \ LAUNCHED=0; \ for p in $$PROVIDERS; do \ @@ -1895,6 +1897,8 @@ run-provider-harness-test: $(if $(HELP),,install-newman) ## Run the Bifrost prov $(if $(filter 1 true TRUE yes YES y Y,$(INCLUDE_SKIP)),--env-var "include_skip=1",) \ $${BEDROCK_GUARDRAIL_IDENTIFIER:+--env-var "bedrockGuardrailIdentifier=$$BEDROCK_GUARDRAIL_IDENTIFIER"} \ $${BEDROCK_GUARDRAIL_VERSION:+--env-var "bedrockGuardrailVersion=$$BEDROCK_GUARDRAIL_VERSION"} \ + $${VERTEX_GCS_BUCKET:+--env-var "vertexGcsBucket=$$VERTEX_GCS_BUCKET"} \ + $${VERTEX_GCS_PREFIX:+--env-var "vertexGcsPrefix=$$VERTEX_GCS_PREFIX"} \ $(if $(ENV_FILE),--environment $(ENV_FILE),) \ $(if $(FOLDER),--folder "$(FOLDER)",) \ --reporters cli,json \ @@ -1959,7 +1963,7 @@ run-provider-harness-test: $(if $(HELP),,install-newman) ## Run the Bifrost prov NEWMAN_EXIT=$$PFAILED; \ else \ SEQ_PROVIDERS="$(PROVIDER)"; \ - if [ -z "$$SEQ_PROVIDERS" ]; then SEQ_PROVIDERS="openai anthropic bedrock gemini vertex azure passthrough"; fi; \ + if [ -z "$$SEQ_PROVIDERS" ]; then SEQ_PROVIDERS="openai anthropic bedrock gemini vertex azure passthrough openrouter"; fi; \ if [ -t 1 ] && [ -z "$$CI" ] && [ -z "$(CI)" ]; then \ : > tmp/newman-cli.log; \ $(USE_NODE); node tests/e2e/api/runners/harness-monitor.mjs \ @@ -1975,6 +1979,8 @@ run-provider-harness-test: $(if $(HELP),,install-newman) ## Run the Bifrost prov $(if $(filter 1 true TRUE yes YES y Y,$(INCLUDE_SKIP)),--env-var "include_skip=1",) \ $${BEDROCK_GUARDRAIL_IDENTIFIER:+--env-var "bedrockGuardrailIdentifier=$$BEDROCK_GUARDRAIL_IDENTIFIER"} \ $${BEDROCK_GUARDRAIL_VERSION:+--env-var "bedrockGuardrailVersion=$$BEDROCK_GUARDRAIL_VERSION"} \ + $${VERTEX_GCS_BUCKET:+--env-var "vertexGcsBucket=$$VERTEX_GCS_BUCKET"} \ + $${VERTEX_GCS_PREFIX:+--env-var "vertexGcsPrefix=$$VERTEX_GCS_PREFIX"} \ $(if $(ENV_FILE),--environment $(ENV_FILE),) \ $(if $(FOLDER),--folder "$(FOLDER)",) \ --reporters cli,json,htmlextra \ @@ -1996,6 +2002,8 @@ run-provider-harness-test: $(if $(HELP),,install-newman) ## Run the Bifrost prov $(if $(filter 1 true TRUE yes YES y Y,$(INCLUDE_SKIP)),--env-var "include_skip=1",) \ $${BEDROCK_GUARDRAIL_IDENTIFIER:+--env-var "bedrockGuardrailIdentifier=$$BEDROCK_GUARDRAIL_IDENTIFIER"} \ $${BEDROCK_GUARDRAIL_VERSION:+--env-var "bedrockGuardrailVersion=$$BEDROCK_GUARDRAIL_VERSION"} \ + $${VERTEX_GCS_BUCKET:+--env-var "vertexGcsBucket=$$VERTEX_GCS_BUCKET"} \ + $${VERTEX_GCS_PREFIX:+--env-var "vertexGcsPrefix=$$VERTEX_GCS_PREFIX"} \ $(if $(ENV_FILE),--environment $(ENV_FILE),) \ $(if $(FOLDER),--folder "$(FOLDER)",) \ --reporters cli,json,htmlextra \ diff --git a/cli/changelog.md b/cli/changelog.md index bc0e9167ef..d5aee5138d 100644 --- a/cli/changelog.md +++ b/cli/changelog.md @@ -9,4 +9,4 @@ - improvement: model chooser treats manual model names (typed into the filter) as a distinct selectable row, with arrow-key wrap-around between the filtered list and the manual entry - improvement: hides the Bifrost logo when re-entering the harness/model/worktree phases from the summary, giving editing flows more vertical space - fix: tab command-mode key handling now correctly distinguishes `Enter` (activate the selected row) from `Esc`/prefix (resume the active tab), and recognises both `Ctrl+B` and `Ctrl+G` as the dismiss key -- fix: arrow-key escape sequences are now mapped to the existing `h`/`j`/`k`/`l` navigation in the command overlay, so users can navigate the tab popup with cursor keys +- fix: arrow-key escape sequences are now mapped to the existing `h`/`j`/`k`/`l` navigation in the command overlay, so users can navigate the tab popup with cursor keys \ No newline at end of file diff --git a/cli/go.mod b/cli/go.mod index 78e2fc6c58..5a140a4a3c 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -46,7 +46,7 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect golang.org/x/arch v0.23.0 // indirect - golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect + golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect golang.org/x/sys v0.45.0 // indirect golang.org/x/text v0.37.0 // indirect ) diff --git a/cli/go.sum b/cli/go.sum index 584cf141e0..6011b6603a 100644 --- a/cli/go.sum +++ b/cli/go.sum @@ -89,8 +89,8 @@ github.com/zalando/go-keyring v0.2.6 h1:r7Yc3+H+Ux0+M72zacZoItR3UDxeWfKTcabvkI8u github.com/zalando/go-keyring v0.2.6/go.mod h1:2TCrxYrbUNYfNS/Kgy/LSrkSQzZ5UPVH85RwfczwvcI= golang.org/x/arch v0.23.0 h1:lKF64A2jF6Zd8L0knGltUnegD62JMFBiCPBmQpToHhg= golang.org/x/arch v0.23.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= -golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0= -golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= diff --git a/community/README.md b/community/README.md new file mode 100644 index 0000000000..fc9a4b3068 --- /dev/null +++ b/community/README.md @@ -0,0 +1,45 @@ +# Community Contributions + +This directory contains community-maintained data that powers parts of the Bifrost platform. Contributions are reviewed by maintainers and, once merged into the `dev` branch, are synced to the live platform. + +## Available Catalogs + +| Catalog | Description | Status | +| ------- | ----------- | ------ | +| [MCP Library](./mcp-library/) | Community-curated catalog of MCP servers | Active | + +## How It Works + +``` +┌──────────────┐ PR ┌──────────────┐ Merge ┌──────────────┐ +│ Contributor │─────────────▶│ Maintainer │──────────────▶│ dev branch │ +│ edits JSON │ │ review │ │ (stable) │ +└──────────────┘ └──────────────┘ └──────┬───────┘ + │ + Periodic sync + │ + ┌──────▼───────┐ + │ Bifrost │ + │ Platform │ + └──────────────┘ +``` + +1. **Contribute** — Edit the relevant JSON file and open a pull request. +2. **Review** — Maintainers review your submission for accuracy, safety, and fit. +3. **Merge** — Once approved, your PR is merged into `dev`. +4. **Live** — Bifrost periodically reads the latest data from `dev` and serves it to all users. + +## General Guidelines + +- Follow each catalog's specific contributing guide before submitting. +- One logical change per pull request (e.g., one new MCP server per PR). +- Keep descriptions concise and factual. +- All contributions are subject to the project's [Code of Conduct](../CODE_OF_CONDUCT.md). + +## Adding a New Catalog + +If you'd like to propose a new community-maintained catalog, open a [feature request](https://github.com/maximhq/bifrost/issues/new?template=feature_request.yml) describing: + +- What data the catalog would contain +- How it would be consumed by the platform +- Example entries diff --git a/community/mcp-library/README.md b/community/mcp-library/README.md new file mode 100644 index 0000000000..5c5f49802a --- /dev/null +++ b/community/mcp-library/README.md @@ -0,0 +1,176 @@ +# MCP Library - Community Catalog + +This directory contains the community-curated catalog of [MCP (Model Context Protocol)](https://modelcontextprotocol.io) servers available in the Bifrost MCP Library. + +Anyone can add an MCP server by editing [`servers.json`](./servers.json) and opening a pull request. + +## Quick Start + +### 1. Fork and clone the repository + +```bash +git clone https://github.com//bifrost.git +cd bifrost +``` + +### 2. Add your server + +Add one entry to the `servers` array in `community/mcp-library/servers.json`. Use the field reference below and copy the shape of nearby entries. + +Bifrost generates the internal server slug from `name`. Do not add a `slug` field to catalog entries. + +### 3. Check your change locally + +```bash +jq empty community/mcp-library/servers.json +``` + +If you have `ajv-cli` installed, you can also validate against the schema: + +```bash +npx --yes ajv-cli validate \ + -s community/mcp-library/schema.json \ + -d community/mcp-library/servers.json \ + --spec=draft7 +``` + +### 4. Open a pull request + +Target the `dev` branch. Use a descriptive title like `community: add to MCP library`. + +## Server Entry Reference + +Each server entry is a JSON object in the `servers` array. + +### HTTP or SSE Server + +```json +{ + "name": "My Server", + "description": "A short, factual description of what this server does.", + "category": "Developer Tools", + "connection_type": "http", + "connection_url": "https://api.example.com/mcp", + "auth_type": "headers", + "required_header_keys": ["Authorization"], + "icon_url": "https://example.com/icon.png", + "docs_url": "https://docs.example.com", + "publisher": "Your Name or Organization", + "tags": ["api", "example"] +} +``` + +### STDIO Server + +```json +{ + "name": "My STDIO Server", + "description": "A local STDIO-based MCP server.", + "category": "Developer Tools", + "connection_type": "stdio", + "stdio_config": { + "command": "npx", + "args": ["-y", "@example/mcp-server"], + "envs": ["API_KEY"] + }, + "auth_type": "none", + "docs_url": "https://github.com/example/mcp-server", + "publisher": "Your Name", + "tags": ["local", "example"] +} +``` + +### Field Reference + +| Field | Type | Required | Description | +| ----- | ---- | -------- | ----------- | +| `name` | `string` | yes | Human-readable display name. Bifrost derives the internal slug from this value. | +| `description` | `string` | no | Short summary of what the server does. | +| `category` | `string` | no | Grouping category used for filtering. Reuse an existing category when possible. | +| `connection_type` | `string` | yes | One of `http`, `stdio`, `sse`, or `inprocess`. | +| `connection_url` | `string` | for `http`/`sse` | The server endpoint URL. | +| `stdio_config` | `object` | for `stdio` | Launch configuration. See [`stdio_config` fields](#stdio_config-fields). | +| `auth_type` | `string` | no | One of `none`, `headers`, `oauth`, `per_user_oauth`, or `per_user_headers`. Defaults to `none` when omitted. | +| `required_header_keys` | `string[]` | no | Header names required for header-based authentication. Never include secret values. | +| `icon_url` | `string` | no | URL or local path to a square icon. Prefer PNG or SVG. | +| `docs_url` | `string` | no | Link to documentation, repository, or homepage. | +| `publisher` | `string` | no | Person or organization maintaining the server. | +| `tags` | `string[]` | no | Search and filtering tags. Keep them short and factual. | +| `metadata` | `object` | no | Additional key/value data. Use sparingly. | + +#### `stdio_config` Fields + +| Field | Type | Required | Description | +| ----- | ---- | -------- | ----------- | +| `command` | `string` | yes | Executable to launch, such as `npx`, `uvx`, `node`, or `python`. | +| `args` | `string[]` | no | Arguments passed to the command. | +| `envs` | `string[]` | no | Environment variable names the user must provide. Never include values. | + +## Guidelines + +### Do + +- Add one server per pull request. +- Use a clear, stable `name`; changing it later changes Bifrost's generated internal slug. +- Write a concise, factual `description`. +- Include a `docs_url` so users can verify setup and requirements. +- Use `required_header_keys` and `stdio_config.envs` for key names only, never values. +- Verify the JSON parses before submitting. + +### Don't + +- Do not include secrets, tokens, API keys, credentials, private URLs, or personal data. +- Do not add duplicate servers; search `servers.json` first. +- Do not add a `slug` or `version` field. +- Do not add servers that require custom binaries unavailable through standard package managers or public installation instructions. +- Do not use promotional language or unverified claims. + +## Categories + +Reuse an existing category when possible: + +| Category | Examples | +| -------- | -------- | +| `AI Tools` | Hugging Face, evaluation, observability for AI workflows | +| `Analytics` | Product analytics, BI, data notebooks | +| `Communication` | Slack, email, messaging | +| `Customer Support` | Intercom, tickets, support conversations | +| `Design` | Figma, Canva, diagrams, whiteboards | +| `Developer Tools` | GitHub, GitLab, hosting, docs, databases, code search | +| `E-Commerce` | Shopify, stores, inventory, orders | +| `Finance` | Payments, billing, accounting, market data | +| `Human Resources` | Hiring, recruiting, people analytics | +| `Legal` | Legal research and documents | +| `Lifestyle` | Events, travel, media, personal workflows | +| `Marketing` | SEO, campaigns, brand analytics | +| `Productivity` | Notion, Google Drive, calendars, tasks | +| `Project Management` | Jira, Linear, Asana, boards, issues | +| `Research` | Academic research, citations, scientific data | +| `Sales` | CRM, prospecting, enrichment | +| `Search` | Web search, research, content extraction | +| `Security` | Compliance, scanning, security findings | +| `Travel` | Flights, hotels, trip planning | + +If none fit, use a short, descriptive new category and explain it in your PR description. + +## Review Criteria + +Maintainers may ask for changes or reject entries that are incomplete, unsafe, duplicative, hard to install, undocumented, or unrelated to MCP usage. + +## How Syncing Works + +1. You submit a PR adding your server to `servers.json`. +2. Maintainers review and merge it into the `dev` branch. +3. The Bifrost platform periodically fetches `servers.json` from the `dev` branch. +4. Bifrost derives an internal slug from `name`, upserts entries by that slug, and serves the catalog to users. + +Changing the `name` of an existing entry can create a new internal slug. Only rename an existing server when that is intentional. + +## Schema Validation + +The [`schema.json`](./schema.json) file contains a [JSON Schema draft-07](https://json-schema.org) definition for `servers.json`. Use it locally when you want editor hints or manual validation. + +## Questions? + +- Open an [issue](https://github.com/maximhq/bifrost/issues/new) for questions or problems. +- See the [Bifrost docs](https://docs.getbifrost.ai) for general platform documentation. diff --git a/community/mcp-library/schema.json b/community/mcp-library/schema.json new file mode 100644 index 0000000000..35f0857a14 --- /dev/null +++ b/community/mcp-library/schema.json @@ -0,0 +1,141 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/maximhq/bifrost/community/mcp-library/schema.json", + "title": "Bifrost MCP Library Catalog", + "description": "Schema for the community-maintained MCP server catalog served by Bifrost.", + "type": "object", + "required": ["servers"], + "additionalProperties": false, + "properties": { + "$schema": { + "type": "string", + "description": "Optional reference to this JSON Schema for editor IntelliSense." + }, + "lastUpdatedAt": { + "type": "string", + "description": "ISO 8601 timestamp of the last update. Optional; maintainers may bump this on merge." + }, + "servers": { + "type": "array", + "description": "The list of MCP servers in the catalog.", + "items": { "$ref": "#/definitions/server" } + } + }, + "definitions": { + "server": { + "type": "object", + "required": ["name", "connection_type"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "description": "Human-readable display name.", + "minLength": 1, + "maxLength": 255 + }, + "description": { + "type": "string", + "description": "Short, factual summary of what the server does.", + "maxLength": 1000 + }, + "category": { + "type": "string", + "description": "Single category used for grouping/filtering in the UI.", + "maxLength": 100 + }, + "connection_type": { + "type": "string", + "description": "How clients connect to the server.", + "enum": ["http", "stdio", "sse", "inprocess"] + }, + "connection_url": { + "type": "string", + "description": "Endpoint URL. Required for 'http' and 'sse' connection types.", + "minLength": 1 + }, + "stdio_config": { + "$ref": "#/definitions/stdioConfig", + "description": "Launch configuration. Required for 'stdio' connection type." + }, + "auth_type": { + "type": "string", + "description": "Authentication mechanism the server expects.", + "enum": ["none", "headers", "oauth", "per_user_oauth", "per_user_headers"], + "default": "none" + }, + "required_header_keys": { + "type": "array", + "description": "Header names the user must supply when auth_type is 'headers' or 'per_user_headers'. Only key names — never include secret values.", + "items": { "type": "string" }, + "uniqueItems": true + }, + "icon_url": { + "type": "string", + "description": "URL or path to a square icon (PNG/SVG recommended)." + }, + "docs_url": { + "type": "string", + "description": "URL to the server's documentation or homepage.", + "minLength": 1 + }, + "publisher": { + "type": "string", + "description": "Person or organization that maintains the server.", + "maxLength": 255 + }, + "tags": { + "type": "array", + "description": "Freeform tags used for search and filtering.", + "items": { "type": "string" }, + "uniqueItems": true + }, + "metadata": { + "type": "object", + "description": "Optional arbitrary key/value metadata. Use sparingly.", + "additionalProperties": true + } + }, + "allOf": [ + { + "if": { + "properties": { "connection_type": { "enum": ["http", "sse"] } } + }, + "then": { + "required": ["connection_url"] + } + }, + { + "if": { + "properties": { "connection_type": { "const": "stdio" } } + }, + "then": { + "required": ["stdio_config"] + } + } + ] + }, + "stdioConfig": { + "type": "object", + "required": ["command"], + "additionalProperties": false, + "properties": { + "command": { + "type": "string", + "description": "Executable to launch (e.g. 'npx', 'uvx', 'node').", + "minLength": 1 + }, + "args": { + "type": "array", + "description": "Arguments passed to the command.", + "items": { "type": "string" } + }, + "envs": { + "type": "array", + "description": "Names of environment variables the user must provide. Only names — never include secret values.", + "items": { "type": "string" }, + "uniqueItems": true + } + } + } + } +} diff --git a/community/mcp-library/servers.json b/community/mcp-library/servers.json new file mode 100644 index 0000000000..ab87c55bab --- /dev/null +++ b/community/mcp-library/servers.json @@ -0,0 +1,2281 @@ +{ + "$schema": "./schema.json", + "lastUpdatedAt": "2026-06-09T00:00:00Z", + "servers": [ + { + "name": "Filesystem", + "description": "Read and write files on the local filesystem within configured directories.", + "category": "Developer Tools", + "connection_type": "stdio", + "stdio_config": { + "command": "npx", + "args": [ + "-y", + "@modelcontextprotocol/server-filesystem", + "/path/to/allowed/dir" + ], + "envs": [] + }, + "auth_type": "none", + "docs_url": "https://github.com/modelcontextprotocol/servers/tree/main/src/filesystem", + "publisher": "Anthropic", + "tags": [ + "files", + "local", + "storage" + ], + "icon_url": "/images/mcp-servers/filesystem.png" + }, + { + "name": "GitHub (Remote)", + "description": "Connect AI tools directly to GitHub's platform to read repositories and code, manage issues and pull requests, analyze code, monitor GitHub Actions, and automate workflows. Hosted remotely by GitHub.", + "category": "Developer Tools", + "connection_type": "http", + "connection_url": "https://api.githubcopilot.com/mcp/", + "auth_type": "oauth", + "docs_url": "https://github.com/github/github-mcp-server", + "publisher": "GitHub", + "tags": [ + "github", + "git", + "repositories", + "issues", + "pull-requests", + "ci-cd", + "code" + ], + "icon_url": "/images/mcp-servers/github_light.svg" + }, + { + "name": "GitHub (Local)", + "description": "Run the GitHub MCP Server locally via Docker to read repositories and code, manage issues and pull requests, analyze code, monitor GitHub Actions, and automate workflows. Requires a GitHub Personal Access Token.", + "category": "Developer Tools", + "connection_type": "stdio", + "stdio_config": { + "command": "docker", + "args": [ + "run", + "-i", + "--rm", + "-e", + "GITHUB_PERSONAL_ACCESS_TOKEN", + "ghcr.io/github/github-mcp-server" + ], + "envs": [ + "GITHUB_PERSONAL_ACCESS_TOKEN" + ] + }, + "auth_type": "none", + "docs_url": "https://github.com/github/github-mcp-server", + "publisher": "GitHub", + "tags": [ + "github", + "git", + "repositories", + "issues", + "pull-requests", + "ci-cd", + "code", + "docker" + ], + "icon_url": "/images/mcp-servers/github_light.svg" + }, + { + "name": "Linear", + "description": "Connect AI tools to Linear to find, create, and update issues, projects, and comments. Hosted remotely by Linear.", + "category": "Project Management", + "connection_type": "http", + "connection_url": "https://mcp.linear.app/mcp", + "auth_type": "oauth", + "docs_url": "https://linear.app/docs/mcp", + "publisher": "Linear", + "tags": [ + "linear", + "issues", + "projects", + "comments", + "project-management" + ], + "icon_url": "/images/mcp-servers/linear.svg" + }, + { + "name": "Notion", + "description": "Connect AI tools to your Notion workspace to search, read, create, and update pages and databases. Hosted remotely by Notion.", + "category": "Productivity", + "connection_type": "http", + "connection_url": "https://mcp.notion.com/mcp", + "auth_type": "oauth", + "docs_url": "https://developers.notion.com/docs/mcp", + "publisher": "Notion", + "tags": [ + "notion", + "docs", + "wiki", + "knowledge-base", + "productivity" + ], + "icon_url": "/images/mcp-servers/notion.svg" + }, + { + "name": "Atlassian", + "description": "Connect AI tools to your Atlassian Cloud site to interact with Jira, Confluence, and Compass data in real time. Hosted remotely by Atlassian.", + "category": "Project Management", + "connection_type": "http", + "connection_url": "https://mcp.atlassian.com/v1/mcp/authv2", + "auth_type": "oauth", + "docs_url": "https://support.atlassian.com/atlassian-rovo-mcp-server/docs/getting-started-with-the-atlassian-remote-mcp-server/", + "publisher": "Atlassian", + "tags": [ + "atlassian", + "jira", + "confluence", + "compass", + "project-management" + ], + "icon_url": "/images/mcp-servers/atlassian.svg" + }, + { + "name": "Maxim", + "description": "Run the Maxim AI MCP server locally to manage log repositories, datasets, and evaluators, and analyze logs of sessions and traces. Requires a Maxim API key.", + "category": "AI Tools", + "connection_type": "stdio", + "stdio_config": { + "command": "npx", + "args": [ + "-y", + "@maximai/mcp-server@latest" + ], + "envs": [ + "MAXIM_API_KEY" + ] + }, + "auth_type": "none", + "docs_url": "https://www.npmjs.com/package/@maximai/mcp-server", + "publisher": "Maxim AI", + "tags": [ + "maxim", + "observability", + "evaluation", + "datasets", + "logs", + "ai" + ], + "icon_url": "/images/mcp-servers/maxim.png" + }, + { + "name": "Ahrefs", + "description": "Connect AI tools to the Ahrefs API to enrich answers with SEO metrics, backlink data, keyword research, and site analysis. Hosted remotely by Ahrefs.", + "category": "Marketing", + "connection_type": "http", + "connection_url": "https://api.ahrefs.com/mcp/mcp", + "auth_type": "oauth", + "docs_url": "https://docs.ahrefs.com/en/mcp/docs/introduction", + "publisher": "Ahrefs", + "tags": [ + "ahrefs", + "seo", + "backlinks", + "keywords", + "marketing" + ], + "icon_url": "/images/mcp-servers/ahrefs.svg" + }, + { + "name": "Calendly", + "description": "Connect AI tools to Calendly to list event types, find available slots, book and cancel events, manage availability, and generate scheduling links. Hosted remotely by Calendly.", + "category": "Productivity", + "connection_type": "http", + "connection_url": "https://mcp.calendly.com", + "auth_type": "oauth", + "docs_url": "https://developer.calendly.com/calendly-mcp-server", + "publisher": "Calendly", + "tags": [ + "calendly", + "scheduling", + "calendar", + "meetings", + "productivity" + ], + "icon_url": "/images/mcp-servers/calendly.svg" + }, + { + "name": "Stripe", + "description": "Connect AI tools to the Stripe API to manage customers, invoices, products, prices, subscriptions, payment links, refunds, and disputes, and to search Stripe documentation. Hosted remotely by Stripe.", + "category": "Finance", + "connection_type": "http", + "connection_url": "https://mcp.stripe.com", + "auth_type": "oauth", + "docs_url": "https://docs.stripe.com/mcp", + "publisher": "Stripe", + "tags": [ + "stripe", + "payments", + "billing", + "invoices", + "subscriptions", + "finance" + ], + "icon_url": "/images/mcp-servers/stripe.svg" + }, + { + "name": "Vercel", + "description": "Connect AI tools to Vercel to search documentation, manage projects and deployments, and analyze deployment logs. Hosted remotely by Vercel.", + "category": "Developer Tools", + "connection_type": "http", + "connection_url": "https://mcp.vercel.com", + "auth_type": "oauth", + "docs_url": "https://vercel.com/docs/agent-resources/vercel-mcp", + "publisher": "Vercel", + "tags": [ + "vercel", + "deployments", + "projects", + "logs", + "hosting", + "developer-tools" + ], + "icon_url": "/images/mcp-servers/vercel.svg" + }, + { + "name": "Netlify", + "description": "Run the Netlify MCP server locally to create, manage, and deploy projects, manage access controls and extensions, handle form submissions, and set environment variables via the Netlify API and CLI. Requires Node.js 22+ and a Netlify account.", + "category": "Developer Tools", + "connection_type": "stdio", + "stdio_config": { + "command": "npx", + "args": [ + "-y", + "@netlify/mcp" + ], + "envs": [ + "NETLIFY_PERSONAL_ACCESS_TOKEN" + ] + }, + "auth_type": "none", + "docs_url": "https://docs.netlify.com/welcome/build-with-ai/netlify-mcp-server/", + "publisher": "Netlify", + "tags": [ + "netlify", + "deployments", + "projects", + "hosting", + "cli", + "developer-tools" + ], + "icon_url": "/images/mcp-servers/netlify.svg" + }, + { + "name": "Vanta", + "description": "Connect AI tools to Vanta to retrieve compliance test results, manage security findings, review framework requirements, track vulnerabilities, and access controls, documents, integrations, and risk scenarios. Hosted remotely by Vanta.", + "category": "Security", + "connection_type": "http", + "connection_url": "https://mcp.vanta.com/mcp", + "auth_type": "oauth", + "docs_url": "https://help.vanta.com/en/articles/14094979-connecting-to-vanta-mcp", + "publisher": "Vanta", + "tags": [ + "vanta", + "compliance", + "security", + "soc2", + "iso27001", + "vulnerabilities" + ], + "icon_url": "/images/mcp-servers/vanta.png" + }, + { + "name": "Airtable", + "description": "Connect AI tools to your Airtable bases to search, analyze, create, and update records, manage tables and fields, and explore schemas and interfaces. Hosted remotely by Airtable.", + "category": "Productivity", + "connection_type": "http", + "connection_url": "https://mcp.airtable.com/mcp", + "auth_type": "oauth", + "docs_url": "https://support.airtable.com/docs/using-the-airtable-mcp-server", + "publisher": "Airtable", + "icon_url": "/images/mcp-servers/airtable.svg", + "tags": [ + "airtable", + "database", + "records", + "spreadsheet", + "productivity" + ] + }, + { + "name": "Cal.com", + "description": "Connect AI tools to Cal.com to manage bookings, event types, schedules, availability, and organization memberships through natural language. Hosted remotely by Cal.com.", + "category": "Productivity", + "connection_type": "http", + "connection_url": "https://mcp.cal.com/mcp", + "auth_type": "oauth", + "docs_url": "https://cal.com/docs/platform/mcp-server", + "publisher": "Cal.com", + "tags": [ + "cal.com", + "scheduling", + "calendar", + "bookings", + "meetings", + "productivity" + ], + "icon_url": "/images/mcp-servers/cal.svg" + }, + { + "name": "Asana", + "description": "Connect AI tools to Asana to access tasks, projects, and workspaces, track status and ownership, and work across workspace activity. Hosted remotely by Asana.", + "category": "Project Management", + "connection_type": "http", + "connection_url": "https://mcp.asana.com/v2/mcp", + "auth_type": "oauth", + "docs_url": "https://developers.asana.com/docs/using-asanas-model-control-protocol-mcp-server", + "publisher": "Asana", + "tags": [ + "asana", + "tasks", + "projects", + "workspaces", + "project-management" + ], + "icon_url": "/images/mcp-servers/asana-logo.svg" + }, + { + "name": "Canva", + "description": "Connect AI tools to Canva to find and discuss designs, assets, exports, and comments, and iterate on creative work conversationally. Hosted remotely by Canva.", + "category": "Design", + "connection_type": "http", + "connection_url": "https://mcp.canva.com/mcp", + "auth_type": "oauth", + "docs_url": "https://www.canva.dev/docs/connect/canva-mcp-server-setup/", + "publisher": "Canva", + "tags": [ + "canva", + "design", + "assets", + "exports", + "comments" + ], + "icon_url": "/images/mcp-servers/canva.svg" + }, + { + "name": "Cloudflare", + "description": "Connect AI tools to Cloudflare to inspect resources, plan infrastructure changes, and work across Workers, D1, R2, DNS, and account APIs. Hosted remotely by Cloudflare.", + "category": "Developer Tools", + "connection_type": "http", + "connection_url": "https://mcp.cloudflare.com/mcp", + "auth_type": "oauth", + "docs_url": "https://developers.cloudflare.com/agents/model-context-protocol/", + "publisher": "Cloudflare", + "tags": [ + "cloudflare", + "workers", + "dns", + "d1", + "r2", + "developer-tools" + ], + "icon_url": "/images/mcp-servers/cloudflare.svg" + }, + { + "name": "Figma", + "description": "Connect AI tools to Figma to bring files, designs, and Dev Mode context into workflows so agents can understand and act on visual work. Hosted remotely by Figma.", + "category": "Design", + "connection_type": "http", + "connection_url": "https://mcp.figma.com/mcp", + "auth_type": "oauth", + "docs_url": "https://developers.figma.com/docs/figma-mcp-server/", + "publisher": "Figma", + "tags": [ + "figma", + "design", + "dev-mode", + "files", + "prototyping" + ], + "icon_url": "/images/mcp-servers/figma.svg" + }, + { + "name": "HubSpot", + "description": "Connect AI tools to HubSpot to read and act on CRM objects, records, and account data, and understand customer workflows. Hosted remotely by HubSpot.", + "category": "Sales", + "connection_type": "http", + "connection_url": "https://mcp.hubspot.com", + "auth_type": "oauth", + "docs_url": "https://developers.hubspot.com/mcp", + "publisher": "HubSpot", + "tags": [ + "hubspot", + "crm", + "sales", + "marketing", + "contacts" + ], + "icon_url": "/images/mcp-servers/hubspot.png" + }, + { + "name": "Neon", + "description": "Connect AI tools to Neon to manage Postgres projects and branches, run SQL, and search documentation. Hosted remotely by Neon.", + "category": "Developer Tools", + "connection_type": "http", + "connection_url": "https://mcp.neon.tech/mcp", + "auth_type": "oauth", + "docs_url": "https://neon.com/docs/ai/neon-mcp-server", + "publisher": "Neon", + "tags": [ + "neon", + "postgres", + "database", + "sql", + "developer-tools" + ], + "icon_url": "/images/mcp-servers/neon.svg" + }, + { + "name": "Railway", + "description": "Connect AI tools to Railway to access project and deployment context, investigate app state, plan changes, and work across environments. Hosted remotely by Railway.", + "category": "Developer Tools", + "connection_type": "http", + "connection_url": "https://mcp.railway.com", + "auth_type": "oauth", + "docs_url": "https://docs.railway.com/reference/mcp-server", + "publisher": "Railway", + "tags": [ + "railway", + "deployments", + "projects", + "environments", + "developer-tools" + ], + "icon_url": "/images/mcp-servers/railway.svg" + }, + { + "name": "Parallel", + "description": "Connect AI tools to Parallel for real-time web search and content extraction in search-grounded AI workflows. Hosted remotely by Parallel.", + "category": "Search", + "connection_type": "http", + "connection_url": "https://search.parallel.ai/mcp", + "auth_type": "none", + "docs_url": "https://docs.parallel.ai/integrations/mcp/quickstart", + "publisher": "Parallel", + "tags": [ + "parallel", + "search", + "web", + "content-extraction", + "research" + ], + "icon_url": "/images/mcp-servers/parallel.png" + }, + { + "name": "Sentry", + "description": "Connect AI tools to Sentry to retrieve, analyze, and debug application issues and errors, and bring project, issue, and reliability context into incident workflows. Hosted remotely by Sentry.", + "category": "Developer Tools", + "connection_type": "http", + "connection_url": "https://mcp.sentry.dev/mcp", + "auth_type": "oauth", + "docs_url": "https://docs.sentry.io/product/sentry-mcp/", + "publisher": "Sentry", + "tags": [ + "sentry", + "errors", + "debugging", + "observability", + "developer-tools" + ], + "icon_url": "/images/mcp-servers/sentry.svg" + }, + { + "name": "Slack", + "description": "Connect AI tools to Slack to find decisions, summarize discussions, and understand channel activity across messages, channels, users, and canvases. Hosted remotely by Slack.", + "category": "Communication", + "connection_type": "http", + "connection_url": "https://mcp.slack.com/mcp", + "auth_type": "oauth", + "docs_url": "https://slack.com/intl/en-in/help/articles/48855576908307-Guide-to-the-Slack-MCP-server", + "publisher": "Slack", + "tags": [ + "slack", + "messages", + "channels", + "communication", + "collaboration" + ], + "icon_url": "/images/mcp-servers/slack.svg" + }, + { + "name": "Supabase", + "description": "Connect AI tools to Supabase to work with project, database, and documentation context across Postgres, auth, storage, and edge functions. Hosted remotely by Supabase.", + "category": "Developer Tools", + "connection_type": "http", + "connection_url": "https://mcp.supabase.com/mcp", + "auth_type": "oauth", + "docs_url": "https://supabase.com/docs/guides/getting-started/mcp", + "publisher": "Supabase", + "tags": [ + "supabase", + "postgres", + "database", + "auth", + "developer-tools" + ], + "icon_url": "/images/mcp-servers/supabase.svg" + }, + { + "name": "GitLab", + "description": "Connect AI tools securely to your GitLab data to work with projects, issues, merge requests, and CI/CD. Hosted remotely by GitLab.", + "category": "Developer Tools", + "connection_type": "http", + "connection_url": "https://gitlab.com/api/v4/mcp", + "auth_type": "oauth", + "docs_url": "https://docs.gitlab.com/user/gitlab_duo/model_context_protocol/mcp_server/", + "publisher": "GitLab", + "tags": [ + "gitlab", + "git", + "repositories", + "merge-requests", + "ci-cd", + "developer-tools" + ], + "icon_url": "/images/mcp-servers/gitlab.svg" + }, + { + "name": "Todoist", + "description": "Connect AI tools to Todoist to search, create, complete, and manage your tasks and projects. Hosted remotely by Todoist.", + "category": "Productivity", + "connection_type": "http", + "connection_url": "https://ai.todoist.net/mcp", + "auth_type": "oauth", + "docs_url": "https://www.todoist.com/help/articles/connect-todoist-to-ai-tools-with-mcp", + "publisher": "Todoist", + "tags": [ + "todoist", + "tasks", + "projects", + "to-do", + "productivity" + ], + "icon_url": "/images/mcp-servers/todoist.svg" + }, + { + "name": "PayPal", + "description": "Connect AI tools to the PayPal platform to access payments, invoicing, subscriptions, and transaction data. Hosted remotely by PayPal.", + "category": "Finance", + "connection_type": "http", + "connection_url": "https://mcp.paypal.com/mcp", + "auth_type": "oauth", + "docs_url": "https://www.paypal.ai/docs/tools/mcp-quickstart", + "publisher": "PayPal", + "tags": [ + "paypal", + "payments", + "invoicing", + "subscriptions", + "finance" + ], + "icon_url": "/images/mcp-servers/paypal.svg" + }, + { + "name": "Monday", + "description": "Connect AI tools to monday.com to manage projects, boards, items, and workflows. Hosted remotely by monday.com.", + "category": "Project Management", + "connection_type": "http", + "connection_url": "https://mcp.monday.com/mcp", + "auth_type": "oauth", + "docs_url": "https://developer.monday.com/apps/docs/mcp", + "publisher": "monday.com", + "tags": [ + "monday", + "boards", + "projects", + "workflows", + "project-management" + ], + "icon_url": "/images/mcp-servers/monday.png" + }, + { + "name": "Honeycomb", + "description": "Connect AI tools to Honeycomb to query and explore observability data, traces, and SLOs. Hosted remotely by Honeycomb.", + "category": "Developer Tools", + "connection_type": "http", + "connection_url": "https://mcp.honeycomb.io/mcp", + "auth_type": "oauth", + "docs_url": "https://docs.honeycomb.io/integrations/mcp/", + "publisher": "Honeycomb", + "tags": [ + "honeycomb", + "observability", + "tracing", + "slo", + "developer-tools" + ], + "icon_url": "/images/mcp-servers/honeycomb.png" + }, + { + "name": "Square", + "description": "Connect AI tools to Square to search and manage transaction, merchant, and payment data. Hosted remotely by Square.", + "category": "Finance", + "connection_type": "sse", + "connection_url": "https://mcp.squareup.com/sse", + "auth_type": "oauth", + "docs_url": "https://developer.squareup.com/docs/mcp", + "publisher": "Square", + "tags": [ + "square", + "payments", + "merchants", + "transactions", + "finance" + ], + "icon_url": "/images/mcp-servers/square.png" + }, + { + "name": "ClickUp", + "description": "Connect AI tools to ClickUp to manage tasks, docs, and projects with real-time workspace context. Hosted remotely by ClickUp.", + "category": "Project Management", + "connection_type": "http", + "connection_url": "https://mcp.clickup.com/mcp", + "auth_type": "oauth", + "docs_url": "https://clickup.com/blog/clickup-mcp-server/", + "publisher": "ClickUp", + "tags": [ + "clickup", + "tasks", + "docs", + "projects", + "project-management" + ], + "icon_url": "/images/mcp-servers/clickup.svg" + }, + { + "name": "Intercom", + "description": "Connect AI tools to Intercom to access conversations, contacts, and support data for better customer insights. Hosted remotely by Intercom.", + "category": "Customer Support", + "connection_type": "http", + "connection_url": "https://mcp.intercom.com/mcp", + "auth_type": "oauth", + "docs_url": "https://developers.intercom.com/docs/guides/mcp", + "publisher": "Intercom", + "tags": [ + "intercom", + "support", + "conversations", + "contacts", + "customer-support" + ], + "icon_url": "/images/mcp-servers/intercom.png" + }, + { + "name": "Miro", + "description": "Connect AI tools to Miro to access and create content on boards for visual collaboration. Hosted remotely by Miro.", + "category": "Design", + "connection_type": "http", + "connection_url": "https://mcp.miro.com/", + "auth_type": "oauth", + "docs_url": "https://developers.miro.com/docs/miro-mcp", + "publisher": "Miro", + "tags": [ + "miro", + "whiteboard", + "boards", + "collaboration", + "design" + ], + "icon_url": "/images/mcp-servers/miro.png" + }, + { + "name": "Mixpanel", + "description": "Connect AI tools to Mixpanel to analyze, query, and manage your product analytics data. Hosted remotely by Mixpanel.", + "category": "Analytics", + "connection_type": "http", + "connection_url": "https://mcp.mixpanel.com/mcp", + "auth_type": "oauth", + "docs_url": "https://docs.mixpanel.com/docs/mcp", + "publisher": "Mixpanel", + "tags": [ + "mixpanel", + "analytics", + "product-analytics", + "events", + "data" + ], + "icon_url": "/images/mcp-servers/mixpanel.png" + }, + { + "name": "Cloudinary", + "description": "Connect AI tools to Cloudinary to manage, transform, and deliver images and videos across your media assets. Hosted remotely by Cloudinary.", + "category": "Developer Tools", + "connection_type": "sse", + "connection_url": "https://asset-management.mcp.cloudinary.com/sse", + "auth_type": "oauth", + "docs_url": "https://cloudinary.com/documentation/cloudinary_llm_mcp", + "publisher": "Cloudinary", + "tags": [ + "cloudinary", + "media", + "images", + "video", + "developer-tools" + ], + "icon_url": "/images/mcp-servers/cloudinary.svg" + }, + { + "name": "PlanetScale", + "description": "Connect AI tools to PlanetScale for authenticated access to your Postgres and MySQL databases. Hosted remotely by PlanetScale.", + "category": "Developer Tools", + "connection_type": "http", + "connection_url": "https://mcp.pscale.dev/mcp/planetscale", + "auth_type": "oauth", + "docs_url": "https://planetscale.com/docs/concepts/planetscale-mcp-server", + "publisher": "PlanetScale", + "tags": [ + "planetscale", + "mysql", + "postgres", + "database", + "developer-tools" + ], + "icon_url": "/images/mcp-servers/planetscale.svg" + }, + { + "name": "Box", + "description": "Connect AI tools to Box for governed access to file and folder context, search, and Box AI within your existing permissions. Hosted remotely by Box.", + "category": "Productivity", + "connection_type": "http", + "connection_url": "https://mcp.box.com", + "auth_type": "oauth", + "docs_url": "https://developer.box.com/guides/box-mcp/remote/", + "publisher": "Box", + "tags": [ + "box", + "files", + "storage", + "documents", + "productivity" + ], + "icon_url": "/images/mcp-servers/box.png" + }, + { + "name": "Sanity", + "description": "Connect AI tools to Sanity to create, query, and manage structured content. Hosted remotely by Sanity.", + "category": "Developer Tools", + "connection_type": "http", + "connection_url": "https://mcp.sanity.io", + "auth_type": "oauth", + "docs_url": "https://www.sanity.io/docs/cli-reference/cli-mcp", + "publisher": "Sanity", + "tags": [ + "sanity", + "cms", + "content", + "structured-content", + "developer-tools" + ], + "icon_url": "/images/mcp-servers/sanity-light.svg" + }, + { + "name": "Postman", + "description": "Connect AI tools to Postman to give API context to your coding agents across collections, requests, and workspaces. Hosted remotely by Postman.", + "category": "Developer Tools", + "connection_type": "http", + "connection_url": "https://mcp.postman.com/minimal", + "auth_type": "oauth", + "docs_url": "https://learning.postman.com/docs/reference/postman-api/postman-mcp-server/overview/", + "publisher": "Postman", + "tags": [ + "postman", + "api", + "collections", + "testing", + "developer-tools" + ], + "icon_url": "/images/mcp-servers/postman.svg" + }, + { + "name": "Mintlify", + "description": "Connect AI tools to Mintlify to search, read, and edit your documentation. Hosted remotely by Mintlify.", + "category": "Developer Tools", + "connection_type": "http", + "connection_url": "https://mcp.mintlify.com", + "auth_type": "oauth", + "docs_url": "https://mintlify.com/docs/ai/model-context-protocol", + "publisher": "Mintlify", + "tags": [ + "mintlify", + "documentation", + "docs", + "search", + "developer-tools" + ], + "icon_url": "/images/mcp-servers/mintlify.svg" + }, + { + "name": "Hugging Face", + "description": "Connect AI tools to the Hugging Face Hub to access models, datasets, Spaces, and thousands of Gradio apps. Hosted remotely by Hugging Face.", + "category": "AI Tools", + "connection_type": "http", + "connection_url": "https://huggingface.co/mcp", + "auth_type": "oauth", + "docs_url": "https://huggingface.co/settings/mcp", + "publisher": "Hugging Face", + "tags": [ + "hugging-face", + "models", + "datasets", + "spaces", + "ai" + ], + "icon_url": "/images/mcp-servers/hugging_face.svg" + }, + { + "name": "Microsoft Learn", + "description": "Connect AI tools to Microsoft Learn to search trusted Microsoft documentation and power your development. Hosted remotely by Microsoft.", + "category": "Developer Tools", + "connection_type": "http", + "connection_url": "https://learn.microsoft.com/api/mcp", + "auth_type": "none", + "docs_url": "https://learn.microsoft.com/en-us/training/support/mcp", + "publisher": "Microsoft", + "tags": [ + "microsoft", + "documentation", + "docs", + "search", + "developer-tools" + ], + "icon_url": "/images/mcp-servers/microsoft.svg" + }, + { + "name": "Context7", + "description": "Connect AI tools to Context7 to fetch up-to-date, version-specific documentation and code examples for libraries and frameworks. Hosted remotely by Context7.", + "category": "Developer Tools", + "connection_type": "http", + "connection_url": "https://mcp.context7.com/mcp", + "auth_type": "none", + "docs_url": "https://context7.com/", + "publisher": "Context7", + "tags": [ + "context7", + "documentation", + "libraries", + "code-examples", + "developer-tools" + ], + "icon_url": "/images/mcp-servers/context7.svg" + }, + { + "name": "Exa", + "description": "Connect AI tools to Exa for web search and code documentation search to ground answers in real-time results. Hosted remotely by Exa.", + "category": "Search", + "connection_type": "http", + "connection_url": "https://mcp.exa.ai/mcp", + "auth_type": "oauth", + "docs_url": "https://docs.exa.ai/reference/exa-mcp", + "publisher": "Exa", + "tags": [ + "exa", + "search", + "web", + "research", + "documentation" + ], + "icon_url": "/images/mcp-servers/exa.png" + }, + { + "name": "Tavily", + "description": "Connect AI tools to Tavily to search the web and extract content for agentic, search-grounded workflows. Hosted remotely by Tavily.", + "category": "Search", + "connection_type": "http", + "connection_url": "https://mcp.tavily.com/mcp", + "auth_type": "oauth", + "docs_url": "https://docs.tavily.com/documentation/mcp", + "publisher": "Tavily", + "tags": [ + "tavily", + "search", + "web", + "research", + "content-extraction" + ], + "icon_url": "/images/mcp-servers/tavily.png" + }, + { + "name": "Semrush", + "description": "Connect AI tools to Semrush to access SEO, market data, and brand visibility insights. Hosted remotely by Semrush.", + "category": "Marketing", + "connection_type": "http", + "connection_url": "https://mcp.semrush.com/claude/v1/mcp", + "auth_type": "oauth", + "docs_url": "https://www.semrush.com/apps/", + "publisher": "Semrush", + "tags": [ + "semrush", + "seo", + "marketing", + "keywords", + "analytics" + ], + "icon_url": "/images/mcp-servers/semrush.png" + }, + { + "name": "Clerk", + "description": "Connect AI tools to Clerk to manage authentication, organizations, users, and billing. Hosted remotely by Clerk.", + "category": "Developer Tools", + "connection_type": "http", + "connection_url": "https://mcp.clerk.com/mcp", + "auth_type": "none", + "docs_url": "https://clerk.com/docs/guides/ai/mcp/clerk-mcp-server", + "publisher": "Clerk", + "tags": [ + "clerk", + "authentication", + "users", + "billing", + "developer-tools" + ], + "icon_url": "/images/mcp-servers/clerk-icon-light.svg" + }, + { + "name": "Amplitude", + "description": "Connect AI tools to Amplitude to give teams powerful behavioral and product analytics insights. Hosted remotely by Amplitude.", + "category": "Analytics", + "connection_type": "http", + "connection_url": "https://mcp.amplitude.com/mcp", + "auth_type": "oauth", + "docs_url": "https://amplitude.com/docs/get-started/mcp", + "publisher": "Amplitude", + "tags": [ + "amplitude", + "analytics", + "product-analytics", + "behavioral", + "data" + ], + "icon_url": "/images/mcp-servers/amplitude.svg" + }, + { + "name": "Hex", + "description": "Connect AI tools to Hex to answer questions with the Hex agent across your data and notebooks. Hosted remotely by Hex.", + "category": "Analytics", + "connection_type": "http", + "connection_url": "https://app.hex.tech/mcp", + "auth_type": "oauth", + "docs_url": "https://learn.hex.tech/docs/api-integrations/mcp-server", + "publisher": "Hex", + "tags": [ + "hex", + "analytics", + "data-science", + "notebooks", + "data" + ], + "icon_url": "/images/mcp-servers/hex.png" + }, + { + "name": "WordPress.com", + "description": "Connect AI tools to WordPress.com for secure, permission-scoped access to manage your sites and content. Hosted remotely by WordPress.com.", + "category": "Developer Tools", + "connection_type": "http", + "connection_url": "https://public-api.wordpress.com/wpcom/v2/mcp/v1", + "auth_type": "oauth", + "docs_url": "https://developer.wordpress.com/docs/mcp/", + "publisher": "Automattic", + "tags": [ + "wordpress", + "cms", + "websites", + "content", + "developer-tools" + ], + "icon_url": "/images/mcp-servers/wordpress.svg" + }, + { + "name": "Jotform", + "description": "Connect AI tools to Jotform to create forms and analyze submissions. Hosted remotely by Jotform.", + "category": "Productivity", + "connection_type": "http", + "connection_url": "https://mcp.jotform.com/mcp-app", + "auth_type": "oauth", + "docs_url": "https://www.jotform.com/developers/mcp/", + "publisher": "Jotform", + "tags": [ + "jotform", + "forms", + "submissions", + "surveys", + "productivity" + ], + "icon_url": "/images/mcp-servers/jotform.png" + }, + { + "name": "Eraser", + "description": "Connect AI tools to Eraser to generate, manage, and update diagrams and files. Hosted remotely by Eraser.", + "category": "Design", + "connection_type": "http", + "connection_url": "https://app.eraser.io/api/mcp", + "auth_type": "oauth", + "docs_url": "https://docs.eraser.io/docs/mcp", + "publisher": "Eraser", + "tags": [ + "eraser", + "diagrams", + "documentation", + "architecture", + "design" + ], + "icon_url": "/images/mcp-servers/eraser.png" + }, + { + "name": "GoCardless", + "description": "Connect AI tools to GoCardless to build and manage bank payment and direct debit integrations. Hosted remotely by GoCardless.", + "category": "Finance", + "connection_type": "http", + "connection_url": "https://mcp.gocardless.com", + "auth_type": "oauth", + "docs_url": "https://developer.gocardless.com/developer-tools/mcp", + "publisher": "GoCardless", + "tags": [ + "gocardless", + "payments", + "direct-debit", + "billing", + "finance" + ], + "icon_url": "/images/mcp-servers/gocardless.png" + }, + { + "name": "Twilio", + "description": "Connect AI tools to Twilio to search and explore Twilio documentation for building communications and customer engagement. Hosted remotely by Twilio.", + "category": "Developer Tools", + "connection_type": "http", + "connection_url": "https://mcp.twilio.com/docs", + "auth_type": "none", + "docs_url": "https://www.twilio.com/docs/alpha/mcp-server", + "publisher": "Twilio", + "tags": [ + "twilio", + "documentation", + "communications", + "sms", + "developer-tools" + ], + "icon_url": "/images/mcp-servers/twilio.svg" + }, + { + "name": "Quartr", + "description": "Connect AI tools to Quartr for financial data and AI infrastructure for company research. Hosted remotely by Quartr.", + "category": "Finance", + "connection_type": "http", + "connection_url": "https://mcp.quartr.com/mcp", + "auth_type": "oauth", + "docs_url": "https://quartr.com/mcp", + "publisher": "Quartr", + "tags": [ + "quartr", + "financial-data", + "research", + "earnings", + "finance" + ], + "icon_url": "/images/mcp-servers/quartr.png" + }, + { + "name": "Bigdata.com", + "description": "Connect AI tools to Bigdata.com for institutional-grade financial data covering global news, transcripts, and regulatory filings. Hosted remotely by Bigdata.com.", + "category": "Finance", + "connection_type": "http", + "connection_url": "https://mcp.bigdata.com/", + "auth_type": "headers", + "required_header_keys": [ + "x-api-key" + ], + "docs_url": "https://docs.bigdata.com/mcp-reference/introduction", + "publisher": "Bigdata.com", + "tags": [ + "bigdata", + "financial-data", + "news", + "filings", + "finance" + ], + "icon_url": "/images/mcp-servers/bigdata.png" + }, + { + "name": "Fiscal.ai", + "description": "Connect AI tools to Fiscal.ai for clean public-equity fundamental data and structured company and market context. Hosted remotely by Fiscal.ai.", + "category": "Finance", + "connection_type": "http", + "connection_url": "https://api.fiscal.ai/mcp", + "auth_type": "oauth", + "docs_url": "https://docs.fiscal.ai/docs/guides/mcp-integration", + "publisher": "Fiscal.ai", + "tags": [ + "fiscal-ai", + "financial-data", + "equities", + "fundamentals", + "finance" + ], + "icon_url": "/images/mcp-servers/fiscal.png" + }, + { + "name": "Ramp", + "description": "Connect AI tools to Ramp to search, access, and analyze your Ramp financial data and spending patterns. Hosted remotely by Ramp.", + "category": "Finance", + "connection_type": "http", + "connection_url": "https://ramp-mcp-remote.ramp.com/mcp", + "auth_type": "oauth", + "docs_url": "https://docs.ramp.com/developer-api/v1/mcp", + "publisher": "Ramp", + "tags": [ + "ramp", + "spend", + "expenses", + "finance", + "procurement" + ], + "icon_url": "/images/mcp-servers/ramp.png" + }, + { + "name": "Ramp Data", + "description": "Connect AI tools to Ramp Data for spend analysis and benchmarking across a large network of business transaction data. Hosted remotely by Ramp.", + "category": "Finance", + "connection_type": "http", + "connection_url": "https://mcp.ramp.com/ramp-data/anthropic/mcp", + "auth_type": "oauth", + "docs_url": "https://docs.ramp.com/developer-api/v1/mcp", + "publisher": "Ramp", + "tags": [ + "ramp", + "spend", + "benchmarking", + "finance", + "data" + ], + "icon_url": "/images/mcp-servers/ramp.png" + }, + { + "name": "CoinDesk", + "description": "Connect AI tools to CoinDesk for real-time and historical digital asset market data and indices. Hosted remotely by CoinDesk.", + "category": "Finance", + "connection_type": "http", + "connection_url": "https://mcp.coindesk.com/mcp", + "auth_type": "oauth", + "docs_url": "https://mcp.coindesk.com/", + "publisher": "CoinDesk", + "tags": [ + "coindesk", + "crypto", + "market-data", + "indices", + "finance" + ], + "icon_url": "/images/mcp-servers/coindesk.png" + }, + { + "name": "Crypto.com", + "description": "Connect AI tools to Crypto.com for real-time prices, order books, conversions, candlestick charts, and token-market information. Hosted remotely by Crypto.com.", + "category": "Finance", + "connection_type": "http", + "connection_url": "https://mcp.crypto.com/market-data/mcp", + "auth_type": "none", + "docs_url": "https://crypto.com/", + "publisher": "Crypto.com", + "tags": [ + "crypto-com", + "crypto", + "market-data", + "trading", + "finance" + ], + "icon_url": "/images/mcp-servers/crypto.png" + }, + { + "name": "Rillet", + "description": "Connect AI tools to Rillet to query your live general ledger and financials in plain English. Hosted remotely by Rillet.", + "category": "Finance", + "connection_type": "http", + "connection_url": "https://api.rillet.com/mcp", + "auth_type": "oauth", + "docs_url": "https://www.rillet.com/", + "publisher": "Rillet", + "tags": [ + "rillet", + "accounting", + "general-ledger", + "erp", + "finance" + ], + "icon_url": "/images/mcp-servers/rillet.png" + }, + { + "name": "Carta", + "description": "Connect AI tools to Carta to work with cap table, investor, fund, accounting, and company financial data for private-capital workflows. Hosted remotely by Carta.", + "category": "Finance", + "connection_type": "http", + "connection_url": "https://mcp.app.carta.com/mcp", + "auth_type": "oauth", + "docs_url": "https://carta.com/", + "publisher": "Carta", + "tags": [ + "carta", + "cap-table", + "equity", + "fund-administration", + "finance" + ], + "icon_url": "/images/mcp-servers/carta.png" + }, + { + "name": "Privacy.com", + "description": "Connect AI tools to Privacy.com to manage virtual cards and track spending patterns. Hosted remotely by Privacy.com.", + "category": "Finance", + "connection_type": "http", + "connection_url": "https://mcp.privacy.com", + "auth_type": "oauth", + "docs_url": "https://developers.privacy.com/docs/mcp-server", + "publisher": "Privacy.com", + "tags": [ + "privacy", + "virtual-cards", + "payments", + "spend", + "finance" + ], + "icon_url": "/images/mcp-servers/privacy.png" + }, + { + "name": "Airwallex", + "description": "Connect AI tools to Airwallex to access documentation, API references, and sandbox testing context for building payment integrations. Hosted remotely by Airwallex.", + "category": "Finance", + "connection_type": "http", + "connection_url": "https://mcp-demo.airwallex.com/developer", + "auth_type": "oauth", + "docs_url": "https://www.airwallex.com/docs/developer-tools/ai/agentos/connectors", + "publisher": "Airwallex", + "tags": [ + "airwallex", + "payments", + "documentation", + "fintech", + "finance" + ], + "icon_url": "/images/mcp-servers/airwallex.svg" + }, + { + "name": "Yahoo Finance", + "description": "Connect AI tools to Yahoo Finance for stock data, market news, financials, and price history. Hosted remotely.", + "category": "Finance", + "connection_type": "http", + "connection_url": "https://gateway.mcpservers.org/yahoo-finance/mcp", + "auth_type": "none", + "docs_url": "https://finance.yahoo.com/", + "publisher": "mcpservers.org", + "tags": [ + "yahoo-finance", + "stocks", + "market-data", + "news", + "finance" + ], + "icon_url": "/images/mcp-servers/yahoo-finance.png" + }, + { + "name": "Expedia", + "description": "Connect AI tools to Expedia to plan trips and search flights and hotels. Hosted remotely by Expedia.", + "category": "Travel", + "connection_type": "http", + "connection_url": "https://www.expedia.com/mcp", + "auth_type": "oauth", + "docs_url": "https://www.expedia.com/", + "publisher": "Expedia", + "tags": [ + "expedia", + "travel", + "flights", + "hotels", + "booking" + ], + "icon_url": "/images/mcp-servers/expedia.png" + }, + { + "name": "Granola", + "description": "Connect AI tools to Granola to capture meeting notes and context with AI-powered summaries and follow-ups. Hosted remotely by Granola.", + "category": "Productivity", + "connection_type": "http", + "connection_url": "https://mcp.granola.ai/mcp", + "auth_type": "oauth", + "docs_url": "https://www.granola.ai/", + "publisher": "Granola", + "tags": [ + "granola", + "meetings", + "notes", + "transcripts", + "productivity" + ], + "icon_url": "/images/mcp-servers/granola-light.svg" + }, + { + "name": "Otter.ai", + "description": "Connect AI tools to Otter.ai to access meeting transcripts, summaries, and conversation records and extract decisions and action items. Hosted remotely by Otter.ai.", + "category": "Productivity", + "connection_type": "http", + "connection_url": "https://mcp.otter.ai/mcp", + "auth_type": "oauth", + "docs_url": "https://help.otter.ai/hc/en-us/articles/35287607569687-Otter-MCP-Server", + "publisher": "Otter.ai", + "tags": [ + "otter", + "meetings", + "transcripts", + "notes", + "productivity" + ], + "icon_url": "/images/mcp-servers/otter.png" + }, + { + "name": "Grain", + "description": "Connect AI tools to Grain to turn meetings into insights, summaries, and next steps. Hosted remotely by Grain.", + "category": "Productivity", + "connection_type": "http", + "connection_url": "https://api.grain.com/_/mcp", + "auth_type": "oauth", + "docs_url": "https://grain.com/", + "publisher": "Grain", + "tags": [ + "grain", + "meetings", + "transcripts", + "insights", + "productivity" + ], + "icon_url": "/images/mcp-servers/grain.png" + }, + { + "name": "Krisp", + "description": "Connect AI tools to Krisp to bring meeting transcripts and notes into workflows for summaries, follow-ups, and decisions. Hosted remotely by Krisp.", + "category": "Productivity", + "connection_type": "http", + "connection_url": "https://mcp.krisp.ai/mcp", + "auth_type": "oauth", + "docs_url": "https://krisp.ai/", + "publisher": "Krisp", + "tags": [ + "krisp", + "meetings", + "transcripts", + "notes", + "productivity" + ], + "icon_url": "/images/mcp-servers/krisp.png" + }, + { + "name": "Mem", + "description": "Connect AI tools to Mem, the AI notebook for notes, thoughts, and personal knowledge. Hosted remotely by Mem.", + "category": "Productivity", + "connection_type": "http", + "connection_url": "https://mcp.mem.ai/mcp", + "auth_type": "oauth", + "docs_url": "https://get.mem.ai/", + "publisher": "Mem", + "tags": [ + "mem", + "notes", + "knowledge-base", + "personal", + "productivity" + ], + "icon_url": "/images/mcp-servers/mem.png" + }, + { + "name": "Craft", + "description": "Connect AI tools to Craft to create structured documents, manage tasks, and organize a personal knowledge base. Hosted remotely by Craft.", + "category": "Productivity", + "connection_type": "http", + "connection_url": "https://mcp.craft.do/my/mcp", + "auth_type": "oauth", + "docs_url": "https://www.craft.do/imagine/guide/mcp/mcp", + "publisher": "Craft", + "tags": [ + "craft", + "notes", + "documents", + "knowledge-base", + "productivity" + ], + "icon_url": "/images/mcp-servers/craft.png" + }, + { + "name": "Workable", + "description": "Connect AI tools to Workable for hiring and HR workflows across candidates, jobs, and pipelines. Hosted remotely by Workable.", + "category": "Human Resources", + "connection_type": "http", + "connection_url": "https://mcp.workable.com/mcp", + "auth_type": "oauth", + "docs_url": "https://workable.readme.io/reference/workable-mcp-server", + "publisher": "Workable", + "tags": [ + "workable", + "hiring", + "recruiting", + "hr", + "ats" + ], + "icon_url": "/images/mcp-servers/workable.png" + }, + { + "name": "Metaview", + "description": "Connect AI tools to Metaview for interview, hiring, and recruiting context. Hosted remotely by Metaview.", + "category": "Human Resources", + "connection_type": "http", + "connection_url": "https://mcp.metaview.ai/mcp", + "auth_type": "oauth", + "docs_url": "https://www.metaview.ai/", + "publisher": "Metaview", + "tags": [ + "metaview", + "recruiting", + "interviews", + "hiring", + "hr" + ], + "icon_url": "/images/mcp-servers/metaview.png" + }, + { + "name": "Harmonic", + "description": "Connect AI tools to Harmonic to discover, research, and enrich companies and people for market mapping and prospect research. Hosted remotely by Harmonic.", + "category": "Sales", + "connection_type": "http", + "connection_url": "https://mcp.api.harmonic.ai", + "auth_type": "oauth", + "docs_url": "https://harmonic.ai/", + "publisher": "Harmonic", + "tags": [ + "harmonic", + "prospecting", + "enrichment", + "companies", + "sales" + ], + "icon_url": "/images/mcp-servers/harmonic.png" + }, + { + "name": "ZoomInfo", + "description": "Connect AI tools to ZoomInfo for verified B2B company and contact data to search companies, identify stakeholders, and build account lists. Hosted remotely by ZoomInfo.", + "category": "Sales", + "connection_type": "http", + "connection_url": "https://mcp.zoominfo.com/mcp", + "auth_type": "oauth", + "docs_url": "https://docs.zoominfo.com/docs/zi-api-mcp-overview", + "publisher": "ZoomInfo", + "tags": [ + "zoominfo", + "prospecting", + "enrichment", + "contacts", + "sales" + ], + "icon_url": "/images/mcp-servers/zoominfo.png" + }, + { + "name": "Sprouts", + "description": "Connect AI tools to Sprouts.ai for natural-language access to a B2B prospect database with role, industry, and lead-qualification signals. Hosted remotely by Sprouts.ai.", + "category": "Sales", + "connection_type": "http", + "connection_url": "https://sprouts-mcp-server.kartikay-dhar.workers.dev", + "auth_type": "oauth", + "docs_url": "https://sprouts.ai/", + "publisher": "Sprouts.ai", + "tags": [ + "sprouts", + "prospecting", + "lead-generation", + "contacts", + "sales" + ], + "icon_url": "/images/mcp-servers/sprouts.png" + }, + { + "name": "Scite", + "description": "Connect AI tools to Scite for evidence-based answers grounded in peer-reviewed research and citation context. Hosted remotely by Scite.", + "category": "Research", + "connection_type": "http", + "connection_url": "https://api.scite.ai/mcp", + "auth_type": "oauth", + "docs_url": "https://scite.ai/", + "publisher": "Scite", + "tags": [ + "scite", + "research", + "citations", + "academic", + "science" + ], + "icon_url": "/images/mcp-servers/scite.png" + }, + { + "name": "Synthesize Bio", + "description": "Connect AI tools to Synthesize Bio to generate and analyze gene-expression data from a virtual human using natural-language experiment descriptions. Hosted remotely by Synthesize Bio.", + "category": "Research", + "connection_type": "http", + "connection_url": "https://app.synthesize.bio/api/mcp", + "auth_type": "oauth", + "docs_url": "https://docs.synthesize.bio/platform", + "publisher": "Synthesize Bio", + "tags": [ + "synthesize-bio", + "biology", + "gene-expression", + "biotech", + "research" + ], + "icon_url": "/images/mcp-servers/synthesize-bio.png" + }, + { + "name": "Legal Data Hunter", + "description": "Connect AI tools to Legal Data Hunter to search 23M+ legal documents across 160+ jurisdictions for research and document comparison. Hosted remotely by Legal Data Hunter.", + "category": "Legal", + "connection_type": "http", + "connection_url": "https://legaldatahunter.com/mcp", + "auth_type": "oauth", + "docs_url": "https://legaldatahunter.com/", + "publisher": "Legal Data Hunter", + "tags": [ + "legal-data-hunter", + "legal", + "research", + "documents", + "jurisdictions" + ], + "icon_url": "/images/mcp-servers/legal-data-hunter.png" + }, + { + "name": "Shopify", + "description": "Connect AI tools to Shopify to build, manage, and analyze online stores across products, inventory, orders, customers, and analytics. Hosted remotely by Shopify.", + "category": "E-Commerce", + "connection_type": "http", + "connection_url": "https://setup.shopify.com/mcp", + "auth_type": "oauth", + "docs_url": "https://shopify.dev/docs/apps/build/model-context-protocol", + "publisher": "Shopify", + "tags": [ + "shopify", + "ecommerce", + "products", + "orders", + "inventory" + ], + "icon_url": "/images/mcp-servers/shopify.svg" + }, + { + "name": "GoDaddy", + "description": "Connect AI tools to GoDaddy to search domains, check availability, and explore naming ideas. Hosted remotely by GoDaddy.", + "category": "Developer Tools", + "connection_type": "http", + "connection_url": "https://api.godaddy.com/v1/domains/mcp", + "auth_type": "none", + "docs_url": "https://developer.godaddy.com/mcp", + "publisher": "GoDaddy", + "tags": [ + "godaddy", + "domains", + "dns", + "hosting", + "developer-tools" + ], + "icon_url": "/images/mcp-servers/godaddy.svg" + }, + { + "name": "Magic Patterns", + "description": "Connect AI tools to Magic Patterns to discuss and iterate on product interface designs, explore UI ideas, and revise concepts. Hosted remotely by Magic Patterns.", + "category": "Design", + "connection_type": "http", + "connection_url": "https://mcp.magicpatterns.com/mcp", + "auth_type": "oauth", + "docs_url": "https://www.magicpatterns.com/", + "publisher": "Magic Patterns", + "tags": [ + "magic-patterns", + "design", + "ui", + "prototyping", + "components" + ], + "icon_url": "/images/mcp-servers/magic-patterns.png" + }, + { + "name": "Excalidraw", + "description": "Connect AI tools to Excalidraw to create interactive hand-drawn diagrams from conversations. Hosted remotely.", + "category": "Design", + "connection_type": "http", + "connection_url": "https://excalidraw-mcp-app.vercel.app/mcp", + "auth_type": "none", + "docs_url": "https://excalidraw.com/", + "publisher": "Excalidraw", + "tags": [ + "excalidraw", + "diagrams", + "whiteboard", + "sketching", + "design" + ], + "icon_url": "/images/mcp-servers/excalidraw.png" + }, + { + "name": "Netlify (Remote)", + "description": "Connect AI tools to Netlify to create, deploy, manage, and secure websites. Hosted remotely by Netlify.", + "category": "Developer Tools", + "connection_type": "http", + "connection_url": "https://netlify-mcp.netlify.app/mcp", + "auth_type": "oauth", + "docs_url": "https://docs.netlify.com/welcome/build-with-ai/netlify-mcp-server/", + "publisher": "Netlify", + "tags": [ + "netlify", + "deployments", + "hosting", + "websites", + "developer-tools" + ], + "icon_url": "/images/mcp-servers/netlify.svg" + }, + { + "name": "MailerLite", + "description": "Connect AI tools to MailerLite for drafting, reviewing, and planning email marketing work. Hosted remotely by MailerLite.", + "category": "Marketing", + "connection_type": "http", + "connection_url": "https://mcp.mailerlite.com/mcp", + "auth_type": "oauth", + "docs_url": "https://developers.mailerlite.com/", + "publisher": "MailerLite", + "tags": [ + "mailerlite", + "email-marketing", + "campaigns", + "newsletters", + "marketing" + ], + "icon_url": "/images/mcp-servers/mailerlite.png" + }, + { + "name": "Coupler.io", + "description": "Connect AI tools to Coupler.io to access business data from hundreds of marketing, sales, finance, and ecommerce sources. Hosted remotely by Coupler.io.", + "category": "Analytics", + "connection_type": "http", + "connection_url": "https://mcp.coupler.io/mcp", + "auth_type": "oauth", + "docs_url": "https://www.coupler.io/", + "publisher": "Coupler.io", + "tags": [ + "coupler", + "data-integration", + "analytics", + "reporting", + "data" + ], + "icon_url": "/images/mcp-servers/coupler.png" + }, + { + "name": "Lumin", + "description": "Connect AI tools to Lumin to manage documents, send signature requests, and convert Markdown to PDF. Hosted remotely by Lumin.", + "category": "Productivity", + "connection_type": "http", + "connection_url": "https://mcp.luminpdf.com/mcp", + "auth_type": "oauth", + "docs_url": "https://www.luminpdf.com/", + "publisher": "Lumin", + "tags": [ + "lumin", + "documents", + "signatures", + "pdf", + "productivity" + ], + "icon_url": "/images/mcp-servers/lumin.png" + }, + { + "name": "Clarify", + "description": "Connect AI tools to Clarify to query, create, and update CRM records conversationally. Hosted remotely by Clarify.", + "category": "Sales", + "connection_type": "http", + "connection_url": "https://api.clarify.ai/mcp", + "auth_type": "oauth", + "docs_url": "https://clarify.ai/", + "publisher": "Clarify", + "tags": [ + "clarify", + "crm", + "sales", + "records", + "contacts" + ], + "icon_url": "/images/mcp-servers/clarify.png" + }, + { + "name": "Crossbeam", + "description": "Connect AI tools to Crossbeam to surface partner overlaps, warm paths, and co-sell opportunities with real-time partnership context. Hosted remotely by Crossbeam.", + "category": "Sales", + "connection_type": "http", + "connection_url": "https://mcp.crossbeam.com", + "auth_type": "oauth", + "docs_url": "https://www.crossbeam.com/", + "publisher": "Crossbeam", + "tags": [ + "crossbeam", + "partnerships", + "co-sell", + "ecosystem", + "sales" + ], + "icon_url": "/images/mcp-servers/crossbeam.png" + }, + { + "name": "Pylon", + "description": "Connect AI tools to Pylon to search and manage support issues, track customer status, and coordinate responses. Hosted remotely by Pylon.", + "category": "Customer Support", + "connection_type": "http", + "connection_url": "https://mcp.usepylon.com", + "auth_type": "oauth", + "docs_url": "https://docs.usepylon.com/pylon-docs/integrations/pylon-mcp", + "publisher": "Pylon", + "tags": [ + "pylon", + "support", + "tickets", + "customer-support", + "helpdesk" + ], + "icon_url": "/images/mcp-servers/pylon.png" + }, + { + "name": "Unthread", + "description": "Connect AI tools to Unthread to search conversations, monitor SLAs, analyze support metrics, and manage support tickets. Hosted remotely by Unthread.", + "category": "Customer Support", + "connection_type": "http", + "connection_url": "https://app.unthread.io/api/mcp", + "auth_type": "oauth", + "docs_url": "https://unthread.io/", + "publisher": "Unthread", + "tags": [ + "unthread", + "support", + "tickets", + "sla", + "customer-support" + ], + "icon_url": "/images/mcp-servers/unthread.png" + }, + { + "name": "Jam", + "description": "Connect AI tools to Jam to capture screen recordings and automatic context for issue reports and bug tracking. Hosted remotely by Jam.", + "category": "Developer Tools", + "connection_type": "http", + "connection_url": "https://mcp.jam.dev/mcp", + "auth_type": "oauth", + "docs_url": "https://jam.dev/docs/jam-mcp", + "publisher": "Jam", + "tags": [ + "jam", + "bug-reports", + "screen-recording", + "debugging", + "developer-tools" + ], + "icon_url": "/images/mcp-servers/jam.png" + }, + { + "name": "Omni Analytics", + "description": "Connect AI tools to Omni to query data through a semantic model with natural language for governed analytics. Hosted remotely by Omni.", + "category": "Analytics", + "connection_type": "http", + "connection_url": "https://callbacks.omniapp.co/callback/mcp", + "auth_type": "oauth", + "docs_url": "https://omni.co/", + "publisher": "Omni", + "tags": [ + "omni", + "analytics", + "semantic-model", + "data", + "bi" + ], + "icon_url": "/images/mcp-servers/omni.png" + }, + { + "name": "Similarweb", + "description": "Connect AI tools to Similarweb for real-time web, mobile app, and market intelligence data on traffic, keywords, and benchmarking. Hosted remotely by Similarweb.", + "category": "Marketing", + "connection_type": "http", + "connection_url": "https://mcp.similarweb.com", + "auth_type": "headers", + "docs_url": "https://developers.similarweb.com/docs/similarweb-mcp", + "publisher": "Similarweb", + "tags": [ + "similarweb", + "traffic", + "market-data", + "competitive-intelligence", + "marketing" + ], + "icon_url": "/images/mcp-servers/similarweb.png" + }, + { + "name": "Peec AI", + "description": "Connect AI tools to Peec AI to analyze brand visibility across LLMs with AI-search and brand-monitoring context. Hosted remotely by Peec AI.", + "category": "Marketing", + "connection_type": "http", + "connection_url": "https://api.peec.ai/mcp", + "auth_type": "oauth", + "docs_url": "https://peec.ai/", + "publisher": "Peec AI", + "tags": [ + "peec-ai", + "brand-monitoring", + "ai-search", + "visibility", + "marketing" + ], + "icon_url": "/images/mcp-servers/peec.png" + }, + { + "name": "Motion", + "description": "Connect AI tools to Motion to analyze Meta ad creative and competitor ad libraries for creative-performance context and paid-social strategy. Hosted remotely by Motion.", + "category": "Marketing", + "connection_type": "http", + "connection_url": "https://projects.motionapp.com/mcp", + "auth_type": "oauth", + "docs_url": "https://motionapp.com/", + "publisher": "Motion", + "tags": [ + "motion", + "ads", + "creative", + "meta", + "marketing" + ], + "icon_url": "/images/mcp-servers/motion.svg" + }, + { + "name": "Splice", + "description": "Connect AI tools to Splice to search the sounds catalog, build stacks, and explore samples. Hosted remotely by Splice.", + "category": "Lifestyle", + "connection_type": "http", + "connection_url": "https://mcp.splice.com/mcp", + "auth_type": "oauth", + "docs_url": "https://splice.com/", + "publisher": "Splice", + "tags": [ + "splice", + "music", + "samples", + "audio", + "lifestyle" + ], + "icon_url": "/images/mcp-servers/splice.png" + }, + { + "name": "Ticket Tailor", + "description": "Connect AI tools to Ticket Tailor to manage events, tickets, and orders. Hosted remotely by Ticket Tailor.", + "category": "Lifestyle", + "connection_type": "http", + "connection_url": "https://mcp.tickettailor.ai/mcp", + "auth_type": "oauth", + "docs_url": "https://developers.tickettailor.com/docs/mcp/", + "publisher": "Ticket Tailor", + "tags": [ + "ticket-tailor", + "events", + "tickets", + "ticketing", + "lifestyle" + ], + "icon_url": "/images/mcp-servers/ticket-tailor.png" + }, + { + "name": "Google Cloud BigQuery", + "description": "Connect AI tools to Google Cloud BigQuery for advanced analytical insights for data agents. Hosted remotely by Google Cloud.", + "category": "Developer Tools", + "connection_type": "http", + "connection_url": "https://bigquery.googleapis.com/mcp", + "auth_type": "oauth", + "docs_url": "https://cloud.google.com/bigquery/docs", + "publisher": "Google Cloud", + "tags": [ + "bigquery", + "data-warehouse", + "analytics", + "sql", + "developer-tools" + ], + "icon_url": "/images/mcp-servers/google-cloud.svg" + }, + { + "name": "Enterpret", + "description": "Connect AI tools to Enterpret to unify customer feedback and understand themes across sources. Hosted remotely by Enterpret.", + "category": "Productivity", + "connection_type": "http", + "connection_url": "https://wisdom-api.enterpret.com/server/mcp", + "auth_type": "oauth", + "docs_url": "https://helpcenter.enterpret.com/en/articles/12665166-enterpret-mcp-server", + "publisher": "Enterpret", + "tags": [ + "enterpret", + "feedback", + "customer-insights", + "themes", + "productivity" + ], + "icon_url": "/images/mcp-servers/enterpret.png" + }, + { + "name": "pg-aiguide", + "description": "Connect AI tools to pg-aiguide to search Postgres and Tiger documentation and learn database skills. Hosted remotely by TigerData.", + "category": "Developer Tools", + "connection_type": "http", + "connection_url": "https://mcp.tigerdata.com/docs", + "auth_type": "none", + "docs_url": "https://www.tigerdata.com/", + "publisher": "TigerData", + "tags": [ + "pg-aiguide", + "postgres", + "documentation", + "database", + "developer-tools" + ], + "icon_url": "/images/mcp-servers/tigerdata.png" + }, + { + "name": "Send", + "description": "Connect AI tools to Send to create shareable documents, one-pagers, decks, and presentations. Hosted remotely by Send.", + "category": "Productivity", + "connection_type": "http", + "connection_url": "https://www.send.co/mcp", + "auth_type": "oauth", + "docs_url": "https://www.send.co/", + "publisher": "Send", + "tags": [ + "send", + "documents", + "presentations", + "decks", + "productivity" + ], + "icon_url": "/images/mcp-servers/send.png" + }, + { + "name": "Malwarebytes", + "description": "Connect AI tools to Malwarebytes to check links, phone numbers, and emails for scams with security-screening context. Hosted remotely by Malwarebytes.", + "category": "Security", + "connection_type": "http", + "connection_url": "https://scamguard.malwarebytes.com/claude/mcp", + "auth_type": "none", + "docs_url": "https://www.malwarebytes.com/", + "publisher": "Malwarebytes", + "tags": [ + "malwarebytes", + "security", + "scam-detection", + "phishing", + "safety" + ], + "icon_url": "/images/mcp-servers/malwarebytes.png" + }, + { + "name": "Era Context", + "description": "Connect AI tools to Era Context to bring personal-finance context into workflows. Hosted remotely by Era.", + "category": "Finance", + "connection_type": "http", + "connection_url": "https://context.era.app", + "auth_type": "oauth", + "docs_url": "https://era.app/", + "publisher": "Era", + "tags": [ + "era", + "personal-finance", + "budgeting", + "money", + "finance" + ], + "icon_url": "/images/mcp-servers/era.png" + }, + { + "name": "Granted", + "description": "Connect AI tools to Granted to discover grant opportunities, compare funding options, and plan applications. Hosted remotely by Granted.", + "category": "Productivity", + "connection_type": "http", + "connection_url": "https://grantedai.com/api/mcp/mcp", + "auth_type": "oauth", + "docs_url": "https://grantedai.com/developers#mcp", + "publisher": "Granted", + "tags": [ + "granted", + "grants", + "funding", + "applications", + "productivity" + ], + "icon_url": "/images/mcp-servers/granted.png" + }, + { + "name": "Quo", + "description": "Connect AI tools to Quo to surface call insights and missed opportunities through conversation intelligence. Hosted remotely by Quo.", + "category": "Sales", + "connection_type": "sse", + "connection_url": "https://mcp.quo.com/sse", + "auth_type": "headers", + "docs_url": "https://mcp.quo.com/", + "publisher": "Quo", + "tags": [ + "quo", + "conversation-intelligence", + "calls", + "sales", + "insights" + ], + "icon_url": "/images/mcp-servers/quo.png" + }, + { + "name": "Zocks", + "description": "Connect AI tools to Zocks for client conversation intelligence for financial advisors, including meeting insights, goals, and planning opportunities. Hosted remotely by Zocks.", + "category": "Finance", + "connection_type": "http", + "connection_url": "https://mcp.zocks.io/v1/mcp", + "auth_type": "oauth", + "docs_url": "https://www.zocks.io/", + "publisher": "Zocks", + "tags": [ + "zocks", + "financial-advisors", + "conversation-intelligence", + "wealth", + "finance" + ], + "icon_url": "/images/mcp-servers/zocks.png" + }, + { + "name": "Shapes", + "description": "Connect AI tools to Shapes for live HR and people data, including headcount, attrition risk, compensation gaps, and time off. Hosted remotely by Shapes.", + "category": "Human Resources", + "connection_type": "http", + "connection_url": "https://mcp.shapes.co/", + "auth_type": "oauth", + "docs_url": "https://www.shapes.co/", + "publisher": "Shapes", + "tags": [ + "shapes", + "hr", + "people-analytics", + "headcount", + "workforce" + ], + "icon_url": "/images/mcp-servers/shapes.png" + }, + { + "name": "Adobe Journey Optimizer", + "description": "Connect AI tools to Adobe Journey Optimizer to review statuses, find draft issues, and understand orchestration portfolios. Hosted remotely by Adobe.", + "category": "Marketing", + "connection_type": "http", + "connection_url": "https://ajo-mcp.adobe.io/mcp", + "auth_type": "oauth", + "docs_url": "https://business.adobe.com/products/journey-optimizer/adobe-journey-optimizer.html", + "publisher": "Adobe", + "tags": [ + "adobe", + "journey-optimizer", + "campaigns", + "orchestration", + "marketing" + ], + "icon_url": "/images/mcp-servers/adobe.svg" + }, + { + "name": "Adobe Marketing Agent", + "description": "Connect AI tools to Adobe for marketing campaign and audience insights. Hosted remotely by Adobe.", + "category": "Marketing", + "connection_type": "http", + "connection_url": "https://aep-ai-ama.adobe.io/mcp", + "auth_type": "oauth", + "docs_url": "https://business.adobe.com/products/experience-platform/adobe-experience-platform.html", + "publisher": "Adobe", + "tags": [ + "adobe", + "marketing", + "campaigns", + "audiences", + "analytics" + ], + "icon_url": "/images/mcp-servers/adobe.svg" + }, + { + "name": "Autodesk Product Help", + "description": "Connect AI tools to Autodesk Product Help to search and retrieve official product documentation across more than 110 products. Hosted remotely by Autodesk.", + "category": "Developer Tools", + "connection_type": "http", + "connection_url": "https://developer.api.autodesk.com/knowledge/public/v1/mcp", + "auth_type": "none", + "docs_url": "https://www.autodesk.com/", + "publisher": "Autodesk", + "tags": [ + "autodesk", + "documentation", + "cad", + "product-help", + "developer-tools" + ], + "icon_url": "/images/mcp-servers/autodesk.png" + }, + { + "name": "Orion by Gravity", + "description": "Connect AI tools to Orion for business insights and data-backed recommendations from an autonomous AI analyst. Hosted remotely by Gravity.", + "category": "Analytics", + "connection_type": "http", + "connection_url": "https://g.runorion.com/mcp", + "auth_type": "oauth", + "docs_url": "https://www.runorion.com/", + "publisher": "Gravity", + "tags": [ + "orion", + "analytics", + "ai-analyst", + "insights", + "data" + ], + "icon_url": "/images/mcp-servers/orion.png" + }, + { + "name": "Tropic", + "description": "Connect AI tools to Tropic to benchmark software and AI contract pricing against verified technology transactions and prepare negotiations. Hosted remotely by Tropic.", + "category": "Finance", + "connection_type": "http", + "connection_url": "https://app.tropicapp.io/mcp", + "auth_type": "oauth", + "docs_url": "https://help.tropicapp.io/hc/en-us/articles/45502083594267-Using-Model-Context-Protocol-MCP-with-Tropic", + "publisher": "Tropic", + "tags": [ + "tropic", + "procurement", + "contracts", + "spend", + "finance" + ], + "icon_url": "/images/mcp-servers/tropic.png" + }, + { + "name": "PostHog", + "description": "Connect AI tools to PostHog to manage feature flags, query product analytics, investigate errors, and explore your PostHog data. Hosted remotely by PostHog.", + "category": "Analytics", + "connection_type": "http", + "connection_url": "https://mcp.posthog.com/mcp", + "auth_type": "oauth", + "docs_url": "https://posthog.com/docs/model-context-protocol", + "publisher": "PostHog", + "tags": [ + "posthog", + "analytics", + "product-analytics", + "feature-flags", + "data" + ], + "icon_url": "/images/mcp-servers/posthog.svg" + }, + { + "name": "Tally", + "description": "Connect AI tools to Tally to create and edit forms, pull submission data, and generate insights from responses. Hosted remotely by Tally.", + "category": "Productivity", + "connection_type": "http", + "connection_url": "https://api.tally.so/mcp", + "auth_type": "oauth", + "docs_url": "https://tally.so/help/mcp", + "publisher": "Tally", + "tags": [ + "tally", + "forms", + "surveys", + "submissions", + "productivity" + ], + "icon_url": "/images/mcp-servers/tally.png" + }, + { + "name": "Playwright", + "description": "Automate browser interactions for testing, scraping, and web automation using Playwright. Runs locally via npx with support for Chrome, Firefox, WebKit, and Edge.", + "category": "Developer Tools", + "connection_type": "stdio", + "stdio_config": { + "command": "npx", + "args": [ + "@playwright/mcp@latest" + ], + "envs": [] + }, + "auth_type": "none", + "docs_url": "https://playwright.dev/docs/getting-started-mcp", + "publisher": "Microsoft", + "tags": [ + "playwright", + "browser", + "testing", + "automation", + "web" + ], + "icon_url": "/images/mcp-servers/playwright.png" + }, + { + "name": "Brave Search", + "description": "Connect AI tools to Brave Search for web search, local business search, image search, video search, news search, and AI-powered summarization. Runs locally via npx.", + "category": "Search", + "connection_type": "stdio", + "stdio_config": { + "command": "npx", + "args": [ + "-y", + "@brave/brave-search-mcp-server", + "--transport", + "stdio" + ], + "envs": [ + "BRAVE_API_KEY" + ] + }, + "auth_type": "none", + "docs_url": "https://github.com/brave/brave-search-mcp-server", + "publisher": "Brave", + "tags": [ + "brave", + "search", + "web-search", + "news", + "local-search" + ], + "icon_url": "/images/mcp-servers/brave.png" + }, + { + "name": "Perplexity", + "description": "Connect AI assistants to Perplexity's search and reasoning capabilities for web search with citations and AI-powered summarization. Runs locally via npx.", + "category": "Search", + "connection_type": "stdio", + "stdio_config": { + "command": "npx", + "args": [ + "-y", + "@perplexity-ai/mcp-server" + ], + "envs": [ + "PERPLEXITY_API_KEY" + ] + }, + "auth_type": "none", + "docs_url": "https://docs.perplexity.ai/docs/getting-started/integrations/mcp-server", + "publisher": "Perplexity", + "tags": [ + "perplexity", + "search", + "web-search", + "ai-search", + "citations" + ], + "icon_url": "/images/mcp-servers/perplexity.png" + } + ] +} diff --git a/core/bifrost.go b/core/bifrost.go index 06e6d6beae..2c53f83d7f 100644 --- a/core/bifrost.go +++ b/core/bifrost.go @@ -458,8 +458,6 @@ func (bifrost *Bifrost) ListAllModels(ctx *schemas.BifrostContext, req *schemas. }, } } - providerKeys = filterProvidersByContext(ctx, providerKeys) - startTime := time.Now() // Result structure for collecting provider responses @@ -604,35 +602,6 @@ func (bifrost *Bifrost) ListAllModels(ctx *schemas.BifrostContext, req *schemas. return response, nil } -func filterProvidersByContext(ctx *schemas.BifrostContext, providerKeys []schemas.ModelProvider) []schemas.ModelProvider { - if ctx == nil { - return providerKeys - } - - rawAvailableProviders := ctx.Value(schemas.BifrostContextKeyAvailableProviders) - if rawAvailableProviders == nil { - return providerKeys - } - - availableProviders, ok := rawAvailableProviders.([]schemas.ModelProvider) - if !ok { - return []schemas.ModelProvider{} - } - - if len(availableProviders) == 0 || len(providerKeys) == 0 { - return []schemas.ModelProvider{} - } - - filteredProviders := make([]schemas.ModelProvider, 0, len(providerKeys)) - for _, providerKey := range providerKeys { - if slices.Contains(availableProviders, providerKey) { - filteredProviders = append(filteredProviders, providerKey) - } - } - - return filteredProviders -} - // TextCompletionRequest sends a text completion request to the specified provider. func (bifrost *Bifrost) TextCompletionRequest(ctx *schemas.BifrostContext, req *schemas.BifrostTextCompletionRequest) (*schemas.BifrostTextCompletionResponse, *schemas.BifrostError) { if req == nil { @@ -2197,7 +2166,8 @@ func (bifrost *Bifrost) FileUploadRequest(ctx *schemas.BifrostContext, req *sche }, } } - if len(req.File) == 0 { + + if len(req.File) == 0 && req.Provider != schemas.Vertex { return nil, &schemas.BifrostError{ IsBifrostError: false, Error: &schemas.ErrorField{ @@ -4197,6 +4167,33 @@ type RealtimeTurnHooks struct { Cleanup func() } +// RunPreRequestHooks acquires a plugin pipeline and runs PreRequestHook on each LLM plugin +// for callers that do not flow through handleRequest/handleStreamRequest — primarily realtime +// WebSocket upgrades, where the upgrade itself is the routing decision (once per WS connection) +// but the per-turn pipeline handles PreLLMHook/PostLLMHook separately. +// +// Mutations to req.Provider/req.Model/req.Fallbacks made by PreRequestHook plugins are committed +// to the shared *BifrostRequest. Plugin errors are non-blocking — they are logged as warnings +// and the pipeline continues to the next plugin (same semantics as RunLLMPreHooks). Callers +// should validate req.Provider after this returns if a provider is required. +func (bifrost *Bifrost) RunPreRequestHooks(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) { + if ctx == nil { + ctx = bifrost.ctx + } + + if _, ok := ctx.Value(schemas.BifrostContextKeyRequestID).(string); !ok { + ctx.SetValue(schemas.BifrostContextKeyRequestID, uuid.New().String()) + } + + pipeline := bifrost.getPluginPipeline() + defer bifrost.releasePluginPipeline(pipeline) + pipeline.RunPreRequestHooks(ctx, req) + // This path has no downstream post-hook cleanup, so drain any plugin logs + // emitted by PreRequestHook here to avoid them bleeding into a later request + // on a reused/long-lived context (e.g. realtime WS connections). + flushPluginLogs(ctx) +} + // RunStreamPreHooks acquires a plugin pipeline, sets up tracing context, runs PreLLMHooks, // and returns a PostHookRunner for per-chunk post-processing. // Used by WebSocket handlers that bypass the normal inference path but still need plugin hooks. @@ -4636,18 +4633,12 @@ func (bifrost *Bifrost) shouldContinueWithFallbacks(fallback schemas.Fallback, f func (bifrost *Bifrost) handleRequest(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) (*schemas.BifrostResponse, *schemas.BifrostError) { defer bifrost.releaseBifrostRequest(req) provider, model, fallbacks := req.GetRequestFields() - if err := validateRequest(req); err != nil { - err.PopulateExtraFields(req.RequestType, provider, model, model) - return nil, err - } // Handle nil context early to prevent blocking if ctx == nil { ctx = bifrost.ctx } - bifrost.logger.Debug(fmt.Sprintf("primary provider %s with model %s and %d fallbacks", provider, model, len(fallbacks))) - // Try the primary provider first ctx.SetValue(schemas.BifrostContextKeyFallbackIndex, 0) // Ensure request ID is set in context before PreHooks @@ -4655,6 +4646,27 @@ func (bifrost *Bifrost) handleRequest(ctx *schemas.BifrostContext, req *schemas. requestID := uuid.New().String() ctx.SetValue(schemas.BifrostContextKeyRequestID, requestID) } + + // PreRequestHook: once-per-request phase where plugins decide provider/model/fallbacks + // (and may mutate other request fields). Mutations commit to req and are observed by + // all downstream phases and fallbacks. Plugin errors are non-blocking (logged + skipped). + preReqPipeline := bifrost.getPluginPipeline() + preReqPipeline.RunPreRequestHooks(ctx, req) + bifrost.releasePluginPipeline(preReqPipeline) + // Re-read after PreRequestHook — provider/model/fallbacks may have changed. + provider, model, fallbacks = req.GetRequestFields() + // Empty provider/model after PreRequestHook means no plugin + // could pick a provider for this model — the caller's input is unresolvable. + if err := validateRequestAfterPreRequestHooks(req); err != nil { + // Returning before tryRequest skips the downstream log drain, so flush + // any PreRequestHook-emitted plugin logs here. + flushPluginLogs(ctx) + err.PopulateExtraFields(req.RequestType, provider, model, model) + return nil, err + } + + bifrost.logger.Debug(fmt.Sprintf("primary provider %s with model %s and %d fallbacks", provider, model, len(fallbacks))) + primaryResult, primaryErr := bifrost.tryRequest(ctx, req) if primaryErr != nil { if primaryErr.Error != nil { @@ -4673,10 +4685,22 @@ func (bifrost *Bifrost) handleRequest(ctx *schemas.BifrostContext, req *schemas. return primaryResult, primaryErr } + // Core is about to make routing decisions of its own (fallback transitions) + // — record it on the request's used-engines list so the audit trail closes + // the loop on whatever plugin-level engine selected the primary upstream. + schemas.AppendToContextList(ctx, schemas.BifrostContextKeyRoutingEnginesUsed, schemas.RoutingEngineCore) + ctx.AppendRoutingEngineLog(schemas.RoutingEngineCore, schemas.LogLevelInfo, fmt.Sprintf("Primary %s/%s failed (%s); evaluating %d configured fallback(s)", provider, model, routingErrorSummary(primaryErr), len(fallbacks))) + + // Tracks the most recent failure so each fallback transition log carries + // the error that triggered it (primary error for the first iteration, the + // prior fallback's error for subsequent iterations). + lastErr := primaryErr + // Try fallbacks in order for i, fallback := range fallbacks { ctx.SetValue(schemas.BifrostContextKeyFallbackIndex, i+1) bifrost.logger.Debug(fmt.Sprintf("trying fallback provider %s with model %s", fallback.Provider, fallback.Model)) + ctx.AppendRoutingEngineLog(schemas.RoutingEngineCore, schemas.LogLevelInfo, fmt.Sprintf("Trying fallback %d/%d: %s/%s (previous attempt failed: %s)", i+1, len(fallbacks), fallback.Provider, fallback.Model, routingErrorSummary(lastErr))) ctx.SetValue(schemas.BifrostContextKeyFallbackRequestID, uuid.New().String()) clearCtxForFallback(ctx) @@ -4692,6 +4716,7 @@ func (bifrost *Bifrost) handleRequest(ctx *schemas.BifrostContext, req *schemas. fallbackReq := bifrost.prepareFallbackRequest(req, fallback) if fallbackReq == nil { bifrost.logger.Debug(fmt.Sprintf("fallback provider %s with model %s is nil", fallback.Provider, fallback.Model)) + ctx.AppendRoutingEngineLog(schemas.RoutingEngineCore, schemas.LogLevelWarn, fmt.Sprintf("Fallback %s/%s skipped: missing provider config", fallback.Provider, fallback.Model)) tracer.SetAttribute(handle, "error", "fallback request preparation failed") tracer.EndSpan(handle, schemas.SpanStatusError, "fallback request preparation failed") continue @@ -4699,8 +4724,14 @@ func (bifrost *Bifrost) handleRequest(ctx *schemas.BifrostContext, req *schemas. // Try the fallback provider result, fallbackErr := bifrost.tryRequest(ctx, fallbackReq) + // Layer on Primary/IsFallback — the per-attempt code populates only + // attempt-level RoutingInfo (Provider/Model/Key/ResolvedKeyAlias); + // fallback-relative signals belong to the orchestrator scope. + result.SetFallbackRoutingInfo(provider, model) + fallbackErr.SetFallbackRoutingInfo(provider, model) if fallbackErr == nil { bifrost.logger.Debug(fmt.Sprintf("successfully used fallback provider %s with model %s", fallback.Provider, fallback.Model)) + ctx.AppendRoutingEngineLog(schemas.RoutingEngineCore, schemas.LogLevelInfo, fmt.Sprintf("Request served by fallback %s/%s (attempt %d/%d)", fallback.Provider, fallback.Model, i+1, len(fallbacks))) tracer.EndSpan(handle, schemas.SpanStatusOk, "") return result, nil } @@ -4713,10 +4744,14 @@ func (bifrost *Bifrost) handleRequest(ctx *schemas.BifrostContext, req *schemas. // Check if we should continue with more fallbacks if !bifrost.shouldContinueWithFallbacks(fallback, fallbackErr) { + ctx.AppendRoutingEngineLog(schemas.RoutingEngineCore, schemas.LogLevelError, fmt.Sprintf("Fallback %s/%s failed (%s); halting further fallbacks", fallback.Provider, fallback.Model, routingErrorSummary(fallbackErr))) return nil, fallbackErr } + + lastErr = fallbackErr } + ctx.AppendRoutingEngineLog(schemas.RoutingEngineCore, schemas.LogLevelError, fmt.Sprintf("All %d fallback(s) exhausted; returning primary error (%s)", len(fallbacks), routingErrorSummary(primaryErr))) // All providers failed, return the original error return nil, primaryErr } @@ -4727,15 +4762,8 @@ func (bifrost *Bifrost) handleRequest(ctx *schemas.BifrostContext, req *schemas. // It is the wrapper for all streaming public API methods. func (bifrost *Bifrost) handleStreamRequest(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { defer bifrost.releaseBifrostRequest(req) - provider, model, fallbacks := req.GetRequestFields() - if err := validateRequest(req); err != nil { - err.PopulateExtraFields(req.RequestType, provider, model, model) - err.StatusCode = schemas.Ptr(fasthttp.StatusBadRequest) - return nil, err - } - // Handle nil context early to prevent blocking if ctx == nil { ctx = bifrost.ctx @@ -4748,7 +4776,36 @@ func (bifrost *Bifrost) handleStreamRequest(ctx *schemas.BifrostContext, req *sc requestID := uuid.New().String() ctx.SetValue(schemas.BifrostContextKeyRequestID, requestID) } + + // PreRequestHook: once-per-request phase. See handleRequest for semantics. + preReqPipeline := bifrost.getPluginPipeline() + preReqPipeline.RunPreRequestHooks(ctx, req) + bifrost.releasePluginPipeline(preReqPipeline) + // Re-read after PreRequestHook — provider/model/fallbacks may have changed. + provider, model, fallbacks = req.GetRequestFields() + // Empty provider after PreRequestHook means no plugin + // could pick a provider for this model — the caller's input is unresolvable. + if err := validateRequestAfterPreRequestHooks(req); err != nil { + // Returning before tryStreamRequest skips the downstream log drain, so + // flush any PreRequestHook-emitted plugin logs here. + flushPluginLogs(ctx) + err.PopulateExtraFields(req.RequestType, provider, model, model) + return nil, err + } + + bifrost.logger.Debug(fmt.Sprintf("primary provider %s with model %s and %d fallbacks", provider, model, len(fallbacks))) + primaryResult, primaryErr := bifrost.tryStreamRequest(ctx, req) + if primaryErr != nil { + if primaryErr.Error != nil { + bifrost.logger.Debug(fmt.Sprintf("primary provider %s with model %s returned error: %s", provider, model, primaryErr.Error.Message)) + } else { + bifrost.logger.Debug(fmt.Sprintf("primary provider %s with model %s returned error: %v", provider, model, primaryErr)) + } + if len(fallbacks) > 0 { + bifrost.logger.Debug(fmt.Sprintf("check if we should try %d fallbacks", len(fallbacks))) + } + } // Check if we should proceed with fallbacks shouldTryFallbacks := bifrost.shouldTryFallbacks(req, primaryErr) @@ -4756,9 +4813,18 @@ func (bifrost *Bifrost) handleStreamRequest(ctx *schemas.BifrostContext, req *sc return primaryResult, primaryErr } + // Mirror handleRequest: register core on the engines-used list and post + // the primary-failure entry to the routing engine log trail before + // iterating fallbacks. See handleRequest for the rationale. + schemas.AppendToContextList(ctx, schemas.BifrostContextKeyRoutingEnginesUsed, schemas.RoutingEngineCore) + ctx.AppendRoutingEngineLog(schemas.RoutingEngineCore, schemas.LogLevelInfo, fmt.Sprintf("Primary %s/%s failed (%s); evaluating %d configured fallback(s)", provider, model, routingErrorSummary(primaryErr), len(fallbacks))) + + lastErr := primaryErr + // Try fallbacks in order for i, fallback := range fallbacks { ctx.SetValue(schemas.BifrostContextKeyFallbackIndex, i+1) + ctx.AppendRoutingEngineLog(schemas.RoutingEngineCore, schemas.LogLevelInfo, fmt.Sprintf("Trying fallback %d/%d: %s/%s (previous attempt failed: %s)", i+1, len(fallbacks), fallback.Provider, fallback.Model, routingErrorSummary(lastErr))) ctx.SetValue(schemas.BifrostContextKeyFallbackRequestID, uuid.New().String()) clearCtxForFallback(ctx) @@ -4773,6 +4839,7 @@ func (bifrost *Bifrost) handleStreamRequest(ctx *schemas.BifrostContext, req *sc fallbackReq := bifrost.prepareFallbackRequest(req, fallback) if fallbackReq == nil { + ctx.AppendRoutingEngineLog(schemas.RoutingEngineCore, schemas.LogLevelWarn, fmt.Sprintf("Fallback %s/%s skipped: missing provider config", fallback.Provider, fallback.Model)) tracer.SetAttribute(handle, "error", "fallback request preparation failed") tracer.EndSpan(handle, schemas.SpanStatusError, "fallback request preparation failed") continue @@ -4780,8 +4847,15 @@ func (bifrost *Bifrost) handleStreamRequest(ctx *schemas.BifrostContext, req *sc // Try the fallback provider result, fallbackErr := bifrost.tryStreamRequest(ctx, fallbackReq) + // Layer on Primary/IsFallback on errors. For the success case the + // result is a chan of stream chunks emitted asynchronously — those + // chunks already carry per-attempt RoutingInfo populated upstream, + // but Primary/IsFallback aren't reachable from here without wrapping + // the channel. See SetFallbackRoutingInfo doc. + fallbackErr.SetFallbackRoutingInfo(provider, model) if fallbackErr == nil { bifrost.logger.Debug(fmt.Sprintf("successfully used fallback provider %s with model %s", fallback.Provider, fallback.Model)) + ctx.AppendRoutingEngineLog(schemas.RoutingEngineCore, schemas.LogLevelInfo, fmt.Sprintf("Request served by fallback %s/%s (attempt %d/%d)", fallback.Provider, fallback.Model, i+1, len(fallbacks))) tracer.EndSpan(handle, schemas.SpanStatusOk, "") return result, nil } @@ -4794,10 +4868,14 @@ func (bifrost *Bifrost) handleStreamRequest(ctx *schemas.BifrostContext, req *sc // Check if we should continue with more fallbacks if !bifrost.shouldContinueWithFallbacks(fallback, fallbackErr) { + ctx.AppendRoutingEngineLog(schemas.RoutingEngineCore, schemas.LogLevelError, fmt.Sprintf("Fallback %s/%s failed (%s); halting further fallbacks", fallback.Provider, fallback.Model, routingErrorSummary(fallbackErr))) return nil, fallbackErr } + + lastErr = fallbackErr } + ctx.AppendRoutingEngineLog(schemas.RoutingEngineCore, schemas.LogLevelError, fmt.Sprintf("All %d fallback(s) exhausted; returning primary error (%s)", len(fallbacks), routingErrorSummary(primaryErr))) // All providers failed, return the original error return nil, primaryErr } @@ -5359,11 +5437,32 @@ func executeRequestWithRetries[T any]( model string, req *schemas.BifrostRequest, logger schemas.Logger, -) (T, *schemas.BifrostError) { - var result T - var bifrostError *schemas.BifrostError +) (result T, bifrostError *schemas.BifrostError) { var attempts int + // Emit the terminal routing-engine entry on every return path — including + // early returns from key-selection failures and tracer-missing — so the + // audit trail isn't truncated when execution exits before reaching the + // natural end of the function. Skipped when attempts == 0: the request + // never crossed core's retry-orchestration boundary, so there's nothing + // to record. + defer func() { + if attempts <= 0 { + return + } + schemas.AppendToContextList(ctx, schemas.BifrostContextKeyRoutingEnginesUsed, schemas.RoutingEngineCore) + switch { + case bifrostError == nil: + ctx.AppendRoutingEngineLog(schemas.RoutingEngineCore, schemas.LogLevelInfo, fmt.Sprintf("Request to %s/%s succeeded after %d retry attempt(s)", providerKey, model, attempts)) + case bifrostError.IsBifrostError: + ctx.AppendRoutingEngineLog(schemas.RoutingEngineCore, schemas.LogLevelError, fmt.Sprintf("Retries halted for %s/%s after %d attempt(s): internal Bifrost error (%s)", providerKey, model, attempts, routingErrorSummary(bifrostError))) + case bifrostError.Error != nil && bifrostError.Error.Type != nil && *bifrostError.Error.Type == schemas.RequestCancelled: + ctx.AppendRoutingEngineLog(schemas.RoutingEngineCore, schemas.LogLevelError, fmt.Sprintf("Request to %s/%s cancelled after %d attempt(s)", providerKey, model, attempts)) + default: + ctx.AppendRoutingEngineLog(schemas.RoutingEngineCore, schemas.LogLevelError, fmt.Sprintf("Retries exhausted for %s/%s after %d attempt(s); last error: %s", providerKey, model, attempts, routingErrorSummary(bifrostError))) + } + }() + var currentKey schemas.Key var usedKeyIDs map[string]bool var deadKeyIDs map[string]bool @@ -5511,6 +5610,25 @@ func executeRequestWithRetries[T any]( // - 429 pool reset that re-picks the same key — no rotation actually happened. // - keyless providers — currentKey.ID stays empty, so keyChanged is false. keyChanged := keyProvider != nil && currentKey.ID != previousKeyID + + // Emit a routing-engine log entry for this retry transition so the + // per-request audit trail records *why* core decided to retry and + // whether it rotated the credential. routingErrorSummary() omits + // the upstream message so keys/PII don't leak into the log row. + // Key.Name is a user-set label (not the secret value) and is safe. + schemas.AppendToContextList(ctx, schemas.BifrostContextKeyRoutingEnginesUsed, schemas.RoutingEngineCore) + // Omit the key=... segment for keyless providers, where currentKey is + // the zero value and the trailing token would render as "same key=". + keyNote := "" + if keyProvider != nil { + rotationNote := "same key" + if keyChanged { + rotationNote = "rotated key" + } + keyNote = fmt.Sprintf("; %s=%s", rotationNote, currentKey.Name) + } + ctx.AppendRoutingEngineLog(schemas.RoutingEngineCore, schemas.LogLevelInfo, fmt.Sprintf("Retry %d/%d for %s/%s (previous attempt failed: %s%s)", attempts, config.NetworkConfig.MaxRetries, providerKey, model, routingErrorSummary(bifrostError), keyNote)) + if !(lastWasPermanentKeyFailure && keyChanged) { backoff := calculateBackoff(attempts-1, config) logger.Debug("sleeping for %s before retry", backoff) @@ -5801,6 +5919,10 @@ func executeRequestWithRetries[T any]( logger.Debug("request failed after %d %s", attempts, map[bool]string{true: "attempts", false: "attempt"}[attempts > 1]) } + // Terminal routing-engine log entry is emitted by the defer at the top of + // the function so it runs on every return path, including the early + // returns from key-selection or tracer-missing. + // On final error, clear selected_key so it only reflects a key that actually served a successful response. // The attempt trail is the authoritative record of which keys were tried. if bifrostError != nil && keyProvider != nil { @@ -5952,6 +6074,29 @@ func (bifrost *Bifrost) requestWorker(provider schemas.Provider, config *schemas } continue } + // Scope to a single key when ListModelsRequest.KeyID is set, so + // callers (the catalog composer) can cache per-key without an + // extra round-trip and without the provider aggregating across + // every configured key. + if lmr := req.BifrostRequest.ListModelsRequest; lmr != nil && lmr.KeyID != nil { + target := *lmr.KeyID + keys = filterKeysByID(keys, target) + if len(keys) == 0 { + req.Err <- schemas.BifrostError{ + IsBifrostError: false, + Error: &schemas.ErrorField{ + Message: fmt.Sprintf("no key found with id %q for provider %s", target, provider.GetProviderKey()), + }, + ExtraFields: schemas.BifrostErrorExtraFields{ + Provider: provider.GetProviderKey(), + RequestType: req.RequestType, + OriginalModelRequested: model, + ResolvedModelUsed: model, + }, + } + continue + } + } } else { // Determine if this is a multi-key batch/file/container operation // BatchCreate, FileUpload, ContainerCreate, ContainerFileCreate use single key; other batch/file/container ops use multiple keys @@ -6067,6 +6212,20 @@ func (bifrost *Bifrost) requestWorker(provider schemas.Provider, config *schemas // line 5653). Streaming postHookRunner must NOT capture this var by reference — it // snapshots its own attemptResolvedModel inside the per-attempt closure. var resolvedModel string + // attemptRoutingInfo holds the LAST attempt's RoutingInfo. Same single-writer/ + // single-reader contract as resolvedModel — assigned inside the per-attempt + // closure, read after retries finish by the post-retry populate below. + // Streaming postHookRunner must NOT capture by reference — it snapshots its + // own copy inside the per-attempt closure. + // Pre-seeded with the provider/model the orchestrator already knows so that + // retries that fail before the per-attempt closure ever runs (e.g. key + // selection error) still produce a populated RoutingInfo on the error — + // otherwise the post-retry populate at line ~6221 would clobber RoutingInfo + // to a zero value, leaving new consumers without provider/model context. + attemptRoutingInfo := schemas.RoutingInfo{ + Provider: provider.GetProviderKey(), + Model: originalModelRequested, + } // lastAttemptFinalizer captures the LAST attempt's postHookSpanFinalizer for the // worker-level error fallback below. Single-threaded write (assigned by the retry // loop's per-attempt closure) and single-threaded read (after retries finish), so @@ -6083,11 +6242,21 @@ func (bifrost *Bifrost) requestWorker(provider schemas.Provider, config *schemas // returned to the pool via its deferred finalizer. if IsStreamRequestType(req.RequestType) { stream, bifrostError = executeRequestWithRetries(req.Context, config, func(k schemas.Key) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { - resolvedModel = k.Aliases.Resolve(originalModelRequested) + if aliasConfig := k.Aliases.ResolveConfig(originalModelRequested); aliasConfig != nil { + resolvedModel = aliasConfig.ModelID + req.Context.SetValue(schemas.BifrostContextKeyResolvedAlias, &schemas.ResolvedAlias{Key: originalModelRequested, Config: aliasConfig}) + } else { + resolvedModel = originalModelRequested + req.Context.SetValue(schemas.BifrostContextKeyResolvedAlias, nil) + } req.SetModel(resolvedModel) // Snapshot per-attempt so postHookRunner doesn't observe a later retry's // alias while this attempt's provider goroutine is still emitting chunks. attemptResolvedModel := resolvedModel + attemptRoutingInfo = schemas.BuildRoutingInfo(req.Context, provider.GetProviderKey(), originalModelRequested, k) + // Per-attempt snapshot for the async postHookRunner closure (it must + // not capture the outer var by reference — a later retry would race). + perAttemptRoutingInfo := attemptRoutingInfo // Snapshot RequestType before the closure. After tryStreamRequest receives // the stream channel it releases the *ChannelMessage back to the pool; // a concurrent request can then reuse it and overwrite RequestType. @@ -6101,9 +6270,11 @@ func (bifrost *Bifrost) requestWorker(provider schemas.Provider, config *schemas // reference would let a later retry's alias bleed into this attempt's chunks. if result != nil { result.PopulateExtraFields(attemptRequestType, provider.GetProviderKey(), originalModelRequested, attemptResolvedModel) + result.PopulateRoutingInfo(perAttemptRoutingInfo) } if err != nil { err.PopulateExtraFields(attemptRequestType, provider.GetProviderKey(), originalModelRequested, attemptResolvedModel) + err.PopulateRoutingInfo(perAttemptRoutingInfo) } resp, bifrostErr := pipeline.RunPostLLMHooks(ctx, result, err, len(*bifrost.llmPlugins.Load())) if IsFinalChunk(ctx) { @@ -6111,9 +6282,11 @@ func (bifrost *Bifrost) requestWorker(provider schemas.Provider, config *schemas } if bifrostErr != nil { bifrostErr.PopulateExtraFields(attemptRequestType, provider.GetProviderKey(), originalModelRequested, attemptResolvedModel) + bifrostErr.PopulateRoutingInfo(perAttemptRoutingInfo) return nil, bifrostErr } else if resp != nil { resp.PopulateExtraFields(attemptRequestType, provider.GetProviderKey(), originalModelRequested, attemptResolvedModel) + resp.PopulateRoutingInfo(perAttemptRoutingInfo) } return resp, nil } @@ -6142,8 +6315,15 @@ func (bifrost *Bifrost) requestWorker(provider schemas.Provider, config *schemas }, keyProvider, req.RequestType, provider.GetProviderKey(), model, &req.BifrostRequest, bifrost.logger) } else { result, bifrostError = executeRequestWithRetries(req.Context, config, func(k schemas.Key) (*schemas.BifrostResponse, *schemas.BifrostError) { - resolvedModel = k.Aliases.Resolve(originalModelRequested) + if aliasConfig := k.Aliases.ResolveConfig(originalModelRequested); aliasConfig != nil { + resolvedModel = aliasConfig.ModelID + req.Context.SetValue(schemas.BifrostContextKeyResolvedAlias, &schemas.ResolvedAlias{Key: originalModelRequested, Config: aliasConfig}) + } else { + resolvedModel = originalModelRequested + req.Context.SetValue(schemas.BifrostContextKeyResolvedAlias, nil) + } req.SetModel(resolvedModel) + attemptRoutingInfo = schemas.BuildRoutingInfo(req.Context, provider.GetProviderKey(), originalModelRequested, k) return bifrost.handleProviderRequest(provider, config, req, k, keys) }, keyProvider, req.RequestType, provider.GetProviderKey(), model, &req.BifrostRequest, bifrost.logger) } @@ -6166,6 +6346,7 @@ func (bifrost *Bifrost) requestWorker(provider schemas.Provider, config *schemas if bifrostError != nil { bifrostError.PopulateExtraFields(req.RequestType, provider.GetProviderKey(), originalModelRequested, resolvedModel) + bifrostError.PopulateRoutingInfo(attemptRoutingInfo) // Send error with context awareness to prevent deadlock select { @@ -6181,6 +6362,7 @@ func (bifrost *Bifrost) requestWorker(provider schemas.Provider, config *schemas } else { if result != nil { result.PopulateExtraFields(req.RequestType, provider.GetProviderKey(), originalModelRequested, resolvedModel) + result.PopulateRoutingInfo(attemptRoutingInfo) } if IsStreamRequestType(req.RequestType) { // Send stream with context awareness to prevent deadlock @@ -6653,6 +6835,49 @@ func (p *PluginPipeline) RunLLMPreHooks(ctx *schemas.BifrostContext, req *schema return req, nil, p.executedPreHooks } +// RunPreRequestHooks executes PreRequestHook on each LLM plugin in registration order, once per +// top-level request. Plugins mutate req.Provider, req.Model, req.Fallbacks (and any other field +// they choose); mutations are committed to the shared *BifrostRequest and observed by every +// subsequent plugin, the provider call, and every fallback attempt. There is no short-circuit +// and errors are non-blocking — same semantics as RunLLMPreHooks: errors are logged as warnings +// and accumulated in p.preHookErrors, then the pipeline continues to the next plugin. The empty- +// provider validation in handleRequest/handleStreamRequest catches the case where no plugin +// successfully resolved a provider. +// +// Per-request semantics: unlike PreLLMHook (which runs again on every fallback), PreRequestHook +// runs exactly once at the top of handleRequest/handleStreamRequest, before any fan-out. +func (p *PluginPipeline) RunPreRequestHooks(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) { + // If the skip plugin pipeline flag is set, skip the plugin pipeline + if skipPluginPipeline, ok := ctx.Value(schemas.BifrostContextKeySkipPluginPipeline).(bool); ok && skipPluginPipeline { + return + } + ctx.BlockRestrictedWrites() + defer ctx.UnblockRestrictedWrites() + for _, plugin := range p.llmPlugins { + pluginName := plugin.GetName() + p.logger.Debug("running pre-request hook for plugin %s", pluginName) + spanCtx, handle := p.tracer.StartSpan(ctx, fmt.Sprintf("plugin.%s.prerequesthook", sanitizeSpanName(pluginName)), schemas.SpanKindPlugin) + if spanCtx != nil { + if spanID, ok := spanCtx.Value(schemas.BifrostContextKeySpanID).(string); ok { + ctx.SetValue(schemas.BifrostContextKeySpanID, spanID) + } + } + + pluginCtx := ctx.WithPluginScope(&pluginName) + err := plugin.PreRequestHook(pluginCtx, req) + pluginCtx.ReleasePluginScope() + + if err != nil { + p.tracer.SetAttribute(handle, "error", err.Error()) + p.tracer.EndSpan(handle, schemas.SpanStatusError, err.Error()) + p.preHookErrors = append(p.preHookErrors, err) + p.logger.Warn("error in PreRequestHook for plugin %s: %s", pluginName, err.Error()) + continue + } + p.tracer.EndSpan(handle, schemas.SpanStatusOk, "") + } +} + // RunPostLLMHooks executes PostHooks in reverse order for the plugins whose PreLLMHook ran. // Accepts the response and error, and allows plugins to transform either (e.g., recover from error, or invalidate a response). // Returns the final response and error after all hooks. If both are set, error takes precedence unless error is nil. @@ -7013,6 +7238,22 @@ func (p *PluginPipeline) resetPluginPipeline() { p.streamingMu.Unlock() } +// flushPluginLogs drains accumulated plugin logs from the BifrostContext and +// attaches them to the active trace when one exists. Unlike drainAndAttachPluginLogs, +// it always drains the buffer first, so logs emitted before any trace is established +// (e.g. by PreRequestHook) are not carried over to a later request on a reused context. +func flushPluginLogs(ctx *schemas.BifrostContext) { + logs := ctx.DrainPluginLogs() + if len(logs) == 0 { + return + } + tracer, traceID, err := GetTracerFromContext(ctx) + if err != nil || tracer == nil || traceID == "" { + return + } + tracer.AttachPluginLogs(traceID, logs) +} + // drainAndAttachPluginLogs drains accumulated plugin logs from the BifrostContext // and attaches them to the trace for later retrieval by observability plugins. func drainAndAttachPluginLogs(ctx *schemas.BifrostContext) { @@ -7324,6 +7565,20 @@ func (bifrost *Bifrost) releaseBifrostRequest(req *schemas.BifrostRequest) { bifrost.bifrostRequestPool.Put(req) } +// filterKeysByID returns the subset of keys whose ID equals target. Used to +// scope a ListModels request to a single key when ListModelsRequest.KeyID is +// set. Returns an empty slice when no key matches; the input slice is not +// mutated. +func filterKeysByID(keys []schemas.Key, target string) []schemas.Key { + out := make([]schemas.Key, 0, len(keys)) + for _, k := range keys { + if k.ID == target { + out = append(out, k) + } + } + return out +} + // getAllSupportedKeys retrieves all valid keys for a ListModels request. // allowing the provider to aggregate results from multiple keys. func (bifrost *Bifrost) getAllSupportedKeys(ctx *schemas.BifrostContext, providerKey schemas.ModelProvider, baseProviderType schemas.ModelProvider) ([]schemas.Key, error) { diff --git a/core/bifrost_test.go b/core/bifrost_test.go index 5c1300687f..5023684c7d 100644 --- a/core/bifrost_test.go +++ b/core/bifrost_test.go @@ -789,51 +789,6 @@ func (t *countingTracer) CompleteAndFlushTrace(_ string) { t.flushed.Add(1) } -func TestFilterProvidersByContext(t *testing.T) { - providers := []schemas.ModelProvider{ - schemas.OpenAI, - schemas.Anthropic, - schemas.Mistral, - } - - t.Run("no context filter keeps all providers", func(t *testing.T) { - filtered := filterProvidersByContext(nil, providers) - if len(filtered) != len(providers) { - t.Fatalf("expected all providers, got %v", filtered) - } - }) - - t.Run("available providers restrict list models fanout", func(t *testing.T) { - ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) - ctx.SetValue(schemas.BifrostContextKeyAvailableProviders, []schemas.ModelProvider{schemas.Anthropic}) - - filtered := filterProvidersByContext(ctx, providers) - if len(filtered) != 1 || filtered[0] != schemas.Anthropic { - t.Fatalf("expected only anthropic, got %v", filtered) - } - }) - - t.Run("empty available providers denies all providers", func(t *testing.T) { - ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) - ctx.SetValue(schemas.BifrostContextKeyAvailableProviders, []schemas.ModelProvider{}) - - filtered := filterProvidersByContext(ctx, providers) - if len(filtered) != 0 { - t.Fatalf("expected no providers, got %v", filtered) - } - }) - - t.Run("malformed available providers fails closed", func(t *testing.T) { - ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) - ctx.SetValue(schemas.BifrostContextKeyAvailableProviders, "openai") - - filtered := filterProvidersByContext(ctx, providers) - if len(filtered) != 0 { - t.Fatalf("expected no providers for malformed context value, got %v", filtered) - } - }) -} - func TestRunStreamPreHooks_FinalChunkFlushesTrace(t *testing.T) { ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) account := NewMockAccount() @@ -2757,3 +2712,47 @@ func TestPluginPipelineStreamingRace(t *testing.T) { wg.Wait() } + +// TestFilterKeysByID covers the KeyID scoping path for ListModels requests: +// a hit returns the single matching key, a miss returns an empty slice +// (which the caller surfaces as "no key found"), and the input slice must +// not be mutated. +func TestFilterKeysByID(t *testing.T) { + keys := []schemas.Key{ + {ID: "k1"}, + {ID: "k2"}, + {ID: "k3"}, + } + + t.Run("match returns single key", func(t *testing.T) { + got := filterKeysByID(keys, "k2") + if len(got) != 1 || got[0].ID != "k2" { + t.Fatalf("filterKeysByID(_, k2) = %+v, want one key with ID=k2", got) + } + }) + + t.Run("no match returns empty slice", func(t *testing.T) { + got := filterKeysByID(keys, "does-not-exist") + if len(got) != 0 { + t.Fatalf("filterKeysByID(_, missing) = %+v, want empty", got) + } + }) + + t.Run("empty target returns empty slice", func(t *testing.T) { + got := filterKeysByID(keys, "") + if len(got) != 0 { + t.Fatalf("filterKeysByID(_, \"\") = %+v, want empty", got) + } + }) + + t.Run("input slice is not mutated", func(t *testing.T) { + before := make([]schemas.Key, len(keys)) + copy(before, keys) + _ = filterKeysByID(keys, "k1") + for i := range keys { + if keys[i].ID != before[i].ID { + t.Fatalf("input mutated at index %d: got %q, want %q", i, keys[i].ID, before[i].ID) + } + } + }) +} diff --git a/core/go.mod b/core/go.mod index 150ecd3e3f..a82049ea51 100644 --- a/core/go.mod +++ b/core/go.mod @@ -7,13 +7,13 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 github.com/andybalholm/brotli v1.2.1 - github.com/aws/aws-sdk-go-v2 v1.41.7 + github.com/aws/aws-sdk-go-v2 v1.41.12 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 github.com/aws/aws-sdk-go-v2/config v1.32.11 github.com/aws/aws-sdk-go-v2/credentials v1.19.14 github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 - github.com/aws/smithy-go v1.25.1 + github.com/aws/smithy-go v1.27.1 github.com/bytedance/sonic v1.15.1 github.com/cespare/xxhash/v2 v2.3.0 github.com/fasthttp/websocket v1.5.12 @@ -37,8 +37,8 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect diff --git a/core/go.sum b/core/go.sum index 845df9306c..b05a47e019 100644 --- a/core/go.sum +++ b/core/go.sum @@ -16,8 +16,8 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgv github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro= github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= -github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= -github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2 v1.41.12 h1:DIKX2c31ekm9RA2D9FBj1EWXx++9AdAqRw+e78Tq2Ck= +github.com/aws/aws-sdk-go-v2 v1.41.12/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= github.com/aws/aws-sdk-go-v2/config v1.32.11 h1:ftxI5sgz8jZkckuUHXfC/wMUc8u3fG1vQS0plr2F2Zs= @@ -26,10 +26,10 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8TH github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 h1:Xf2j7NdVcUKomlZ4iihOP4AZ3Fzlr8h4yKpXeP+OFPg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28/go.mod h1:O8cDo1dW63jU7ki//kRe1z+tLGcpnD1jrouitsQddDw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 h1:KqIfN9kpkKkcBqBbNpNGTIrXO6ExTUvFKvXkC+YAzVo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28/go.mod h1:uxtQiKvLtNS4iXVsH2McVD/ls8FKN/uUhe1hGxPjrw0= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 h1:clHU5fm//kWS1C2HgtgWxfQbFbx4b6rx+5jzhgX9HrI= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= @@ -52,8 +52,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6f github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= -github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= -github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8= +github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= diff --git a/core/internal/llmtests/account.go b/core/internal/llmtests/account.go index c78ffbd508..3dfc797b3b 100644 --- a/core/internal/llmtests/account.go +++ b/core/internal/llmtests/account.go @@ -131,8 +131,10 @@ type ComprehensiveTestConfig struct { VideoGenerationModel string // Model for video generation ExternalTTSProvider schemas.ModelProvider // External TTS provider to use for testing ExternalTTSModel string // External TTS model to use for testing - BatchExtraParams map[string]interface{} // Extra params for batch operations (e.g., role_arn, output_s3_uri for Bedrock) - FileExtraParams map[string]interface{} // Extra params for file operations (e.g., s3_bucket for Bedrock) + BatchExtraParams map[string]interface{} // Extra params for batch operations (e.g., role_arn, output_s3_uri for Bedrock) + BatchOutputFolder *schemas.BatchOutputFolder // Typed batch output location (e.g., GCS gs:// prefix for Vertex) + FileExtraParams map[string]interface{} // Extra params for file operations (e.g., s3_bucket for Bedrock) + FileStorageConfig *schemas.FileStorageConfig // Typed storage config for file operations (e.g., GCS bucket for Vertex) DisableParallelFor []string // Test scenarios to disable parallel execution for (e.g., "Transcription" for rate-limited APIs) ExpectRawRequestResponse bool // When true, validate rawRequest/rawResponse in ExtraFields PassthroughModel string // Model for passthrough API tests; defaults to ChatModel when empty @@ -241,12 +243,12 @@ func (account *ComprehensiveTestAccount) GetKeysForProvider(ctx context.Context, { Models: []string{"*"}, Weight: 1.0, - Aliases: map[string]string{ - "claude-3.7-sonnet": "us.anthropic.claude-3-7-sonnet-20250219-v1:0", - "claude-4-sonnet": "global.anthropic.claude-sonnet-4-20250514-v1:0", - "claude-4.5-sonnet": "global.anthropic.claude-sonnet-4-5-20250929-v1:0", - "claude-4.6-sonnet": "global.anthropic.claude-sonnet-4-6", - "claude-4.5-haiku": "global.anthropic.claude-haiku-4-5-20251001-v1:0", + Aliases: schemas.KeyAliases{ + "claude-3.7-sonnet": {ModelID: "us.anthropic.claude-3-7-sonnet-20250219-v1:0"}, + "claude-4-sonnet": {ModelID: "global.anthropic.claude-sonnet-4-20250514-v1:0"}, + "claude-4.5-sonnet": {ModelID: "global.anthropic.claude-sonnet-4-5-20250929-v1:0"}, + "claude-4.6-sonnet": {ModelID: "global.anthropic.claude-sonnet-4-6"}, + "claude-4.5-haiku": {ModelID: "global.anthropic.claude-haiku-4-5-20251001-v1:0"}, }, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("env.AWS_ACCESS_KEY_ID"), @@ -259,13 +261,13 @@ func (account *ComprehensiveTestAccount) GetKeysForProvider(ctx context.Context, { Models: []string{"*"}, Weight: 1.0, - Aliases: map[string]string{ - "claude-3.5-sonnet": "anthropic.claude-3-5-sonnet-20240620-v1:0", - "claude-3.7-sonnet": "us.anthropic.claude-3-7-sonnet-20250219-v1:0", - "claude-4-sonnet": "global.anthropic.claude-sonnet-4-20250514-v1:0", - "claude-4.5-sonnet": "global.anthropic.claude-sonnet-4-5-20250929-v1:0", - "claude-4.6-sonnet": "global.anthropic.claude-sonnet-4-6", - "claude-4.5-haiku": "global.anthropic.claude-haiku-4-5-20251001-v1:0", + Aliases: schemas.KeyAliases{ + "claude-3.5-sonnet": {ModelID: "anthropic.claude-3-5-sonnet-20240620-v1:0"}, + "claude-3.7-sonnet": {ModelID: "us.anthropic.claude-3-7-sonnet-20250219-v1:0"}, + "claude-4-sonnet": {ModelID: "global.anthropic.claude-sonnet-4-20250514-v1:0"}, + "claude-4.5-sonnet": {ModelID: "global.anthropic.claude-sonnet-4-5-20250929-v1:0"}, + "claude-4.6-sonnet": {ModelID: "global.anthropic.claude-sonnet-4-6"}, + "claude-4.5-haiku": {ModelID: "global.anthropic.claude-haiku-4-5-20251001-v1:0"}, }, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("env.AWS_ACCESS_KEY_ID"), @@ -303,13 +305,13 @@ func (account *ComprehensiveTestAccount) GetKeysForProvider(ctx context.Context, Models: []string{"*"}, Weight: 1.0, Aliases: schemas.KeyAliases{ - "gpt-4o": "gpt-4o", - "gpt-4o-backup": "gpt-4o-3", - "claude-opus-4-5": "claude-opus-4-5", - "o1": "o1", - "gpt-image-1": "gpt-image-1", - "text-embedding-ada-002": "text-embedding-ada-002", - "sora-2": "sora-2", + "gpt-4o": {ModelID: "gpt-4o"}, + "gpt-4o-backup": {ModelID: "gpt-4o-3"}, + "claude-opus-4-5": {ModelID: "claude-opus-4-5"}, + "o1": {ModelID: "o1"}, + "gpt-image-1": {ModelID: "gpt-image-1"}, + "text-embedding-ada-002": {ModelID: "text-embedding-ada-002"}, + "sora-2": {ModelID: "sora-2"}, }, AzureKeyConfig: &schemas.AzureKeyConfig{ Endpoint: *schemas.NewEnvVar("env.AZURE_ENDPOINT"), @@ -324,10 +326,10 @@ func (account *ComprehensiveTestAccount) GetKeysForProvider(ctx context.Context, Models: []string{"*"}, Weight: 1.0, Aliases: schemas.KeyAliases{ - "whisper": "whisper", - "whisper-1": "whisper", - "gpt-4o-mini-tts": "gpt-4o-mini-tts", - "gpt-4o-mini-audio-preview": "gpt-4o-mini-audio-preview", + "whisper": {ModelID: "whisper"}, + "whisper-1": {ModelID: "whisper"}, + "gpt-4o-mini-tts": {ModelID: "gpt-4o-mini-tts"}, + "gpt-4o-mini-audio-preview": {ModelID: "gpt-4o-mini-audio-preview"}, }, AzureKeyConfig: &schemas.AzureKeyConfig{ Endpoint: *schemas.NewEnvVar("env.AZURE_ENDPOINT"), @@ -365,9 +367,9 @@ func (account *ComprehensiveTestAccount) GetKeysForProvider(ctx context.Context, Models: []string{"claude-sonnet-4-5", "claude-4.5-haiku", "claude-opus-4-5"}, Weight: 1.0, Aliases: schemas.KeyAliases{ - "claude-sonnet-4-5": "claude-sonnet-4-5", - "claude-4.5-haiku": "claude-haiku-4-5@20251001", - "claude-opus-4-5": "claude-opus-4-5", + "claude-sonnet-4-5": {ModelID: "claude-sonnet-4-5"}, + "claude-4.5-haiku": {ModelID: "claude-haiku-4-5@20251001"}, + "claude-opus-4-5": {ModelID: "claude-opus-4-5"}, }, VertexKeyConfig: &schemas.VertexKeyConfig{ ProjectID: *schemas.NewEnvVar("env.VERTEX_PROJECT_ID"), diff --git a/core/internal/llmtests/batch.go b/core/internal/llmtests/batch.go index a6717ebe37..6c118f1fcc 100644 --- a/core/internal/llmtests/batch.go +++ b/core/internal/llmtests/batch.go @@ -84,6 +84,7 @@ func RunBatchCreateTest(t *testing.T, client *bifrost.Bifrost, ctx context.Conte }, CompletionWindow: "24h", ExtraParams: testConfig.BatchExtraParams, + OutputFolder: testConfig.BatchOutputFolder, } bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline) return client.BatchCreateRequest(bfCtx, request) @@ -231,6 +232,7 @@ func RunBatchRetrieveTest(t *testing.T, client *bifrost.Bifrost, ctx context.Con }, CompletionWindow: "24h", ExtraParams: testConfig.BatchExtraParams, + OutputFolder: testConfig.BatchOutputFolder, } bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline) return client.BatchCreateRequest(bfCtx, createRequest) @@ -358,6 +360,7 @@ func RunBatchCancelTest(t *testing.T, client *bifrost.Bifrost, ctx context.Conte }, CompletionWindow: "24h", ExtraParams: testConfig.BatchExtraParams, + OutputFolder: testConfig.BatchOutputFolder, } bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline) return client.BatchCreateRequest(bfCtx, createRequest) @@ -603,9 +606,10 @@ func RunFileUploadTest(t *testing.T, client *bifrost.Bifrost, ctx context.Contex request := &schemas.BifrostFileUploadRequest{ Provider: testConfig.Provider, File: fileContent, - Filename: "test_batch.jsonl", - Purpose: "batch", - ExtraParams: testConfig.FileExtraParams, + Filename: "test_batch.jsonl", + Purpose: "batch", + ExtraParams: testConfig.FileExtraParams, + StorageConfig: testConfig.FileStorageConfig, } bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline) return client.FileUploadRequest(bfCtx, request) @@ -672,9 +676,10 @@ func RunFileListTest(t *testing.T, client *bifrost.Bifrost, ctx context.Context, response, err := WithFileListTestRetry(t, fileListRetryConfig, retryContext, expectations, "FileList", func() (*schemas.BifrostFileListResponse, *schemas.BifrostError) { request := &schemas.BifrostFileListRequest{ - Provider: testConfig.Provider, - Limit: 10, - ExtraParams: testConfig.FileExtraParams, + Provider: testConfig.Provider, + Limit: 10, + ExtraParams: testConfig.FileExtraParams, + StorageConfig: testConfig.FileStorageConfig, } bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline) return client.FileListRequest(bfCtx, request) @@ -742,9 +747,10 @@ func RunFileRetrieveTest(t *testing.T, client *bifrost.Bifrost, ctx context.Cont uploadRequest := &schemas.BifrostFileUploadRequest{ Provider: testConfig.Provider, File: fileContent, - Filename: "test_retrieve.jsonl", - Purpose: "batch", - ExtraParams: testConfig.FileExtraParams, + Filename: "test_retrieve.jsonl", + Purpose: "batch", + ExtraParams: testConfig.FileExtraParams, + StorageConfig: testConfig.FileStorageConfig, } bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline) return client.FileUploadRequest(bfCtx, uploadRequest) @@ -860,9 +866,10 @@ func RunFileDeleteTest(t *testing.T, client *bifrost.Bifrost, ctx context.Contex uploadRequest := &schemas.BifrostFileUploadRequest{ Provider: testConfig.Provider, File: fileContent, - Filename: "test_delete.jsonl", - Purpose: "batch", - ExtraParams: testConfig.FileExtraParams, + Filename: "test_delete.jsonl", + Purpose: "batch", + ExtraParams: testConfig.FileExtraParams, + StorageConfig: testConfig.FileStorageConfig, } bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline) return client.FileUploadRequest(bfCtx, uploadRequest) @@ -978,9 +985,10 @@ func RunFileContentTest(t *testing.T, client *bifrost.Bifrost, ctx context.Conte uploadRequest := &schemas.BifrostFileUploadRequest{ Provider: testConfig.Provider, File: originalContent, - Filename: "test_content.jsonl", - Purpose: "batch", - ExtraParams: testConfig.FileExtraParams, + Filename: "test_content.jsonl", + Purpose: "batch", + ExtraParams: testConfig.FileExtraParams, + StorageConfig: testConfig.FileStorageConfig, } bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline) return client.FileUploadRequest(bfCtx, uploadRequest) @@ -1117,9 +1125,10 @@ func RunFileAndBatchIntegrationTest(t *testing.T, client *bifrost.Bifrost, ctx c uploadRequest := &schemas.BifrostFileUploadRequest{ Provider: testConfig.Provider, File: fileContent, - Filename: "integration_test_batch.jsonl", - Purpose: "batch", - ExtraParams: testConfig.FileExtraParams, + Filename: "integration_test_batch.jsonl", + Purpose: "batch", + ExtraParams: testConfig.FileExtraParams, + StorageConfig: testConfig.FileStorageConfig, } bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline) @@ -1148,6 +1157,7 @@ func RunFileAndBatchIntegrationTest(t *testing.T, client *bifrost.Bifrost, ctx c Endpoint: schemas.BatchEndpointChatCompletions, CompletionWindow: "24h", ExtraParams: testConfig.BatchExtraParams, + OutputFolder: testConfig.BatchOutputFolder, } bfCtx2 := schemas.NewBifrostContext(ctx, schemas.NoDeadline) diff --git a/core/mcp/healthmonitor.go b/core/mcp/healthmonitor.go index 2f09fabff3..cadf1d7156 100644 --- a/core/mcp/healthmonitor.go +++ b/core/mcp/healthmonitor.go @@ -174,9 +174,15 @@ func (chm *ClientHealthMonitor) performHealthCheck() { err = fmt.Errorf("no active connection") } else { // Perform health check with timeout - ctx, cancel := context.WithTimeout(context.Background(), chm.timeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), chm.timeout) defer cancel() + // Mark the request as bifrost-generated for health checks so plugins/hooks can + // distinguish these internal pings/list_tools probes from caller-initiated requests. + // runPingWithHooks / runListToolsWithHooks wrap this ctx, so the marker propagates. + ctx := schemas.NewBifrostContext(timeoutCtx, schemas.NoDeadline) + ctx.SetValue(schemas.BifrostContextKeyMCPHealthCheckRequest, true) + if chm.isPingAvailable { err = chm.runPingWithHooks(ctx, conn, clientName) } else { diff --git a/core/providers/anthropic/chat.go b/core/providers/anthropic/chat.go index d9df31f9c3..bac88fe1a7 100644 --- a/core/providers/anthropic/chat.go +++ b/core/providers/anthropic/chat.go @@ -255,8 +255,9 @@ func ToAnthropicChatRequest(ctx *schemas.BifrostContext, bifrostReq *schemas.Bif anthropicReq.MaxTokens = *bifrostReq.Params.MaxCompletionTokens } - // Opus 4.7+ rejects temperature, top_p, and top_k with a 400 error. - if !IsOpus47Plus(bifrostReq.Model) { + // Opus 4.7+ and the Fable/Mythos family reject temperature, top_p, and + // top_k with a 400 error. + if !IsAdaptiveOnlyThinkingModel(bifrostReq.Model) { // Anthropic doesn't allow both temperature and top_p to be specified. // If both are present, prefer temperature (more commonly used). if bifrostReq.Params.Temperature != nil { @@ -268,14 +269,14 @@ func ToAnthropicChatRequest(ctx *schemas.BifrostContext, bifrostReq *schemas.Bif anthropicReq.StopSequences = bifrostReq.Params.Stop // TopK — prefer the promoted neutral field; fall back to ExtraParams. - // Opus 4.7+ rejects top_k with a 400 error. + // Opus 4.7+ and the Fable/Mythos family reject top_k with a 400 error. if bifrostReq.Params.TopK != nil { - if !IsOpus47Plus(bifrostReq.Model) { + if !IsAdaptiveOnlyThinkingModel(bifrostReq.Model) { anthropicReq.TopK = bifrostReq.Params.TopK } } else if topK, ok := schemas.SafeExtractIntPointer(bifrostReq.Params.ExtraParams["top_k"]); ok { delete(anthropicReq.ExtraParams, "top_k") - if !IsOpus47Plus(bifrostReq.Model) { + if !IsAdaptiveOnlyThinkingModel(bifrostReq.Model) { anthropicReq.TopK = topK } } @@ -476,8 +477,8 @@ func ToAnthropicChatRequest(ctx *schemas.BifrostContext, bifrostReq *schemas.Bif // Convert reasoning if bifrostReq.Params.Reasoning != nil { if bifrostReq.Params.Reasoning.MaxTokens != nil { - if IsOpus47Plus(bifrostReq.Model) { - // Opus 4.7+: budget_tokens removed; adaptive thinking is the only thinking-on mode. + if IsAdaptiveOnlyThinkingModel(bifrostReq.Model) { + // Opus 4.7+ and Fable/Mythos: budget_tokens removed; adaptive thinking is the only thinking-on mode. anthropicReq.Thinking = &AnthropicThinking{Type: "adaptive"} } else { budgetTokens := *bifrostReq.Params.Reasoning.MaxTokens @@ -522,7 +523,11 @@ func ToAnthropicChatRequest(ctx *schemas.BifrostContext, bifrostReq *schemas.Bif BudgetTokens: schemas.Ptr(budgetTokens), } } - } else { + } else if !IsFableFamily(bifrostReq.Model) { + // Fable/Mythos reject thinking:{type:"disabled"} with a 400 — + // adaptive thinking is always on and cannot be disabled. Omit + // the thinking param entirely for that family; all other models + // take the explicit disabled path. anthropicReq.Thinking = &AnthropicThinking{ Type: "disabled", } @@ -534,12 +539,13 @@ func ToAnthropicChatRequest(ctx *schemas.BifrostContext, bifrostReq *schemas.Bif // nothing to display", per the extended-thinking doc). We attach // on non-disabled modes and let the upstream provider enforce // model-level support. - // Opus 4.7+ omits reasoning text by default; default to "summarized" - // so the text is visible unless the caller explicitly requests "omitted". + // Opus 4.7+ and the Fable/Mythos family omit reasoning text by + // default; default to "summarized" so the text is visible unless + // the caller explicitly requests "omitted". if anthropicReq.Thinking != nil && anthropicReq.Thinking.Type != "disabled" { if bifrostReq.Params.Reasoning.Display != nil { anthropicReq.Thinking.Display = bifrostReq.Params.Reasoning.Display - } else if IsOpus47Plus(bifrostReq.Model) { + } else if IsAdaptiveOnlyThinkingModel(bifrostReq.Model) { anthropicReq.Thinking.Display = schemas.Ptr("summarized") } } @@ -978,6 +984,10 @@ func (response *AnthropicMessageResponse) ToBifrostChatResponse(ctx *schemas.Bif mapped := MapAnthropicServiceTierToBifrost(*response.Usage.ServiceTier) bifrostResponse.ServiceTier = &mapped } + // Forward the speed actually served (fast mode) — drives fast-mode billing. + if response.Usage.Speed != nil { + bifrostResponse.Speed = response.Usage.Speed + } } return bifrostResponse @@ -1025,6 +1035,10 @@ func ToAnthropicChatResponse(bifrostResp *schemas.BifrostChatResponse) *Anthropi mapped := MapBifrostServiceTierToAnthropicResponse(*bifrostResp.ServiceTier) anthropicResp.Usage.ServiceTier = &mapped } + // Forward the speed actually served (fast mode) + if bifrostResp.Speed != nil { + anthropicResp.Usage.Speed = bifrostResp.Speed + } } // Convert choices to content diff --git a/core/providers/anthropic/models.go b/core/providers/anthropic/models.go index 3815a0244b..a09174c661 100644 --- a/core/providers/anthropic/models.go +++ b/core/providers/anthropic/models.go @@ -8,7 +8,7 @@ import ( "github.com/maximhq/bifrost/core/schemas" ) -func (response *AnthropicListModelsResponse) ToBifrostListModelsResponse(providerKey schemas.ModelProvider, allowedModels schemas.WhiteList, blacklistedModels schemas.BlackList, aliases map[string]string, unfiltered bool) *schemas.BifrostListModelsResponse { +func (response *AnthropicListModelsResponse) ToBifrostListModelsResponse(providerKey schemas.ModelProvider, allowedModels schemas.WhiteList, blacklistedModels schemas.BlackList, aliases schemas.KeyAliases, unfiltered bool) *schemas.BifrostListModelsResponse { if response == nil { return nil } diff --git a/core/providers/anthropic/passthrough_usage.go b/core/providers/anthropic/passthrough_usage.go index 1996e4521e..bddf96dd42 100644 --- a/core/providers/anthropic/passthrough_usage.go +++ b/core/providers/anthropic/passthrough_usage.go @@ -74,6 +74,9 @@ func buildAnthropicPassthroughUsage(au *AnthropicUsage) *schemas.BifrostPassthro t := MapAnthropicServiceTierToBifrost(*au.ServiceTier) u.ServiceTier = &t } + if au.Speed != nil { + u.Speed = au.Speed + } return u } @@ -136,6 +139,9 @@ func (a *AnthropicPassthroughStreamUsage) ObserveEvent(event []byte) *schemas.Bi if u.ServiceTier != nil { c.ServiceTier = u.ServiceTier } + if u.Speed != nil { + c.Speed = u.Speed + } return a.usage() } diff --git a/core/providers/anthropic/responses.go b/core/providers/anthropic/responses.go index 85ddf8f8df..f9d399cd12 100644 --- a/core/providers/anthropic/responses.go +++ b/core/providers/anthropic/responses.go @@ -2187,7 +2187,7 @@ func ToAnthropicResponsesStreamResponse(ctx *schemas.BifrostContext, bifrostResp // ToBifrostResponsesRequest converts an Anthropic message request to Bifrost format func (req *AnthropicMessageRequest) ToBifrostResponsesRequest(ctx *schemas.BifrostContext) *schemas.BifrostResponsesRequest { - provider, model := schemas.ParseModelString(req.Model, providerUtils.CheckAndSetDefaultProvider(ctx, schemas.Anthropic)) + provider, model := schemas.ParseModelString(req.Model, "") bifrostReq := &schemas.BifrostResponsesRequest{ Provider: provider, @@ -2402,8 +2402,9 @@ func ToAnthropicResponsesRequest(ctx *schemas.BifrostContext, bifrostReq *schema if bifrostReq.Params.MaxOutputTokens != nil { anthropicReq.MaxTokens = *bifrostReq.Params.MaxOutputTokens } - // Opus 4.7+ rejects temperature, top_p, and top_k with a 400 error. - if !IsOpus47Plus(bifrostReq.Model) { + // Opus 4.7+ and the Fable/Mythos family reject temperature, top_p, and + // top_k with a 400 error. + if !IsAdaptiveOnlyThinkingModel(bifrostReq.Model) { // Anthropic doesn't allow both temperature and top_p to be specified. // If both are present, prefer temperature (more commonly used). if bifrostReq.Params.Temperature != nil { @@ -2474,9 +2475,14 @@ func ToAnthropicResponsesRequest(ctx *schemas.BifrostContext, bifrostReq *schema } if bifrostReq.Params.Reasoning != nil { if bifrostReq.Params.Reasoning.MaxTokens != nil { - if IsOpus47Plus(bifrostReq.Model) { - // Opus 4.7+: budget_tokens removed; adaptive thinking is the only thinking-on mode. + if IsAdaptiveOnlyThinkingModel(bifrostReq.Model) { + // Opus 4.7+ and Fable/Mythos: budget_tokens removed; adaptive thinking is the only thinking-on mode. anthropicReq.Thinking = &AnthropicThinking{Type: "adaptive"} + // Preserve a co-present effort — these models support + // output_config.effort, and the budget is otherwise dropped. + if bifrostReq.Params.Reasoning.Effort != nil && *bifrostReq.Params.Reasoning.Effort != "none" { + setEffortOnOutputConfig(anthropicReq, MapBifrostEffortToAnthropic(*bifrostReq.Params.Reasoning.Effort)) + } } else { budgetTokens := *bifrostReq.Params.Reasoning.MaxTokens if *bifrostReq.Params.Reasoning.MaxTokens == -1 { @@ -2523,7 +2529,11 @@ func ToAnthropicResponsesRequest(ctx *schemas.BifrostContext, bifrostReq *schema BudgetTokens: schemas.Ptr(budgetTokens), } } - } else { + } else if !IsFableFamily(bifrostReq.Model) { + // Fable/Mythos reject thinking:{type:"disabled"} with a 400 — + // adaptive thinking is always on and cannot be disabled. Omit + // the thinking param entirely for that family; all other + // models take the explicit disabled path. anthropicReq.Thinking = &AnthropicThinking{ Type: "disabled", } @@ -2538,7 +2548,7 @@ func ToAnthropicResponsesRequest(ctx *schemas.BifrostContext, bifrostReq *schema } else { anthropicReq.Thinking.Display = schemas.Ptr("summarized") } - } else if IsOpus47Plus(bifrostReq.Model) { + } else if IsAdaptiveOnlyThinkingModel(bifrostReq.Model) { anthropicReq.Thinking.Display = schemas.Ptr("summarized") } } @@ -2579,13 +2589,15 @@ func ToAnthropicResponsesRequest(ctx *schemas.BifrostContext, bifrostReq *schema topK, ok := schemas.SafeExtractIntPointer(bifrostReq.Params.ExtraParams["top_k"]) if ok { delete(anthropicReq.ExtraParams, "top_k") - if !IsOpus47Plus(bifrostReq.Model) { + if !IsAdaptiveOnlyThinkingModel(bifrostReq.Model) { anthropicReq.TopK = topK } } if speed, ok := schemas.SafeExtractStringPointer(bifrostReq.Params.ExtraParams["speed"]); ok { delete(anthropicReq.ExtraParams, "speed") - anthropicReq.Speed = speed + if SupportsFastMode(bifrostReq.Model) { + anthropicReq.Speed = speed + } } if stop, ok := schemas.SafeExtractStringSlice(bifrostReq.Params.ExtraParams["stop"]); ok { delete(anthropicReq.ExtraParams, "stop") @@ -2854,6 +2866,11 @@ func (response *AnthropicMessageResponse) ToBifrostResponsesResponse(ctx *schema bifrostResp.ServiceTier = &mapped } + // Forward the speed actually served (fast mode) — drives fast-mode billing. + if response.Usage != nil && response.Usage.Speed != nil { + bifrostResp.Speed = response.Usage.Speed + } + return bifrostResp } @@ -2916,6 +2933,13 @@ func ToAnthropicResponsesResponse(ctx *schemas.BifrostContext, bifrostResp *sche anthropicResp.Usage.ServiceTier = &mapped } + if bifrostResp.Speed != nil { + if anthropicResp.Usage == nil { + anthropicResp.Usage = &AnthropicUsage{} + } + anthropicResp.Usage.Speed = bifrostResp.Speed + } + return anthropicResp } @@ -5286,7 +5310,7 @@ func convertBifrostToolToAnthropic(model string, tool *schemas.ResponsesTool, pr // Dynamic filtering (web_search_20260209) available on Anthropic + Azure for Opus 4.6+. features, ok := ProviderFeatures[provider] if ok && features.WebSearchDynamic && - (strings.Contains(model, "4.6") || strings.Contains(model, "4-6") || IsOpus47Plus(model)) { + (strings.Contains(model, "4.6") || strings.Contains(model, "4-6") || IsOpus47Plus(model) || IsFableFamily(model)) { webSearchType = AnthropicToolTypeWebSearch20260209 } anthropicTool := &AnthropicTool{ diff --git a/core/providers/anthropic/text.go b/core/providers/anthropic/text.go index 39a700499b..df6f488d7c 100644 --- a/core/providers/anthropic/text.go +++ b/core/providers/anthropic/text.go @@ -54,7 +54,7 @@ func (req *AnthropicTextRequest) ToBifrostTextCompletionRequest(ctx *schemas.Bif return nil } - provider, model := schemas.ParseModelString(req.Model, providerUtils.CheckAndSetDefaultProvider(ctx, schemas.Anthropic)) + provider, model := schemas.ParseModelString(req.Model, "") bifrostReq := &schemas.BifrostTextCompletionRequest{ Provider: provider, diff --git a/core/providers/anthropic/types.go b/core/providers/anthropic/types.go index 7c196d2d51..fc84fdd9af 100644 --- a/core/providers/anthropic/types.go +++ b/core/providers/anthropic/types.go @@ -1429,6 +1429,7 @@ type AnthropicUsage struct { OutputTokens int `json:"output_tokens"` ServerToolUse *AnthropicServerToolUseUsage `json:"server_tool_use,omitempty"` // Server tool use statistics (e.g., web search) ServiceTier *string `json:"service_tier,omitempty"` // "standard", "priority", or "batch" + Speed *string `json:"speed,omitempty"` // "fast" or "standard" — which speed was actually served (fast mode research preview) InferenceGeo *string `json:"inference_geo,omitempty"` // the geographic region for inference processing. If not specified, the workspace's default_inference_geo is used. Iterations []AnthropicUsage `json:"iterations,omitempty"` // Iterations statistics } diff --git a/core/providers/anthropic/utils.go b/core/providers/anthropic/utils.go index 7c8dfd12e4..d161275fe7 100644 --- a/core/providers/anthropic/utils.go +++ b/core/providers/anthropic/utils.go @@ -673,6 +673,35 @@ func IsOpus47Plus(model string) bool { strings.Contains(model, "4-8") || strings.Contains(model, "4.8") } +// IsFableFamily returns true for Claude Fable / Mythos models (Fable 5, +// Mythos 5, Mythos Preview). These share Opus 4.7+'s request surface +// (adaptive-only thinking, temperature/top_p/top_k removed) AND additionally +// reject thinking:{type:"disabled"} — adaptive thinking is always on and must +// not be explicitly disabled. The thinking param should be omitted entirely +// rather than sent as disabled. +// +// Sources: +// - https://platform.claude.com/docs/en/build-with-claude/effort +// ("Claude Fable 5 and Claude Mythos 5 use adaptive thinking, which is +// always on ... thinking: {type: "disabled"} is rejected.") +// - https://platform.claude.com/docs/en/build-with-claude/fast-mode +// (fast mode is NOT supported on Fable — Opus 4.6/4.7/4.8 only; this is why +// Fable is kept separate from IsOpus47Plus, which gates SupportsFastMode). +func IsFableFamily(model string) bool { + m := strings.ToLower(model) + return strings.Contains(m, "fable") || strings.Contains(m, "mythos") +} + +// IsAdaptiveOnlyThinkingModel returns true for models where budget_tokens +// extended thinking is removed (adaptive is the only thinking-on mode) and +// temperature/top_p/top_k are rejected with a 400. Covers Opus 4.7+ and the +// Fable/Mythos family. Use this — not IsOpus47Plus — for the thinking and +// sampling-parameter gates so Fable is handled correctly. (Fast mode is gated +// on IsOpus47Plus instead, since Fable does not support speed:"fast".) +func IsAdaptiveOnlyThinkingModel(model string) bool { + return IsOpus47Plus(model) || IsFableFamily(model) +} + // SupportsNativeEffort returns true if the model supports Anthropic's native output_config.effort parameter. // Currently supported on Claude Opus 4.5 and Opus 4.6. func SupportsNativeEffort(model string) bool { @@ -685,9 +714,9 @@ func SupportsNativeEffort(model string) bool { } // SupportsEffortParameter returns true if the model accepts the -// output_config.effort parameter. Supported models: Claude Mythos Preview, -// Opus 4.8, Opus 4.7, Opus 4.6, Sonnet 4.6, and Opus 4.5. -// All other models reject effort with a 400: +// output_config.effort parameter. Supported models: Claude Fable 5, +// Claude Mythos 5, Claude Mythos Preview, Opus 4.8, Opus 4.7, Opus 4.6, +// Sonnet 4.6, and Opus 4.5. All other models reject effort with a 400: // // "This model does not support the effort parameter." // @@ -695,9 +724,11 @@ func SupportsNativeEffort(model string) bool { // support the effort knob without supporting adaptive thinking (Opus 4.5), // and adaptive thinking is a distinct surface (thinking.type:"adaptive") // from effort. Future models may shift either flag independently. +// +// Source: https://platform.claude.com/docs/en/build-with-claude/effort func SupportsEffortParameter(model string) bool { m := strings.ToLower(model) - if strings.Contains(m, "mythos") { + if IsFableFamily(m) { return true } if strings.Contains(m, "haiku") { @@ -743,7 +774,9 @@ func appendToSystemContent(existing *AnthropicContent, newContent AnthropicConte // SupportsMidConversationSystem returns true if the provider+model combination // supports role:"system" entries inside the messages array (mid-conversation // system messages). Available on the Anthropic API only — not on Bedrock or -// Vertex — and only for Claude Opus 4.8+. No beta header is required. +// Vertex. Supported on Claude Opus 4.8+ and the Claude Fable/Mythos family +// (Fable post-dates Opus 4.8; the public doc lists Opus 4.8 but Fable supports +// it as well). No beta header is required. // // Source: https://platform.claude.com/docs/en/build-with-claude/mid-conversation-system-messages func SupportsMidConversationSystem(provider schemas.ModelProvider, model string) bool { @@ -751,6 +784,9 @@ func SupportsMidConversationSystem(provider schemas.ModelProvider, model string) return false } m := strings.ToLower(model) + if IsFableFamily(m) { + return true + } return strings.Contains(m, "opus") && (strings.Contains(m, "4-8") || strings.Contains(m, "4.8")) } @@ -771,11 +807,13 @@ func SupportsFastMode(model string) bool { } // SupportsAdaptiveThinking returns true if the model supports thinking.type: "adaptive". -// Currently supported on Claude Opus 4.6, Claude Sonnet 4.6, and Claude Opus 4.7+. -// On Opus 4.7+ adaptive is the only thinking-on mode; on Opus 4.6 and Sonnet 4.6 it -// coexists with the deprecated budget_tokens-based extended thinking. +// Currently supported on Claude Opus 4.6, Claude Sonnet 4.6, Claude Opus 4.7+, and +// the Claude Fable/Mythos family. On Opus 4.7+ and Fable/Mythos adaptive is the only +// thinking-on mode; on Opus 4.6 and Sonnet 4.6 it coexists with the deprecated +// budget_tokens-based extended thinking. On Fable/Mythos adaptive is always on and +// thinking:{type:"disabled"} is rejected (see IsFableFamily). func SupportsAdaptiveThinking(model string) bool { - if IsOpus47Plus(model) { + if IsOpus47Plus(model) || IsFableFamily(model) { return true } model = strings.ToLower(model) @@ -802,8 +840,8 @@ const ( // - Which `name` literal Anthropic's Pydantic validator demands for text_editor. func ComputerUseGeneration(model string) string { m := strings.ToLower(model) - // Opus 4.7+ falls into the new generation. - if IsOpus47Plus(m) { + // Opus 4.7+ and the Fable/Mythos family use the new generation. + if IsOpus47Plus(m) || IsFableFamily(m) { return ComputerUseGen20251124 } // Opus 4.6 / Sonnet 4.6 / Opus 4.5 also use the new generation. @@ -832,7 +870,7 @@ func ComputerUseGeneration(model string) string { // - Sonnet 4.5 / 4.6 (sonnet-4-5 differs from ComputerUseGeneration which keeps it old-gen) func TextEditorGeneration(model string) string { m := strings.ToLower(model) - if IsOpus47Plus(m) { + if IsOpus47Plus(m) || IsFableFamily(m) { return ComputerUseGen20251124 } if strings.Contains(m, "opus") { @@ -1041,7 +1079,7 @@ func AddMissingBetaHeadersToContext(ctx *schemas.BifrostContext, req *AnthropicM // Check for fast mode. Only add the beta header when both the provider // supports fast mode AND the model does (Opus 4.6 only per // SupportsFastMode); otherwise sending the header guarantees a 400. - if req.Speed != nil && *req.Speed == "fast" { + if req.Speed != nil { if (!hasProvider || features.FastMode) && SupportsFastMode(req.Model) { headers = appendUniqueHeader(headers, AnthropicFastModeBetaHeader) } diff --git a/core/providers/anthropic/utils_test.go b/core/providers/anthropic/utils_test.go index af6c20150a..ea5f5beba4 100644 --- a/core/providers/anthropic/utils_test.go +++ b/core/providers/anthropic/utils_test.go @@ -2225,6 +2225,11 @@ func TestSupportsAdaptiveThinking(t *testing.T) { {"claude-opus-4.6-20250514", true}, {"claude-sonnet-4-6-20250514", true}, {"claude-sonnet-4.6-20250514", true}, + // Fable/Mythos family: adaptive thinking is always on. + {"claude-fable-5", true}, + {"claude-mythos-5", true}, + {"claude-mythos-preview", true}, + {"global.anthropic.claude-fable-5", true}, {"claude-opus-4-5-20241022", false}, {"claude-sonnet-4-5-20241022", false}, {"claude-haiku-4-6-20250514", false}, // haiku does not support adaptive @@ -2243,6 +2248,70 @@ func TestSupportsAdaptiveThinking(t *testing.T) { } } +// TestIsFableFamily pins the Fable/Mythos family predicate. These models share +// Opus 4.7+'s adaptive-only / no-sampling surface and additionally reject +// thinking:{type:"disabled"}. +func TestIsFableFamily(t *testing.T) { + tests := []struct { + model string + expected bool + }{ + {"claude-fable-5", true}, + {"claude-mythos-5", true}, + {"claude-mythos-preview", true}, + {"global.anthropic.claude-fable-5", true}, + {"anthropic.claude-mythos-5-v1", true}, + // Not Fable/Mythos. + {"claude-opus-4-8", false}, + {"claude-opus-4-7", false}, + {"claude-sonnet-4-6", false}, + {"claude-haiku-4-5", false}, + {"", false}, + {"some-non-claude-model", false}, + } + + for _, tt := range tests { + t.Run(tt.model, func(t *testing.T) { + if got := IsFableFamily(tt.model); got != tt.expected { + t.Errorf("IsFableFamily(%q) = %v, want %v", tt.model, got, tt.expected) + } + }) + } +} + +// TestIsAdaptiveOnlyThinkingModel covers the union gate used for the thinking +// and sampling-parameter surfaces: Opus 4.7+ OR the Fable/Mythos family. +func TestIsAdaptiveOnlyThinkingModel(t *testing.T) { + tests := []struct { + model string + expected bool + }{ + // Opus 4.7+. + {"claude-opus-4-8", true}, + {"claude-opus-4-7", true}, + {"claude-opus-4.8-20260601", true}, + // Fable/Mythos. + {"claude-fable-5", true}, + {"claude-mythos-5", true}, + {"claude-mythos-preview", true}, + // Adaptive-capable but NOT adaptive-only (budget_tokens still accepted). + {"claude-opus-4-6", false}, + {"claude-sonnet-4-6", false}, + // Other. + {"claude-opus-4-5", false}, + {"claude-haiku-4-5", false}, + {"", false}, + } + + for _, tt := range tests { + t.Run(tt.model, func(t *testing.T) { + if got := IsAdaptiveOnlyThinkingModel(tt.model); got != tt.expected { + t.Errorf("IsAdaptiveOnlyThinkingModel(%q) = %v, want %v", tt.model, got, tt.expected) + } + }) + } +} + // TestSupportsFastMode pins the helper against Anthropic's fast-mode docs. // TestSupportsMidConversationSystem pins the helper against Anthropic docs: // available on the Anthropic API only, Opus 4.8+ only, no beta header required. @@ -2265,6 +2334,13 @@ func TestSupportsMidConversationSystem(t *testing.T) { // Not supported: other model families. {schemas.Anthropic, "claude-sonnet-4-8", false}, {schemas.Anthropic, "claude-haiku-4-8", false}, + // Supported: Fable/Mythos family (Anthropic provider). Fable post-dates + // Opus 4.8 and supports mid-conversation system messages. + {schemas.Anthropic, "claude-fable-5", true}, + {schemas.Anthropic, "claude-mythos-5", true}, + // Not supported off the Anthropic provider, even for Fable. + {schemas.Bedrock, "claude-fable-5", false}, + {schemas.Vertex, "claude-fable-5", false}, // Defensive cases. {schemas.Anthropic, "", false}, {"", "claude-opus-4-8", false}, @@ -2303,6 +2379,10 @@ func TestSupportsFastMode(t *testing.T) { {"claude-haiku-4-5", false}, {"claude-opus-4-5", false}, {"claude-opus-4-1", false}, + // Fable/Mythos do NOT support fast mode (Opus 4.6/4.7/4.8 only). + {"claude-fable-5", false}, + {"claude-mythos-5", false}, + {"claude-mythos-preview", false}, // Defensive cases. {"", false}, {"some-non-claude-model", false}, @@ -2327,7 +2407,10 @@ func TestSupportsEffortParameter(t *testing.T) { expected bool }{ // Supported per docs. + {"claude-fable-5", true}, + {"claude-mythos-5", true}, {"claude-mythos-preview", true}, + {"global.anthropic.claude-fable-5", true}, {"claude-opus-4-8", true}, {"claude-opus-4.8-20260601", true}, {"claude-opus-4-7", true}, @@ -2647,6 +2730,11 @@ func TestComputerUseGeneration(t *testing.T) { {"claude-sonnet-4.6", ComputerUseGen20251124}, {"claude-opus-4-5", ComputerUseGen20251124}, {"claude-opus-4-5-20251101", ComputerUseGen20251124}, + // Fable/Mythos family uses the new generation, like Opus 4.8. + {"claude-fable-5", ComputerUseGen20251124}, + {"claude-mythos-5", ComputerUseGen20251124}, + {"claude-mythos-preview", ComputerUseGen20251124}, + {"global.anthropic.claude-fable-5", ComputerUseGen20251124}, {"claude-sonnet-4-5", ComputerUseGen20250124}, {"claude-sonnet-4-5-20250929", ComputerUseGen20250124}, {"claude-haiku-4-5", ComputerUseGen20250124}, @@ -2977,8 +3065,8 @@ func TestBudgetTokensMaxEffortCapsBelowMaxTokens(t *testing.T) { const minBudget = MinimumReasoningMaxTokens cases := []struct { - maxTokens int - wantBudget int + maxTokens int + wantBudget int }{ {maxTokens: 16000, wantBudget: 15999}, {maxTokens: 32000, wantBudget: 31999}, diff --git a/core/providers/azure/azure.go b/core/providers/azure/azure.go index c1e7781325..9bd1e2c80f 100644 --- a/core/providers/azure/azure.go +++ b/core/providers/azure/azure.go @@ -226,11 +226,10 @@ func (provider *AzureProvider) completeRequest( }() var url string - isAnthropicModel := schemas.IsAnthropicModel(model) // Set any extra headers from network config. // For Anthropic models, exclude anthropic-beta — it is merged and filtered explicitly below. - if isAnthropicModel { + if schemas.IsAnthropicModelFamily(ctx, model) { providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, []string{anthropic.AnthropicBetaHeader}) } else { providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) @@ -239,7 +238,7 @@ func (provider *AzureProvider) completeRequest( req.Header.SetContentType("application/json") // Get authentication headers - authHeaders, bifrostErr := provider.getAzureAuthHeaders(ctx, key, isAnthropicModel) + authHeaders, bifrostErr := provider.getAzureAuthHeaders(ctx, key, schemas.IsAnthropicModelFamily(ctx, model)) if bifrostErr != nil { return nil, 0, nil, bifrostErr } @@ -249,13 +248,13 @@ func (provider *AzureProvider) completeRequest( req.Header.Set(k, v) } - endpoint := key.AzureKeyConfig.Endpoint.GetValue() + endpoint := resolveAzureEndpoint(ctx, key) if endpoint == "" { return nil, 0, nil, providerUtils.NewConfigurationError("endpoint not set") } - if isAnthropicModel { - req.Header.Set("anthropic-version", AzureAnthropicAPIVersionDefault) + if schemas.IsAnthropicModelFamily(ctx, model) { + req.Header.Set("anthropic-version", resolveAnthropicVersion(ctx)) url = fmt.Sprintf("%s/%s", endpoint, path) // Merge ExtraHeaders + context anthropic-beta, filter for Azure, then set as HTTP header @@ -307,6 +306,11 @@ func (provider *AzureProvider) completeRequest( // listModelsByKey performs a list models request for a single key. // Returns the response and latency, or an error if the request fails. func (provider *AzureProvider) listModelsByKey(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostListModelsRequest) (*schemas.BifrostListModelsResponse, *schemas.BifrostError) { + endpoint := resolveAzureEndpoint(ctx, key) + if endpoint == "" { + return nil, providerUtils.NewConfigurationError("endpoint not set") + } + // Create the request req := fasthttp.AcquireRequest() resp := fasthttp.AcquireResponse() @@ -316,7 +320,7 @@ func (provider *AzureProvider) listModelsByKey(ctx *schemas.BifrostContext, key // Set any extra headers from network config providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) - req.SetRequestURI(key.AzureKeyConfig.Endpoint.GetValue() + providerUtils.GetPathFromContext(ctx, "/openai/v1/models")) + req.SetRequestURI(endpoint + providerUtils.GetPathFromContext(ctx, "/openai/v1/models")) req.Header.SetMethod(http.MethodGet) req.Header.SetContentType("application/json") @@ -460,7 +464,11 @@ func (provider *AzureProvider) TextCompletion(ctx *schemas.BifrostContext, key s // It formats the request, sends it to Azure, and processes the response. // Returns a channel of BifrostStreamChunk objects or an error if the request fails. func (provider *AzureProvider) TextCompletionStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostTextCompletionRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { - url := fmt.Sprintf("%s/openai/v1/completions", key.AzureKeyConfig.Endpoint.GetValue()) + endpoint := resolveAzureEndpoint(ctx, key) + if endpoint == "" { + return nil, providerUtils.NewConfigurationError("endpoint not set") + } + url := fmt.Sprintf("%s/openai/v1/completions", endpoint) // Get Azure authentication headers authHeader, err := provider.getAzureAuthHeaders(ctx, key, false) @@ -496,7 +504,7 @@ func (provider *AzureProvider) ChatCompletion(ctx *schemas.BifrostContext, key s ctx, request, func() (providerUtils.RequestBodyWithExtraParams, error) { - if schemas.IsAnthropicModel(request.Model) { + if schemas.ResolveFamily(ctx, request.Model) == schemas.ModelFamilyAnthropic { reqBody, err := anthropic.ToAnthropicChatRequest(ctx, request) if err != nil { return nil, err @@ -515,7 +523,7 @@ func (provider *AzureProvider) ChatCompletion(ctx *schemas.BifrostContext, key s } var path string - if schemas.IsAnthropicModel(request.Model) { + if schemas.ResolveFamily(ctx, request.Model) == schemas.ModelFamilyAnthropic { path = "anthropic/v1/messages" } else { path = "openai/v1/chat/completions" @@ -550,7 +558,7 @@ func (provider *AzureProvider) ChatCompletion(ctx *schemas.BifrostContext, key s var rawRequest interface{} var rawResponse interface{} - if schemas.IsAnthropicModel(request.Model) { + if schemas.ResolveFamily(ctx, request.Model) == schemas.ModelFamilyAnthropic { anthropicResponse := anthropic.AcquireAnthropicMessageResponse() defer anthropic.ReleaseAnthropicMessageResponse(anthropicResponse) rawRequest, rawResponse, bifrostErr = providerUtils.HandleProviderResponse(responseBody, anthropicResponse, jsonData, providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest), providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse)) @@ -586,14 +594,18 @@ func (provider *AzureProvider) ChatCompletion(ctx *schemas.BifrostContext, key s // Uses Azure-specific URL construction with deployments and supports both api-key and Bearer token authentication. // Returns a channel containing BifrostResponse objects representing the stream or an error if the request fails. func (provider *AzureProvider) ChatCompletionStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostChatRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { + endpoint := resolveAzureEndpoint(ctx, key) + if endpoint == "" { + return nil, providerUtils.NewConfigurationError("endpoint not set") + } var url string - if schemas.IsAnthropicModel(request.Model) { + if schemas.ResolveFamily(ctx, request.Model) == schemas.ModelFamilyAnthropic { authHeader, err := provider.getAzureAuthHeaders(ctx, key, true) if err != nil { return nil, err } - authHeader["anthropic-version"] = AzureAnthropicAPIVersionDefault - url = fmt.Sprintf("%s/anthropic/v1/messages", key.AzureKeyConfig.Endpoint.GetValue()) + authHeader["anthropic-version"] = resolveAnthropicVersion(ctx) + url = fmt.Sprintf("%s/anthropic/v1/messages", endpoint) jsonData, err := providerUtils.CheckContextAndGetRequestBody( ctx, @@ -637,7 +649,7 @@ func (provider *AzureProvider) ChatCompletionStream(ctx *schemas.BifrostContext, if err != nil { return nil, err } - url = fmt.Sprintf("%s/openai/v1/chat/completions", key.AzureKeyConfig.Endpoint.GetValue()) + url = fmt.Sprintf("%s/openai/v1/chat/completions", endpoint) // Use shared streaming logic from OpenAI return openai.HandleOpenAIChatCompletionStreaming( @@ -669,7 +681,7 @@ func (provider *AzureProvider) ChatCompletionStream(ctx *schemas.BifrostContext, func (provider *AzureProvider) Responses(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostResponsesRequest) (*schemas.BifrostResponsesResponse, *schemas.BifrostError) { var jsonData []byte var bifrostErr *schemas.BifrostError - if schemas.IsAnthropicModel(request.Model) { + if schemas.IsAnthropicModelFamily(ctx, request.Model) { jsonData, bifrostErr = getRequestBodyForAnthropicResponses(ctx, request, request.Model, false, provider.sendBackRawRequest, provider.sendBackRawResponse) } else { jsonData, bifrostErr = providerUtils.CheckContextAndGetRequestBody( @@ -685,10 +697,10 @@ func (provider *AzureProvider) Responses(ctx *schemas.BifrostContext, key schema } var path string - if schemas.IsAnthropicModel(request.Model) { + if schemas.IsAnthropicModelFamily(ctx, request.Model) { path = "anthropic/v1/messages" } else { - path = fmt.Sprintf("openai/v1/responses?api-version=%s", AzureAPIVersionPreview) + path = fmt.Sprintf("openai/v1/responses?api-version=%s", resolveAPIVersion(ctx, AzureAPIVersionPreview)) } responseBody, latency, providerResponseHeaders, err := provider.completeRequest( @@ -720,7 +732,7 @@ func (provider *AzureProvider) Responses(ctx *schemas.BifrostContext, key schema var rawRequest interface{} var rawResponse interface{} - if schemas.IsAnthropicModel(request.Model) { + if schemas.IsAnthropicModelFamily(ctx, request.Model) { anthropicResponse := anthropic.AcquireAnthropicMessageResponse() defer anthropic.ReleaseAnthropicMessageResponse(anthropicResponse) rawRequest, rawResponse, bifrostErr = providerUtils.HandleProviderResponse(responseBody, anthropicResponse, jsonData, providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest), providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse)) @@ -753,14 +765,18 @@ func (provider *AzureProvider) Responses(ctx *schemas.BifrostContext, key schema // ResponsesStream performs a streaming responses request to Azure's API. func (provider *AzureProvider) ResponsesStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostResponsesRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { + endpoint := resolveAzureEndpoint(ctx, key) + if endpoint == "" { + return nil, providerUtils.NewConfigurationError("endpoint not set") + } var url string - if schemas.IsAnthropicModel(request.Model) { + if schemas.IsAnthropicModelFamily(ctx, request.Model) { authHeader, err := provider.getAzureAuthHeaders(ctx, key, true) if err != nil { return nil, err } - authHeader["anthropic-version"] = AzureAnthropicAPIVersionDefault - url = fmt.Sprintf("%s/anthropic/v1/messages", key.AzureKeyConfig.Endpoint.GetValue()) + authHeader["anthropic-version"] = resolveAnthropicVersion(ctx) + url = fmt.Sprintf("%s/anthropic/v1/messages", endpoint) jsonData, bifrostErr := getRequestBodyForAnthropicResponses(ctx, request, request.Model, true, provider.sendBackRawRequest, provider.sendBackRawResponse) if bifrostErr != nil { @@ -790,7 +806,7 @@ func (provider *AzureProvider) ResponsesStream(ctx *schemas.BifrostContext, post if err != nil { return nil, err } - url = fmt.Sprintf("%s/openai/v1/responses?api-version=%s", key.AzureKeyConfig.Endpoint.GetValue(), AzureAPIVersionPreview) + url = fmt.Sprintf("%s/openai/v1/responses?api-version=%s", endpoint, resolveAPIVersion(ctx, AzureAPIVersionPreview)) // Use shared streaming logic from OpenAI return openai.HandleOpenAIResponsesStreaming( @@ -881,7 +897,7 @@ func (provider *AzureProvider) Embedding(ctx *schemas.BifrostContext, key schema // Speech is not supported by the Azure provider. func (provider *AzureProvider) Speech(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostSpeechRequest) (*schemas.BifrostSpeechResponse, *schemas.BifrostError) { - endpoint := key.AzureKeyConfig.Endpoint.GetValue() + endpoint := resolveAzureEndpoint(ctx, key) if endpoint == "" { return nil, providerUtils.NewConfigurationError("endpoint not set") } @@ -922,13 +938,18 @@ func (provider *AzureProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, // SpeechStream handles streaming for speech synthesis with Azure. // Azure sends raw binary audio bytes in SSE format, unlike OpenAI which sends JSON. func (provider *AzureProvider) SpeechStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostSpeechRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { + endpoint := resolveAzureEndpoint(ctx, key) + if endpoint == "" { + return nil, providerUtils.NewConfigurationError("endpoint not set") + } + // Get Azure authentication headers authHeader, err := provider.getAzureAuthHeaders(ctx, key, false) if err != nil { return nil, err } - url := fmt.Sprintf("%s/openai/v1/audio/speech", key.AzureKeyConfig.Endpoint.GetValue()) + url := fmt.Sprintf("%s/openai/v1/audio/speech", endpoint) // Create HTTP request for streaming req := fasthttp.AcquireRequest() @@ -1209,7 +1230,11 @@ func (provider *AzureProvider) SpeechStream(ctx *schemas.BifrostContext, postHoo // Transcription is not supported by the Azure provider. func (provider *AzureProvider) Transcription(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostTranscriptionRequest) (*schemas.BifrostTranscriptionResponse, *schemas.BifrostError) { - url := fmt.Sprintf("%s/openai/deployments/%s/audio/transcriptions?api-version=%s", key.AzureKeyConfig.Endpoint.GetValue(), request.Model, DefaultAzureAPIVersion) + endpoint := resolveAzureEndpoint(ctx, key) + if endpoint == "" { + return nil, providerUtils.NewConfigurationError("endpoint not set") + } + url := fmt.Sprintf("%s/openai/deployments/%s/audio/transcriptions?api-version=%s", endpoint, request.Model, resolveAPIVersion(ctx, DefaultAzureAPIVersion)) response, err := openai.HandleOpenAITranscriptionRequest( ctx, @@ -1241,7 +1266,7 @@ func (provider *AzureProvider) TranscriptionStream(ctx *schemas.BifrostContext, // Returns a BifrostResponse containing the bifrost response or an error if the request fails. func (provider *AzureProvider) ImageGeneration(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostImageGenerationRequest) (*schemas.BifrostImageGenerationResponse, *schemas.BifrostError) { - endpoint := key.AzureKeyConfig.Endpoint.GetValue() + endpoint := resolveAzureEndpoint(ctx, key) if endpoint == "" { return nil, providerUtils.NewConfigurationError("endpoint not set") } @@ -1275,7 +1300,7 @@ func (provider *AzureProvider) ImageGenerationStream( key schemas.Key, request *schemas.BifrostImageGenerationRequest, ) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { - endpoint := key.AzureKeyConfig.Endpoint.GetValue() + endpoint := resolveAzureEndpoint(ctx, key) if endpoint == "" { return nil, providerUtils.NewConfigurationError("endpoint not set") } @@ -1311,7 +1336,7 @@ func (provider *AzureProvider) ImageGenerationStream( // ImageEdit performs an image edit request to Azure's API. func (provider *AzureProvider) ImageEdit(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostImageEditRequest) (*schemas.BifrostImageGenerationResponse, *schemas.BifrostError) { - endpoint := key.AzureKeyConfig.Endpoint.GetValue() + endpoint := resolveAzureEndpoint(ctx, key) if endpoint == "" { return nil, providerUtils.NewConfigurationError("endpoint not set") } @@ -1338,7 +1363,7 @@ func (provider *AzureProvider) ImageEdit(ctx *schemas.BifrostContext, key schema // ImageEditStream performs a streaming image edit request to Azure's API. func (provider *AzureProvider) ImageEditStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostImageEditRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { - endpoint := key.AzureKeyConfig.Endpoint.GetValue() + endpoint := resolveAzureEndpoint(ctx, key) if endpoint == "" { return nil, providerUtils.NewConfigurationError("endpoint not set") } @@ -1380,7 +1405,7 @@ func (provider *AzureProvider) ImageVariation(ctx *schemas.BifrostContext, key s // VideoGeneration creates a video using Azure's OpenAI-compatible Sora API. // This delegates to the OpenAI handler with Azure-specific URL and authentication. func (provider *AzureProvider) VideoGeneration(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostVideoGenerationRequest) (*schemas.BifrostVideoGenerationResponse, *schemas.BifrostError) { - endpoint := key.AzureKeyConfig.Endpoint.GetValue() + endpoint := resolveAzureEndpoint(ctx, key) if endpoint == "" { return nil, providerUtils.NewConfigurationError("endpoint not set") } @@ -1415,7 +1440,7 @@ func (provider *AzureProvider) VideoRetrieve(ctx *schemas.BifrostContext, key sc } videoID := providerUtils.StripVideoIDProviderSuffix(request.ID, providerName) - endpoint := key.AzureKeyConfig.Endpoint.GetValue() + endpoint := resolveAzureEndpoint(ctx, key) if endpoint == "" { return nil, providerUtils.NewConfigurationError("endpoint not set") } @@ -1450,7 +1475,7 @@ func (provider *AzureProvider) VideoDownload(ctx *schemas.BifrostContext, key sc } videoID := providerUtils.StripVideoIDProviderSuffix(request.ID, providerName) - endpoint := key.AzureKeyConfig.Endpoint.GetValue() + endpoint := resolveAzureEndpoint(ctx, key) if endpoint == "" { return nil, providerUtils.NewConfigurationError("endpoint not set") } @@ -1525,7 +1550,7 @@ func (provider *AzureProvider) VideoDelete(ctx *schemas.BifrostContext, key sche } videoID := providerUtils.StripVideoIDProviderSuffix(request.ID, providerName) - endpoint := key.AzureKeyConfig.Endpoint.GetValue() + endpoint := resolveAzureEndpoint(ctx, key) if endpoint == "" { return nil, providerUtils.NewConfigurationError("endpoint not set") } @@ -1554,7 +1579,7 @@ func (provider *AzureProvider) VideoDelete(ctx *schemas.BifrostContext, key sche // VideoList lists videos from Azure's OpenAI-compatible API. func (provider *AzureProvider) VideoList(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostVideoListRequest) (*schemas.BifrostVideoListResponse, *schemas.BifrostError) { - endpoint := key.AzureKeyConfig.Endpoint.GetValue() + endpoint := resolveAzureEndpoint(ctx, key) if endpoint == "" { return nil, providerUtils.NewConfigurationError("endpoint not set") } @@ -1588,6 +1613,10 @@ func (provider *AzureProvider) VideoRemix(_ *schemas.BifrostContext, _ schemas.K // FileUpload uploads a file to Azure OpenAI. func (provider *AzureProvider) FileUpload(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostFileUploadRequest) (*schemas.BifrostFileUploadResponse, *schemas.BifrostError) { + endpoint := resolveAzureEndpoint(ctx, key) + if endpoint == "" { + return nil, providerUtils.NewConfigurationError("endpoint not set") + } if len(request.File) == 0 { return nil, providerUtils.NewBifrostOperationError("file content is required", nil) } @@ -1629,7 +1658,7 @@ func (provider *AzureProvider) FileUpload(ctx *schemas.BifrostContext, key schem defer fasthttp.ReleaseResponse(resp) // Build URL - requestURL := fmt.Sprintf("%s/openai/v1/files", key.AzureKeyConfig.Endpoint.GetValue()) + requestURL := fmt.Sprintf("%s/openai/v1/files", endpoint) // Set headers providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) @@ -1700,6 +1729,11 @@ func (provider *AzureProvider) FileList(ctx *schemas.BifrostContext, keys []sche }, nil } + endpoint := resolveAzureEndpoint(ctx, key) + if endpoint == "" { + return nil, providerUtils.NewConfigurationError("endpoint not set") + } + // Create request req := fasthttp.AcquireRequest() resp := fasthttp.AcquireResponse() @@ -1707,7 +1741,7 @@ func (provider *AzureProvider) FileList(ctx *schemas.BifrostContext, keys []sche defer fasthttp.ReleaseResponse(resp) // Build URL with query params - requestURL := fmt.Sprintf("%s/openai/v1/files", key.AzureKeyConfig.Endpoint.GetValue()) + requestURL := fmt.Sprintf("%s/openai/v1/files", endpoint) values := url.Values{} if request.Purpose != "" { values.Set("purpose", string(request.Purpose)) @@ -1803,12 +1837,17 @@ func (provider *AzureProvider) FileRetrieve(ctx *schemas.BifrostContext, keys [] var lastErr *schemas.BifrostError for _, key := range keys { + endpoint := resolveAzureEndpoint(ctx, key) + if endpoint == "" { + lastErr = providerUtils.NewConfigurationError("endpoint not set") + continue + } // Create request req := fasthttp.AcquireRequest() resp := fasthttp.AcquireResponse() // Build URL - requestURL := fmt.Sprintf("%s/openai/v1/files/%s", key.AzureKeyConfig.Endpoint.GetValue(), url.PathEscape(request.FileID)) + requestURL := fmt.Sprintf("%s/openai/v1/files/%s", endpoint, url.PathEscape(request.FileID)) // Set headers providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) @@ -1887,12 +1926,17 @@ func (provider *AzureProvider) FileDelete(ctx *schemas.BifrostContext, keys []sc var lastErr *schemas.BifrostError for _, key := range keys { + endpoint := resolveAzureEndpoint(ctx, key) + if endpoint == "" { + lastErr = providerUtils.NewConfigurationError("endpoint not set") + continue + } // Create request req := fasthttp.AcquireRequest() resp := fasthttp.AcquireResponse() // Build URL - requestURL := fmt.Sprintf("%s/openai/v1/files/%s", key.AzureKeyConfig.Endpoint.GetValue(), url.PathEscape(request.FileID)) + requestURL := fmt.Sprintf("%s/openai/v1/files/%s", endpoint, url.PathEscape(request.FileID)) // Set headers providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) @@ -2000,12 +2044,17 @@ func (provider *AzureProvider) FileContent(ctx *schemas.BifrostContext, keys []s var lastErr *schemas.BifrostError for _, key := range keys { + endpoint := resolveAzureEndpoint(ctx, key) + if endpoint == "" { + lastErr = providerUtils.NewConfigurationError("endpoint not set") + continue + } // Create request req := fasthttp.AcquireRequest() resp := fasthttp.AcquireResponse() // Build URL - requestURL := fmt.Sprintf("%s/openai/v1/files/%s/content", key.AzureKeyConfig.Endpoint.GetValue(), url.PathEscape(request.FileID)) + requestURL := fmt.Sprintf("%s/openai/v1/files/%s/content", endpoint, url.PathEscape(request.FileID)) // Set headers providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) @@ -2075,6 +2124,10 @@ func (provider *AzureProvider) FileContent(ctx *schemas.BifrostContext, keys []s // BatchCreate creates a new batch job on Azure OpenAI. // Azure Batch API uses the same format as OpenAI but with Azure-specific URL patterns. func (provider *AzureProvider) BatchCreate(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostBatchCreateRequest) (*schemas.BifrostBatchCreateResponse, *schemas.BifrostError) { + endpoint := resolveAzureEndpoint(ctx, key) + if endpoint == "" { + return nil, providerUtils.NewConfigurationError("endpoint not set") + } inputFileID := request.InputFileID // If no file_id provided but inline requests are available, upload them first @@ -2110,7 +2163,7 @@ func (provider *AzureProvider) BatchCreate(ctx *schemas.BifrostContext, key sche defer fasthttp.ReleaseResponse(resp) // Build URL - requestURL := fmt.Sprintf("%s/openai/v1/batches", key.AzureKeyConfig.Endpoint.GetValue()) + requestURL := fmt.Sprintf("%s/openai/v1/batches", endpoint) // Set headers providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) @@ -2209,6 +2262,11 @@ func (provider *AzureProvider) BatchList(ctx *schemas.BifrostContext, keys []sch }, nil } + endpoint := resolveAzureEndpoint(ctx, key) + if endpoint == "" { + return nil, providerUtils.NewConfigurationError("endpoint not set") + } + // Create request req := fasthttp.AcquireRequest() resp := fasthttp.AcquireResponse() @@ -2216,7 +2274,7 @@ func (provider *AzureProvider) BatchList(ctx *schemas.BifrostContext, keys []sch defer fasthttp.ReleaseResponse(resp) // Build URL with query params - baseURL := fmt.Sprintf("%s/openai/v1/batches", key.AzureKeyConfig.Endpoint.GetValue()) + baseURL := fmt.Sprintf("%s/openai/v1/batches", endpoint) values := url.Values{} if request.Limit > 0 { values.Set("limit", fmt.Sprintf("%d", request.Limit)) @@ -2303,12 +2361,17 @@ func (provider *AzureProvider) BatchRetrieve(ctx *schemas.BifrostContext, keys [ var lastErr *schemas.BifrostError for _, key := range keys { + endpoint := resolveAzureEndpoint(ctx, key) + if endpoint == "" { + lastErr = providerUtils.NewConfigurationError("endpoint not set") + continue + } // Create request req := fasthttp.AcquireRequest() resp := fasthttp.AcquireResponse() // Build URL - requestURL := fmt.Sprintf("%s/openai/v1/batches/%s", key.AzureKeyConfig.Endpoint.GetValue(), url.PathEscape(request.BatchID)) + requestURL := fmt.Sprintf("%s/openai/v1/batches/%s", endpoint, url.PathEscape(request.BatchID)) // Set headers providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) @@ -2388,12 +2451,17 @@ func (provider *AzureProvider) BatchCancel(ctx *schemas.BifrostContext, keys []s var lastErr *schemas.BifrostError for _, key := range keys { + endpoint := resolveAzureEndpoint(ctx, key) + if endpoint == "" { + lastErr = providerUtils.NewConfigurationError("endpoint not set") + continue + } // Create request req := fasthttp.AcquireRequest() resp := fasthttp.AcquireResponse() // Build URL - requestURL := fmt.Sprintf("%s/openai/v1/batches/%s/cancel", key.AzureKeyConfig.Endpoint.GetValue(), url.PathEscape(request.BatchID)) + requestURL := fmt.Sprintf("%s/openai/v1/batches/%s/cancel", endpoint, url.PathEscape(request.BatchID)) // Set headers providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) @@ -2729,8 +2797,9 @@ func (provider *AzureProvider) Compaction(ctx *schemas.BifrostContext, key schem // buildContainerURL constructs the Azure container API URL. // Container endpoints are not per-deployment, so they use the openai/v1 prefix directly. -func (provider *AzureProvider) buildContainerURL(key schemas.Key, path string) string { - endpoint := strings.TrimRight(key.AzureKeyConfig.Endpoint.GetValue(), "/") +// ctx carries the resolved alias so per-alias Endpoint overrides are honored. +func (provider *AzureProvider) buildContainerURL(ctx *schemas.BifrostContext, key schemas.Key, path string) string { + endpoint := strings.TrimRight(resolveAzureEndpoint(ctx, key), "/") return fmt.Sprintf("%s/openai/v1%s", endpoint, path) } @@ -2742,7 +2811,7 @@ func (provider *AzureProvider) ContainerCreate(ctx *schemas.BifrostContext, key if request.Name == "" { return nil, providerUtils.NewBifrostOperationError("invalid request: name is required", nil) } - if key.AzureKeyConfig == nil || key.AzureKeyConfig.Endpoint.GetValue() == "" { + if resolveAzureEndpoint(ctx, key) == "" { return nil, providerUtils.NewConfigurationError("endpoint not set") } @@ -2779,7 +2848,7 @@ func (provider *AzureProvider) ContainerCreate(ctx *schemas.BifrostContext, key defer fasthttp.ReleaseResponse(resp) providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) - req.SetRequestURI(provider.buildContainerURL(key, "/containers")) + req.SetRequestURI(provider.buildContainerURL(ctx, key, "/containers")) req.Header.SetMethod(http.MethodPost) req.Header.SetContentType("application/json") req.SetBody(jsonBody) @@ -2863,7 +2932,7 @@ func (provider *AzureProvider) ContainerRetrieve(ctx *schemas.BifrostContext, ke var lastErr *schemas.BifrostError for _, key := range keys { - if key.AzureKeyConfig == nil || key.AzureKeyConfig.Endpoint.GetValue() == "" { + if resolveAzureEndpoint(ctx, key) == "" { lastErr = providerUtils.NewConfigurationError("endpoint not set") continue } @@ -2872,7 +2941,7 @@ func (provider *AzureProvider) ContainerRetrieve(ctx *schemas.BifrostContext, ke resp := fasthttp.AcquireResponse() providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) - req.SetRequestURI(provider.buildContainerURL(key, "/containers/"+url.PathEscape(request.ContainerID))) + req.SetRequestURI(provider.buildContainerURL(ctx, key, "/containers/"+url.PathEscape(request.ContainerID))) req.Header.SetMethod(http.MethodGet) authHeaders, bifrostErr := provider.getAzureAuthHeaders(ctx, key, false) @@ -2969,7 +3038,7 @@ func (provider *AzureProvider) ContainerDelete(ctx *schemas.BifrostContext, keys var lastErr *schemas.BifrostError for _, key := range keys { - if key.AzureKeyConfig == nil || key.AzureKeyConfig.Endpoint.GetValue() == "" { + if resolveAzureEndpoint(ctx, key) == "" { lastErr = providerUtils.NewConfigurationError("endpoint not set") continue } @@ -2978,7 +3047,7 @@ func (provider *AzureProvider) ContainerDelete(ctx *schemas.BifrostContext, keys resp := fasthttp.AcquireResponse() providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) - req.SetRequestURI(provider.buildContainerURL(key, "/containers/"+url.PathEscape(request.ContainerID))) + req.SetRequestURI(provider.buildContainerURL(ctx, key, "/containers/"+url.PathEscape(request.ContainerID))) req.Header.SetMethod(http.MethodDelete) req.Header.SetContentType("application/json") @@ -3061,7 +3130,7 @@ func (provider *AzureProvider) ContainerFileCreate(ctx *schemas.BifrostContext, if len(request.File) == 0 { return nil, providerUtils.NewBifrostOperationError("invalid request: file is required", nil) } - if key.AzureKeyConfig == nil || key.AzureKeyConfig.Endpoint.GetValue() == "" { + if resolveAzureEndpoint(ctx, key) == "" { return nil, providerUtils.NewConfigurationError("endpoint not set") } @@ -3084,7 +3153,7 @@ func (provider *AzureProvider) ContainerFileCreate(ctx *schemas.BifrostContext, defer fasthttp.ReleaseResponse(resp) providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) - req.SetRequestURI(provider.buildContainerURL(key, fmt.Sprintf("/containers/%s/files", url.PathEscape(request.ContainerID)))) + req.SetRequestURI(provider.buildContainerURL(ctx, key, fmt.Sprintf("/containers/%s/files", url.PathEscape(request.ContainerID)))) req.Header.SetMethod(http.MethodPost) req.Header.Set("Content-Type", writer.FormDataContentType()) req.SetBody(body.Bytes()) @@ -3169,11 +3238,11 @@ func (provider *AzureProvider) ContainerFileList(ctx *schemas.BifrostContext, ke if !ok { return &schemas.BifrostContainerFileListResponse{Object: "list", Data: []schemas.ContainerFileObject{}, HasMore: false}, nil } - if key.AzureKeyConfig == nil || key.AzureKeyConfig.Endpoint.GetValue() == "" { + if resolveAzureEndpoint(ctx, key) == "" { return nil, providerUtils.NewConfigurationError("endpoint not set") } - requestURL := provider.buildContainerURL(key, fmt.Sprintf("/containers/%s/files", url.PathEscape(request.ContainerID))) + requestURL := provider.buildContainerURL(ctx, key, fmt.Sprintf("/containers/%s/files", url.PathEscape(request.ContainerID))) queryParams := url.Values{} if request.Limit > 0 { queryParams.Set("limit", fmt.Sprintf("%d", request.Limit)) @@ -3275,7 +3344,7 @@ func (provider *AzureProvider) ContainerFileRetrieve(ctx *schemas.BifrostContext var lastErr *schemas.BifrostError for _, key := range keys { - if key.AzureKeyConfig == nil || key.AzureKeyConfig.Endpoint.GetValue() == "" { + if resolveAzureEndpoint(ctx, key) == "" { lastErr = providerUtils.NewConfigurationError("endpoint not set") continue } @@ -3284,7 +3353,7 @@ func (provider *AzureProvider) ContainerFileRetrieve(ctx *schemas.BifrostContext resp := fasthttp.AcquireResponse() providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) - req.SetRequestURI(provider.buildContainerURL(key, fmt.Sprintf("/containers/%s/files/%s", url.PathEscape(request.ContainerID), url.PathEscape(request.FileID)))) + req.SetRequestURI(provider.buildContainerURL(ctx, key, fmt.Sprintf("/containers/%s/files/%s", url.PathEscape(request.ContainerID), url.PathEscape(request.FileID)))) req.Header.SetMethod(http.MethodGet) authHeaders, bifrostErr := provider.getAzureAuthHeaders(ctx, key, false) @@ -3380,7 +3449,7 @@ func (provider *AzureProvider) ContainerFileContent(ctx *schemas.BifrostContext, var lastErr *schemas.BifrostError for _, key := range keys { - if key.AzureKeyConfig == nil || key.AzureKeyConfig.Endpoint.GetValue() == "" { + if resolveAzureEndpoint(ctx, key) == "" { lastErr = providerUtils.NewConfigurationError("endpoint not set") continue } @@ -3389,7 +3458,7 @@ func (provider *AzureProvider) ContainerFileContent(ctx *schemas.BifrostContext, resp := fasthttp.AcquireResponse() providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) - req.SetRequestURI(provider.buildContainerURL(key, fmt.Sprintf("/containers/%s/files/%s/content", url.PathEscape(request.ContainerID), url.PathEscape(request.FileID)))) + req.SetRequestURI(provider.buildContainerURL(ctx, key, fmt.Sprintf("/containers/%s/files/%s/content", url.PathEscape(request.ContainerID), url.PathEscape(request.FileID)))) req.Header.SetMethod(http.MethodGet) authHeaders, bifrostErr := provider.getAzureAuthHeaders(ctx, key, false) @@ -3470,7 +3539,7 @@ func (provider *AzureProvider) ContainerFileDelete(ctx *schemas.BifrostContext, var lastErr *schemas.BifrostError for _, key := range keys { - if key.AzureKeyConfig == nil || key.AzureKeyConfig.Endpoint.GetValue() == "" { + if resolveAzureEndpoint(ctx, key) == "" { lastErr = providerUtils.NewConfigurationError("endpoint not set") continue } @@ -3479,7 +3548,7 @@ func (provider *AzureProvider) ContainerFileDelete(ctx *schemas.BifrostContext, resp := fasthttp.AcquireResponse() providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) - req.SetRequestURI(provider.buildContainerURL(key, fmt.Sprintf("/containers/%s/files/%s", url.PathEscape(request.ContainerID), url.PathEscape(request.FileID)))) + req.SetRequestURI(provider.buildContainerURL(ctx, key, fmt.Sprintf("/containers/%s/files/%s", url.PathEscape(request.ContainerID), url.PathEscape(request.FileID)))) req.Header.SetMethod(http.MethodDelete) authHeaders, bifrostErr := provider.getAzureAuthHeaders(ctx, key, false) @@ -3556,7 +3625,10 @@ func (provider *AzureProvider) Passthrough( key schemas.Key, req *schemas.BifrostPassthroughRequest, ) (*schemas.BifrostPassthroughResponse, *schemas.BifrostError) { - url := provider.buildPassthroughURL(key, req.Path, req.RawQuery) + url, err := provider.buildPassthroughURL(ctx, key, req.Path, req.RawQuery) + if err != nil { + return nil, providerUtils.NewConfigurationError(fmt.Sprintf("failed to build passthrough URL: %s", err.Error())) + } fasthttpReq := fasthttp.AcquireRequest() resp := fasthttp.AcquireResponse() @@ -3572,7 +3644,7 @@ func (provider *AzureProvider) Passthrough( fasthttpReq.Header.Set(k, v) } - authHeaders, bifrostErr := provider.getAzureAuthHeaders(ctx, key, schemas.IsAnthropicModel(req.Model)) + authHeaders, bifrostErr := provider.getAzureAuthHeaders(ctx, key, schemas.IsAnthropicModelFamily(ctx, req.Model)) if bifrostErr != nil { return nil, bifrostErr } @@ -3625,7 +3697,10 @@ func (provider *AzureProvider) PassthroughStream( key schemas.Key, req *schemas.BifrostPassthroughRequest, ) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { - url := provider.buildPassthroughURL(key, req.Path, req.RawQuery) + url, err := provider.buildPassthroughURL(ctx, key, req.Path, req.RawQuery) + if err != nil { + return nil, providerUtils.NewConfigurationError(fmt.Sprintf("failed to build passthrough URL: %s", err.Error())) + } fasthttpReq := fasthttp.AcquireRequest() resp := fasthttp.AcquireResponse() @@ -3643,7 +3718,7 @@ func (provider *AzureProvider) PassthroughStream( fasthttpReq.Header.Set("Connection", "close") - authHeaders, bifrostErr := provider.getAzureAuthHeaders(ctx, key, schemas.IsAnthropicModel(req.Model)) + authHeaders, bifrostErr := provider.getAzureAuthHeaders(ctx, key, schemas.IsAnthropicModelFamily(ctx, req.Model)) if bifrostErr != nil { return nil, bifrostErr } @@ -3686,7 +3761,7 @@ func (provider *AzureProvider) PassthroughStream( } var anthropicUsage *anthropic.AnthropicPassthroughStreamUsage - if schemas.IsAnthropicModel(req.Model) { + if schemas.IsAnthropicModelFamily(ctx, req.Model) { anthropicUsage = &anthropic.AnthropicPassthroughStreamUsage{} } return providerUtils.StreamPassthrough( @@ -3716,8 +3791,13 @@ func (provider *AzureProvider) PassthroughStream( } // buildPassthroughURL constructs the full Azure URL for a passthrough request. -func (provider *AzureProvider) buildPassthroughURL(key schemas.Key, path, rawQuery string) string { - endpoint := key.AzureKeyConfig.Endpoint.GetValue() +// ctx carries the resolved alias used to pick a per-alias api-version override +// when the caller did not supply one in rawQuery. +func (provider *AzureProvider) buildPassthroughURL(ctx *schemas.BifrostContext, key schemas.Key, path, rawQuery string) (string, error) { + endpoint := resolveAzureEndpoint(ctx, key) + if endpoint == "" { + return "", fmt.Errorf("endpoint not set") + } // Normalise paths emitted by the Azure SDK. path = strings.Replace(path, "/openai/responses", "/openai/v1/responses", 1) @@ -3734,14 +3814,14 @@ func (provider *AzureProvider) buildPassthroughURL(key schemas.Key, path, rawQue // Responses API requires api-version=preview. values, _ := url.ParseQuery(rawQuery) if values.Get("api-version") == "" { - values.Set("api-version", AzureAPIVersionPreview) + values.Set("api-version", resolveAPIVersion(ctx, AzureAPIVersionPreview)) rawQuery = values.Encode() } case strings.Contains(path, "/openai/deployments/"): // Classic /deployments/ routes require api-version. Inject a default if absent. values, _ := url.ParseQuery(rawQuery) if values.Get("api-version") == "" { - values.Set("api-version", DefaultAzureAPIVersion) + values.Set("api-version", resolveAPIVersion(ctx, DefaultAzureAPIVersion)) rawQuery = values.Encode() } } @@ -3750,7 +3830,7 @@ func (provider *AzureProvider) buildPassthroughURL(key schemas.Key, path, rawQue if rawQuery != "" { fullURL += "?" + rawQuery } - return fullURL + return fullURL, nil } // extractAzurePassthroughUsage dispatches usage extraction by the upstream API the diff --git a/core/providers/azure/azure_passthrough_test.go b/core/providers/azure/azure_passthrough_test.go index b070c49f9f..684c02dfd1 100644 --- a/core/providers/azure/azure_passthrough_test.go +++ b/core/providers/azure/azure_passthrough_test.go @@ -119,10 +119,153 @@ func TestBuildPassthroughURL(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - got := provider.buildPassthroughURL(makeKey(endpoint), tt.path, tt.rawQuery) + got, _ := provider.buildPassthroughURL(nil, makeKey(endpoint), tt.path, tt.rawQuery) if got != tt.want { t.Errorf("\ngot: %s\nwant: %s", got, tt.want) } }) } } + +// TestBuildPassthroughURL_AliasAPIVersionOverride verifies that when the +// resolved alias carries an AzureAliasCfg.APIVersion override, it takes +// precedence over the route default (DefaultAzureAPIVersion for /deployments/, +// AzureAPIVersionPreview for /openai/v1/responses) — only in the path where +// the caller did NOT supply api-version themselves. Caller-supplied wins over +// alias override; alias override wins over route default. +func TestBuildPassthroughURL_AliasAPIVersionOverride(t *testing.T) { + t.Parallel() + + provider := &AzureProvider{} + endpoint := "https://my-resource.openai.azure.com" + makeKey := schemas.Key{ + AzureKeyConfig: &schemas.AzureKeyConfig{ + Endpoint: *schemas.NewEnvVar(endpoint), + }, + } + + // Build a ctx carrying an alias with APIVersion override. + overrideVer := "2024-10-21" + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + ctx.SetValue(schemas.BifrostContextKeyResolvedAlias, &schemas.ResolvedAlias{ + Key: "best-model", + Config: &schemas.AliasConfig{ + ModelID: "gpt-4o-deployment", + AzureAliasCfg: &schemas.AzureAliasCfg{ + APIVersion: &overrideVer, + }, + }, + }) + + t.Run("deployments route: alias APIVersion overrides default", func(t *testing.T) { + got, _ := provider.buildPassthroughURL(ctx, makeKey, "/openai/deployments/gpt-4o/chat/completions", "") + want := endpoint + "/openai/deployments/gpt-4o/chat/completions?api-version=" + overrideVer + if got != want { + t.Errorf("\ngot: %s\nwant: %s", got, want) + } + }) + + t.Run("responses route: alias APIVersion overrides preview default", func(t *testing.T) { + got, _ := provider.buildPassthroughURL(ctx, makeKey, "/openai/v1/responses", "") + want := endpoint + "/openai/v1/responses?api-version=" + overrideVer + if got != want { + t.Errorf("\ngot: %s\nwant: %s", got, want) + } + }) + + t.Run("caller-supplied api-version wins over alias override", func(t *testing.T) { + got, _ := provider.buildPassthroughURL(ctx, makeKey, "/openai/deployments/gpt-4o/chat/completions", "api-version=2023-01-01") + want := endpoint + "/openai/deployments/gpt-4o/chat/completions?api-version=2023-01-01" + if got != want { + t.Errorf("\ngot: %s\nwant: %s", got, want) + } + }) +} + +// TestResolveAPIVersion_NoAlias verifies the helper returns the route default +// when no resolved alias is in ctx (covers the legacy code path). +func TestResolveAPIVersion_NoAlias(t *testing.T) { + if got := resolveAPIVersion(nil, DefaultAzureAPIVersion); got != DefaultAzureAPIVersion { + t.Errorf("got %q, want %q", got, DefaultAzureAPIVersion) + } + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + if got := resolveAPIVersion(ctx, AzureAPIVersionPreview); got != AzureAPIVersionPreview { + t.Errorf("got %q, want %q", got, AzureAPIVersionPreview) + } +} + +// TestResolveAzureEndpoint_AliasOverride verifies the Endpoint override path. +// Lets one Azure credential cover deployments hosted on multiple cognitive- +// services resources. +func TestResolveAzureEndpoint_AliasOverride(t *testing.T) { + keyEndpoint := "https://primary.openai.azure.com" + aliasEndpoint := "https://anthropic-resource.openai.azure.com" + key := schemas.Key{ + AzureKeyConfig: &schemas.AzureKeyConfig{ + Endpoint: *schemas.NewEnvVar(keyEndpoint), + }, + } + + // No alias: falls back to key-level endpoint. + if got := resolveAzureEndpoint(nil, key); got != keyEndpoint { + t.Errorf("nil ctx: got %q, want key-level %q", got, keyEndpoint) + } + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + if got := resolveAzureEndpoint(ctx, key); got != keyEndpoint { + t.Errorf("empty ctx: got %q, want key-level %q", got, keyEndpoint) + } + + // With alias-level Endpoint override, alias wins. + override := schemas.NewEnvVar(aliasEndpoint) + ctx.SetValue(schemas.BifrostContextKeyResolvedAlias, &schemas.ResolvedAlias{ + Key: "best-claude", + Config: &schemas.AliasConfig{ + ModelID: "claude-deployment", + AzureAliasCfg: &schemas.AzureAliasCfg{ + Endpoint: override, + }, + }, + }) + if got := resolveAzureEndpoint(ctx, key); got != aliasEndpoint { + t.Errorf("alias override: got %q, want %q", got, aliasEndpoint) + } + + // Alias with empty Endpoint value falls through to key-level — guards against + // a misconfigured alias accidentally erasing the endpoint. + emptyOverride := schemas.NewEnvVar("") + ctx2 := schemas.NewBifrostContext(nil, schemas.NoDeadline) + ctx2.SetValue(schemas.BifrostContextKeyResolvedAlias, &schemas.ResolvedAlias{ + Key: "x", + Config: &schemas.AliasConfig{ + ModelID: "x", + AzureAliasCfg: &schemas.AzureAliasCfg{ + Endpoint: emptyOverride, + }, + }, + }) + if got := resolveAzureEndpoint(ctx2, key); got != keyEndpoint { + t.Errorf("empty alias endpoint should fall through: got %q, want %q", got, keyEndpoint) + } +} + +// TestResolveAnthropicVersion_AliasOverride verifies the AnthropicVersion +// override path mirrors the APIVersion behavior. +func TestResolveAnthropicVersion_AliasOverride(t *testing.T) { + if got := resolveAnthropicVersion(nil); got != AzureAnthropicAPIVersionDefault { + t.Errorf("nil ctx: got %q, want default", got) + } + override := "2024-10-22" + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + ctx.SetValue(schemas.BifrostContextKeyResolvedAlias, &schemas.ResolvedAlias{ + Key: "best-claude", + Config: &schemas.AliasConfig{ + ModelID: "claude-deployment", + AzureAliasCfg: &schemas.AzureAliasCfg{ + AnthropicVersion: &override, + }, + }, + }) + if got := resolveAnthropicVersion(ctx); got != override { + t.Errorf("got %q, want %q", got, override) + } +} diff --git a/core/providers/azure/models.go b/core/providers/azure/models.go index 5daca3836d..99f4c5ae86 100644 --- a/core/providers/azure/models.go +++ b/core/providers/azure/models.go @@ -7,7 +7,7 @@ import ( "github.com/maximhq/bifrost/core/schemas" ) -func (response *AzureListModelsResponse) ToBifrostListModelsResponse(allowedModels schemas.WhiteList, blacklistedModels schemas.BlackList, aliases map[string]string, unfiltered bool) *schemas.BifrostListModelsResponse { +func (response *AzureListModelsResponse) ToBifrostListModelsResponse(allowedModels schemas.WhiteList, blacklistedModels schemas.BlackList, aliases schemas.KeyAliases, unfiltered bool) *schemas.BifrostListModelsResponse { if response == nil { return nil } diff --git a/core/providers/azure/utils.go b/core/providers/azure/utils.go index e1ef5d02f1..2aecf5e04f 100644 --- a/core/providers/azure/utils.go +++ b/core/providers/azure/utils.go @@ -37,3 +37,45 @@ func getAzureScopes(configuredScopes []string) []string { } return scopes } + +// resolveAnthropicVersion returns the anthropic-version header value for the +// current attempt. Uses the AzureAliasCfg.AnthropicVersion override from the +// resolved alias when present, otherwise the Azure default. +func resolveAnthropicVersion(ctx *schemas.BifrostContext) string { + if ra := schemas.GetResolvedAlias(ctx); ra != nil && ra.Config != nil && ra.Config.AzureAliasCfg != nil && ra.Config.AzureAliasCfg.AnthropicVersion != nil && *ra.Config.AzureAliasCfg.AnthropicVersion != "" { + return *ra.Config.AzureAliasCfg.AnthropicVersion + } + return AzureAnthropicAPIVersionDefault +} + +// resolveAPIVersion returns the Azure api-version query parameter value for +// the current attempt. Uses the AzureAliasCfg.APIVersion override from the +// resolved alias when present, otherwise the provided default. Different +// Azure routes have different defaults (DefaultAzureAPIVersion for classic +// /openai/deployments/, AzureAPIVersionPreview for /openai/v1/responses); +// callers pass the route's default so the override can take precedence +// without losing the route-specific fallback. +func resolveAPIVersion(ctx *schemas.BifrostContext, defaultVersion string) string { + if ra := schemas.GetResolvedAlias(ctx); ra != nil && ra.Config != nil && ra.Config.AzureAliasCfg != nil && ra.Config.AzureAliasCfg.APIVersion != nil && *ra.Config.AzureAliasCfg.APIVersion != "" { + return *ra.Config.AzureAliasCfg.APIVersion + } + return defaultVersion +} + +// resolveAzureEndpoint returns the Azure cognitive-services endpoint URL for +// the current attempt. Uses the AzureAliasCfg.Endpoint override from the +// resolved alias when present, otherwise the key-level endpoint. Lets one +// Azure credential (ClientID/Secret/TenantID or API key) span deployments +// hosted on different Azure resources (e.g. OpenAI on east-us, Anthropic on +// west-us2). +func resolveAzureEndpoint(ctx *schemas.BifrostContext, key schemas.Key) string { + if ra := schemas.GetResolvedAlias(ctx); ra != nil && ra.Config != nil && ra.Config.AzureAliasCfg != nil && ra.Config.AzureAliasCfg.Endpoint != nil { + if v := ra.Config.AzureAliasCfg.Endpoint.GetValue(); v != "" { + return v + } + } + if key.AzureKeyConfig != nil { + return key.AzureKeyConfig.Endpoint.GetValue() + } + return "" +} diff --git a/core/providers/bedrock/bedrock.go b/core/providers/bedrock/bedrock.go index 8b7c169b04..4b7558bd62 100644 --- a/core/providers/bedrock/bedrock.go +++ b/core/providers/bedrock/bedrock.go @@ -205,7 +205,7 @@ var retryableBedrockExceptions = map[string]int{ // Returns the response body, request latency, or an error if the request fails. func (provider *BedrockProvider) completeRequest(ctx *schemas.BifrostContext, jsonData []byte, path string, key schemas.Key, model string) ([]byte, time.Duration, map[string]string, *schemas.BifrostError) { config := key.BedrockKeyConfig - region := resolveBedrockRegion(key, model) + region := resolveBedrockRegion(ctx, key, model) // Create the request with the JSON body requestURL := fmt.Sprintf("https://bedrock-runtime.%s.amazonaws.com/model/%s", region, path) @@ -434,7 +434,7 @@ func (provider *BedrockProvider) completeAgentRuntimeRequest(ctx *schemas.Bifros // Returns the response body and an error if the request fails. func (provider *BedrockProvider) makeStreamingRequest(ctx *schemas.BifrostContext, jsonData []byte, key schemas.Key, model string, action string) (*http.Response, *schemas.BifrostError) { // Parse region and path in one pass to avoid running the regex twice. - path, region := provider.getModelPathAndRegion(action, model, key) + path, region := provider.getModelPathAndRegion(ctx, action, model, key) // Create HTTP request for streaming requestURL := fmt.Sprintf("https://bedrock-runtime.%s.amazonaws.com/model/%s", region, path) @@ -860,7 +860,7 @@ func (provider *BedrockProvider) TextCompletion(ctx *schemas.BifrostContext, key return nil, bifrostErr } - path, _ := provider.getModelPathAndRegion("invoke", request.Model, key) + path, _ := provider.getModelPathAndRegion(ctx, "invoke", request.Model, key) body, latency, providerResponseHeaders, err := provider.completeRequest(ctx, jsonData, path, key, request.Model) if providerResponseHeaders != nil { ctx.SetValue(schemas.BifrostContextKeyProviderResponseHeaders, providerResponseHeaders) @@ -872,14 +872,14 @@ func (provider *BedrockProvider) TextCompletion(ctx *schemas.BifrostContext, key // Handle model-specific response conversion var bifrostResponse *schemas.BifrostTextCompletionResponse switch { - case schemas.IsAnthropicModel(request.Model): + case schemas.IsAnthropicModelFamily(ctx, request.Model): var response BedrockAnthropicTextResponse if err := sonic.Unmarshal(body, &response); err != nil { return nil, providerUtils.NewBifrostOperationError("error parsing anthropic response", err) } bifrostResponse = response.ToBifrostTextCompletionResponse() - case schemas.IsMistralModel(request.Model): + case schemas.IsMistralModelFamily(ctx, request.Model): var response BedrockMistralTextResponse if err := sonic.Unmarshal(body, &response); err != nil { return nil, providerUtils.NewBifrostOperationError("error parsing mistral response", err) @@ -1089,7 +1089,7 @@ func (provider *BedrockProvider) ChatCompletion(ctx *schemas.BifrostContext, key } // Format the path with proper model identifier - path, _ := provider.getModelPathAndRegion("converse", request.Model, key) + path, _ := provider.getModelPathAndRegion(ctx, "converse", request.Model, key) // Create the signed request responseBody, latency, providerResponseHeaders, bifrostErr := provider.completeRequest(ctx, jsonData, path, key, request.Model) @@ -1477,7 +1477,7 @@ func (provider *BedrockProvider) Responses(ctx *schemas.BifrostContext, key sche } // Format the path with proper model identifier - path, _ := provider.getModelPathAndRegion("converse", request.Model, key) + path, _ := provider.getModelPathAndRegion(ctx, "converse", request.Model, key) // Create the signed request responseBody, latency, providerResponseHeaders, bifrostErr := provider.completeRequest(ctx, jsonData, path, key, request.Model) @@ -1837,7 +1837,7 @@ func (provider *BedrockProvider) Embedding(ctx *schemas.BifrostContext, key sche } // Determine model type - modelType, err := DetermineEmbeddingModelType(request.Model) + modelType, err := DetermineEmbeddingModelType(ctx, request.Model) if err != nil { return nil, providerUtils.NewConfigurationError(err.Error()) } @@ -1861,7 +1861,7 @@ func (provider *BedrockProvider) Embedding(ctx *schemas.BifrostContext, key sche if bifrostError != nil { return nil, bifrostError } - path, _ = provider.getModelPathAndRegion("invoke", request.Model, key) + path, _ = provider.getModelPathAndRegion(ctx, "invoke", request.Model, key) rawResponse, latency, providerResponseHeaders, bifrostError = provider.completeRequest(ctx, jsonData, path, key, request.Model) case "cohere": @@ -1874,7 +1874,7 @@ func (provider *BedrockProvider) Embedding(ctx *schemas.BifrostContext, key sche if bifrostError != nil { return nil, bifrostError } - path, _ = provider.getModelPathAndRegion("invoke", request.Model, key) + path, _ = provider.getModelPathAndRegion(ctx, "invoke", request.Model, key) rawResponse, latency, providerResponseHeaders, bifrostError = provider.completeRequest(ctx, jsonData, path, key, request.Model) default: @@ -1920,6 +1920,18 @@ func (provider *BedrockProvider) Embedding(ctx *schemas.BifrostContext, key sche } } + // Bedrock Cohere embed models omit token usage from the response body and instead + // return it in the X-Amzn-Bedrock-Input-Token-Count response header. Backfill Usage + // from that header when the body did not provide it. (#3917) + if bifrostResponse.Usage == nil { + if inputTokens, ok := inputTokensFromHeaders(providerResponseHeaders); ok { + bifrostResponse.Usage = &schemas.BifrostLLMUsage{ + PromptTokens: inputTokens, + TotalTokens: inputTokens, + } + } + } + // Set ExtraFields bifrostResponse.ExtraFields.Latency = latency.Milliseconds() bifrostResponse.ExtraFields.ProviderResponseHeaders = providerResponseHeaders @@ -1979,6 +1991,17 @@ func (provider *BedrockProvider) Rerank(ctx *schemas.BifrostContext, key schemas bifrostResponse := response.ToBifrostRerankResponse(request.Documents, returnDocuments) bifrostResponse.Model = request.Model + // Bedrock returns rerank input token usage only in the X-Amzn-Bedrock-Input-Token-Count + // response header (it is absent from the body); backfill Usage from it. (#3917) + if bifrostResponse.Usage == nil { + if inputTokens, ok := inputTokensFromHeaders(providerResponseHeaders); ok { + bifrostResponse.Usage = &schemas.BifrostLLMUsage{ + PromptTokens: inputTokens, + TotalTokens: inputTokens, + } + } + } + bifrostResponse.ExtraFields.Latency = latency.Milliseconds() bifrostResponse.ExtraFields.ProviderResponseHeaders = providerResponseHeaders @@ -2027,7 +2050,7 @@ func (provider *BedrockProvider) ImageGeneration(ctx *schemas.BifrostContext, ke var providerResponseHeaders map[string]string var path string - path, _ = provider.getModelPathAndRegion("invoke", request.Model, key) + path, _ = provider.getModelPathAndRegion(ctx, "invoke", request.Model, key) jsonData, bifrostError = providerUtils.CheckContextAndGetRequestBody( ctx, @@ -2100,7 +2123,7 @@ func (provider *BedrockProvider) ImageEdit(ctx *schemas.BifrostContext, key sche var bifrostError *schemas.BifrostError // Stability AI routing and task-type inference use the actual model ID. - path, _ := provider.getModelPathAndRegion("invoke", request.Model, key) + path, _ := provider.getModelPathAndRegion(ctx, "invoke", request.Model, key) jsonData, bifrostError = providerUtils.CheckContextAndGetRequestBody( ctx, @@ -2182,7 +2205,7 @@ func (provider *BedrockProvider) ImageVariation(ctx *schemas.BifrostContext, key } // Make API request (same URL as image generation) - path, _ := provider.getModelPathAndRegion("invoke", request.Model, key) + path, _ := provider.getModelPathAndRegion(ctx, "invoke", request.Model, key) rawResponse, latency, providerResponseHeaders, bifrostError := provider.completeRequest(ctx, jsonData, path, key, request.Model) if providerResponseHeaders != nil { ctx.SetValue(schemas.BifrostContextKeyProviderResponseHeaders, providerResponseHeaders) @@ -3575,32 +3598,29 @@ func (provider *BedrockProvider) BatchResults(ctx *schemas.BifrostContext, keys return batchResultsResp, nil } -// resolveBedrockRegion returns the AWS region to use for a request. -// the priority is: model string region > key configured region > default region -func resolveBedrockRegion(key schemas.Key, model string) string { - if region, _ := parseBedrockRegionAndModel(model); region != "" { - return region - } - if key.BedrockKeyConfig != nil && key.BedrockKeyConfig.Region != nil && key.BedrockKeyConfig.Region.GetValue() != "" { - return key.BedrockKeyConfig.Region.GetValue() - } - return DefaultBedrockRegion -} - // getModelPathAndRegion is a helper that calls parseBedrockRegionAndModel -// once and returns both the request path and the AWS signing region -func (provider *BedrockProvider) getModelPathAndRegion(basePath, model string, key schemas.Key) (path, region string) { +// once and returns both the request path and the AWS signing region. +// Honors per-alias Region and BedrockAliasCfg.InferenceProfileARN overrides +// via the resolved alias in ctx. +func (provider *BedrockProvider) getModelPathAndRegion(ctx *schemas.BifrostContext, basePath, model string, key schemas.Key) (path, region string) { r, bareModel := parseBedrockRegionAndModel(model) if r == "" { - if key.BedrockKeyConfig != nil && key.BedrockKeyConfig.Region != nil && key.BedrockKeyConfig.Region.GetValue() != "" { - r = key.BedrockKeyConfig.Region.GetValue() - } else { - r = DefaultBedrockRegion + if ra := schemas.GetResolvedAlias(ctx); ra != nil && ra.Config != nil && ra.Config.Region != nil { + if v := ra.Config.Region.GetValue(); v != "" { + r = v + } + } + if r == "" { + if key.BedrockKeyConfig != nil && key.BedrockKeyConfig.Region != nil && key.BedrockKeyConfig.Region.GetValue() != "" { + r = key.BedrockKeyConfig.Region.GetValue() + } else { + r = DefaultBedrockRegion + } } } p := fmt.Sprintf("%s/%s", bareModel, basePath) - if key.BedrockKeyConfig != nil && key.BedrockKeyConfig.ARN != nil && key.BedrockKeyConfig.ARN.GetValue() != "" { - encodedModelIdentifier := url.PathEscape(fmt.Sprintf("%s/%s", key.BedrockKeyConfig.ARN.GetValue(), bareModel)) + if arn := resolveBedrockARN(ctx, key); arn != "" { + encodedModelIdentifier := url.PathEscape(fmt.Sprintf("%s/%s", arn, bareModel)) p = fmt.Sprintf("%s/%s", encodedModelIdentifier, basePath) } return p, r @@ -3627,7 +3647,7 @@ func (provider *BedrockProvider) CountTokens(ctx *schemas.BifrostContext, key sc } // Format the path with proper model identifier - path, _ := provider.getModelPathAndRegion("count-tokens", request.Model, key) + path, _ := provider.getModelPathAndRegion(ctx, "count-tokens", request.Model, key) // Send the request responseBody, latency, providerResponseHeaders, bifrostErr := provider.completeRequest(ctx, jsonData, path, key, request.Model) diff --git a/core/providers/bedrock/chat.go b/core/providers/bedrock/chat.go index ed089e7098..175b5973c2 100644 --- a/core/providers/bedrock/chat.go +++ b/core/providers/bedrock/chat.go @@ -25,7 +25,7 @@ func ToBedrockChatCompletionRequest(ctx *schemas.BifrostContext, bifrostReq *sch } input := bifrostReq.Input - if schemas.IsAnthropicModel(bifrostReq.Model) && ctx.Value(schemas.BifrostContextKeySupportsAssistantPrefill) == false { + if schemas.IsAnthropicModelFamily(ctx, bifrostReq.Model) && ctx.Value(schemas.BifrostContextKeySupportsAssistantPrefill) == false { trimmed := len(input) for trimmed > 0 && input[trimmed-1].Role == schemas.ChatMessageRoleAssistant { trimmed-- @@ -46,7 +46,7 @@ func ToBedrockChatCompletionRequest(ctx *schemas.BifrostContext, bifrostReq *sch // Trim trailing whitespace from the last assistant message text blocks // (only for Anthropic models which use text-based prefill) lastMsgIndex := len(bedrockReq.Messages) - 1 - if schemas.IsAnthropicModel(bifrostReq.Model) && lastMsgIndex >= 0 && bedrockReq.Messages[lastMsgIndex].Role == BedrockMessageRoleAssistant { + if schemas.IsAnthropicModelFamily(ctx, bifrostReq.Model) && lastMsgIndex >= 0 && bedrockReq.Messages[lastMsgIndex].Role == BedrockMessageRoleAssistant { blocks := bedrockReq.Messages[lastMsgIndex].Content for j := len(blocks) - 1; j >= 0; j-- { if blocks[j].Text != nil { diff --git a/core/providers/bedrock/embedding.go b/core/providers/bedrock/embedding.go index cb4ef19e88..4342ca2667 100644 --- a/core/providers/bedrock/embedding.go +++ b/core/providers/bedrock/embedding.go @@ -3,11 +3,33 @@ package bedrock import ( "encoding/json" "fmt" + "strconv" "strings" "github.com/maximhq/bifrost/core/schemas" ) +// bedrockInputTokenCountHeader is the HTTP response header Bedrock uses to report input +// token counts for models — notably Cohere embed and rerank — that omit token usage from +// the response body. +const bedrockInputTokenCountHeader = "X-Amzn-Bedrock-Input-Token-Count" + +// inputTokensFromHeaders extracts the X-Amzn-Bedrock-Input-Token-Count value from a provider +// response-headers map (case-insensitive, since header casing depends on the transport). +// It returns (count, true) only when the header is present and parses as a non-negative int. +func inputTokensFromHeaders(headers map[string]string) (int, bool) { + for k, v := range headers { + if strings.EqualFold(k, bedrockInputTokenCountHeader) { + n, err := strconv.Atoi(strings.TrimSpace(v)) + if err != nil || n < 0 { + return 0, false + } + return n, true + } + } + return 0, false +} + // ToBedrockTitanEmbeddingRequest converts a Bifrost embedding request to Bedrock Titan format func ToBedrockTitanEmbeddingRequest(bifrostReq *schemas.BifrostEmbeddingRequest) (*BedrockTitanEmbeddingRequest, error) { if bifrostReq == nil { @@ -159,12 +181,16 @@ func ToBedrockCohereEmbeddingRequest(bifrostReq *schemas.BifrostEmbeddingRequest return req, nil } -// DetermineEmbeddingModelType determines the embedding model type from the model name -func DetermineEmbeddingModelType(model string) (string, error) { +// DetermineEmbeddingModelType determines the embedding model type for the +// current attempt. It consults the resolved alias family first +// (model_family / model_name / model_id / alias key) and falls back to the +// substring detectors against the wire model — so an alias to an opaque +// Bedrock deployment that's tagged with the right family routes correctly. +func DetermineEmbeddingModelType(ctx *schemas.BifrostContext, model string) (string, error) { switch { - case strings.Contains(model, "amazon.titan-embed-text"): + case schemas.IsTitanModelFamily(ctx, model): return "titan", nil - case strings.Contains(model, "cohere.embed"): + case schemas.IsCohereModelFamily(ctx, model): return "cohere", nil default: return "", fmt.Errorf("unsupported embedding model: %s", model) @@ -189,7 +215,7 @@ func (r *BedrockCohereEmbeddingResponse) ToBifrostEmbeddingResponse() (*schemas. Float [][]float32 `json:"float"` Base64 []string `json:"base64"` Int8 [][]int8 `json:"int8"` - Uint8 [][]int32 `json:"uint8"` // int32 avoids []byte→base64 JSON issue + Uint8 [][]int32 `json:"uint8"` // int32 avoids []byte→base64 JSON issue Binary [][]int8 `json:"binary"` Ubinary [][]int32 `json:"ubinary"` // int32 avoids []byte→base64 JSON issue } diff --git a/core/providers/bedrock/invoke.go b/core/providers/bedrock/invoke.go index 8227e8639a..3edcd4e12c 100644 --- a/core/providers/bedrock/invoke.go +++ b/core/providers/bedrock/invoke.go @@ -387,7 +387,7 @@ func (r *BedrockInvokeRequest) ToBifrostEmbeddingRequest(ctx *schemas.BifrostCon if unescaped, err := url.PathUnescape(r.ModelID); err == nil { modelID = unescaped } - provider, model := schemas.ParseModelString(modelID, providerUtils.CheckAndSetDefaultProvider(ctx, schemas.Bedrock)) + provider, model := schemas.ParseModelString(modelID, "") req := &schemas.BifrostEmbeddingRequest{ Provider: provider, Model: model, @@ -451,7 +451,7 @@ func (r *BedrockInvokeRequest) ToBifrostImageGenerationRequest(ctx *schemas.Bifr if unescaped, err := url.PathUnescape(r.ModelID); err == nil { modelID = unescaped } - provider, model := schemas.ParseModelString(modelID, providerUtils.CheckAndSetDefaultProvider(ctx, schemas.Bedrock)) + provider, model := schemas.ParseModelString(modelID, "") req := &schemas.BifrostImageGenerationRequest{ Provider: provider, Model: model, @@ -515,7 +515,7 @@ func (r *BedrockInvokeRequest) ToBifrostImageEditRequest(ctx *schemas.BifrostCon if unescaped, err := url.PathUnescape(r.ModelID); err == nil { modelID = unescaped } - provider, model := schemas.ParseModelString(modelID, providerUtils.CheckAndSetDefaultProvider(ctx, schemas.Bedrock)) + provider, model := schemas.ParseModelString(modelID, "") req := &schemas.BifrostImageEditRequest{ Provider: provider, Model: model, @@ -690,7 +690,7 @@ func (r *BedrockInvokeRequest) ToBifrostImageVariationRequest(ctx *schemas.Bifro if unescaped, err := url.PathUnescape(r.ModelID); err == nil { modelID = unescaped } - provider, model := schemas.ParseModelString(modelID, providerUtils.CheckAndSetDefaultProvider(ctx, schemas.Bedrock)) + provider, model := schemas.ParseModelString(modelID, "") req := &schemas.BifrostImageVariationRequest{ Provider: provider, Model: model, @@ -956,7 +956,7 @@ func ToBedrockInvokeImagesResponse(ctx *schemas.BifrostContext, resp *schemas.Bi // Bedrock invoke API response format. // Single-embedding (Titan) responses use: {"embedding": [...], "inputTextTokenCount": N} // Multi-embedding (Cohere) responses use: {"embeddings": [[...],[...]], "response_type": "embeddings_floats"} -func ToBedrockEmbeddingInvokeResponse(resp *schemas.BifrostEmbeddingResponse) (interface{}, error) { +func ToBedrockEmbeddingInvokeResponse(ctx *schemas.BifrostContext, resp *schemas.BifrostEmbeddingResponse) (interface{}, error) { if resp == nil { return nil, fmt.Errorf("bifrost embedding response is nil") } @@ -975,7 +975,7 @@ func ToBedrockEmbeddingInvokeResponse(resp *schemas.BifrostEmbeddingResponse) (i return &BedrockInvokeEmbeddingResp{InputTextTokenCount: tokenCount}, nil } - // Use model name to distinguish Cohere from Titan — not batch size. + // Use the resolved family to distinguish Cohere from Titan — not batch size. // A single-input Cohere request must still return the Cohere envelope format. model := resp.Model if model == "" { @@ -986,7 +986,7 @@ func ToBedrockEmbeddingInvokeResponse(resp *schemas.BifrostEmbeddingResponse) (i } } - if strings.Contains(strings.ToLower(model), "cohere") { + if schemas.IsCohereModelFamily(ctx, model) { floats := make([][]float32, 0, len(resp.Data)) for _, d := range resp.Data { float32Emb := make([]float32, len(d.Embedding.EmbeddingArray)) diff --git a/core/providers/bedrock/mantle.go b/core/providers/bedrock/mantle.go index 5c85a1cbdc..932a9df544 100644 --- a/core/providers/bedrock/mantle.go +++ b/core/providers/bedrock/mantle.go @@ -60,7 +60,7 @@ func (provider *BedrockProvider) chatCompletionViaMantle( key schemas.Key, request *schemas.BifrostChatRequest, ) (*schemas.BifrostChatResponse, *schemas.BifrostError) { - region := resolveBedrockRegion(key, request.Model) + region := resolveBedrockRegion(ctx, key, request.Model) url := mantleURL(region, "chat/completions") // Build extraHeaders: always start with network-config headers, then overlay SigV4 if needed. @@ -106,7 +106,7 @@ func (provider *BedrockProvider) chatCompletionStreamViaMantle( key schemas.Key, request *schemas.BifrostChatRequest, ) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { - region := resolveBedrockRegion(key, request.Model) + region := resolveBedrockRegion(ctx, key, request.Model) url := mantleURL(region, "chat/completions") // Bearer: identical to Groq / any OpenAI-compatible provider. @@ -162,7 +162,7 @@ func (provider *BedrockProvider) responsesViaMantle( key schemas.Key, request *schemas.BifrostResponsesRequest, ) (*schemas.BifrostResponsesResponse, *schemas.BifrostError) { - region := resolveBedrockRegion(key, request.Model) + region := resolveBedrockRegion(ctx, key, request.Model) url := mantleURL(region, "responses") extraHeaders := make(map[string]string, len(provider.networkConfig.ExtraHeaders)) @@ -204,7 +204,7 @@ func (provider *BedrockProvider) responsesStreamViaMantle( key schemas.Key, request *schemas.BifrostResponsesRequest, ) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { - region := resolveBedrockRegion(key, request.Model) + region := resolveBedrockRegion(ctx, key, request.Model) url := mantleURL(region, "responses") // Bearer: identical to Groq / any OpenAI-compatible provider. diff --git a/core/providers/bedrock/models.go b/core/providers/bedrock/models.go index 549db2e3bd..817a43f2b4 100644 --- a/core/providers/bedrock/models.go +++ b/core/providers/bedrock/models.go @@ -81,7 +81,7 @@ type BedrockRerankResponseDocument struct { TextDocument *BedrockRerankTextValue `json:"textDocument,omitempty"` } -func (response *BedrockListModelsResponse) ToBifrostListModelsResponse(providerKey schemas.ModelProvider, allowedModels schemas.WhiteList, blacklistedModels schemas.BlackList, aliases map[string]string, unfiltered bool) *schemas.BifrostListModelsResponse { +func (response *BedrockListModelsResponse) ToBifrostListModelsResponse(providerKey schemas.ModelProvider, allowedModels schemas.WhiteList, blacklistedModels schemas.BlackList, aliases schemas.KeyAliases, unfiltered bool) *schemas.BifrostListModelsResponse { if response == nil { return nil } diff --git a/core/providers/bedrock/region_test.go b/core/providers/bedrock/region_test.go index cbd8b3d1cc..55bdf538e9 100644 --- a/core/providers/bedrock/region_test.go +++ b/core/providers/bedrock/region_test.go @@ -59,7 +59,7 @@ func TestGetModelPathStripsRegion(t *testing.T) { } for _, tc := range cases { t.Run(tc.model, func(t *testing.T) { - got, _ := provider.getModelPathAndRegion(tc.basePath, tc.model, key) + got, _ := provider.getModelPathAndRegion(nil, tc.basePath, tc.model, key) assert.Equal(t, tc.wantPath, got) }) } @@ -91,12 +91,98 @@ func TestGetModelPathStripsRegionWithARN(t *testing.T) { } for _, tc := range cases { t.Run(tc.model, func(t *testing.T) { - got, _ := provider.getModelPathAndRegion("converse", tc.model, key) + got, _ := provider.getModelPathAndRegion(nil, "converse", tc.model, key) assert.Equal(t, tc.wantPath, got) }) } } +// TestResolveBedrockRegion_AliasOverride verifies the per-alias Region +// override slots between the model-string prefix (highest priority) and the +// key-level Region (lower priority). +func TestResolveBedrockRegion_AliasOverride(t *testing.T) { + keyRegion := "us-east-1" + aliasRegion := "us-west-2" + key := schemas.Key{ + BedrockKeyConfig: &schemas.BedrockKeyConfig{ + Region: schemas.NewEnvVar(keyRegion), + }, + } + + // Build ctx carrying an alias with Region override. + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + ctx.SetValue(schemas.BifrostContextKeyResolvedAlias, &schemas.ResolvedAlias{ + Key: "best-claude", + Config: &schemas.AliasConfig{ + ModelID: "anthropic.claude-3-5-sonnet-20241022-v2:0", + Region: schemas.NewEnvVar(aliasRegion), + }, + }) + + // Bare model — alias.Region wins over key.Region. + if got := resolveBedrockRegion(ctx, key, "anthropic.claude-3-5-sonnet-20241022-v2:0"); got != aliasRegion { + t.Errorf("alias override should win over key region: got %q, want %q", got, aliasRegion) + } + + // Model string with explicit region prefix — wins over alias override. + if got := resolveBedrockRegion(ctx, key, "eu-west-1/anthropic.claude-v2"); got != "eu-west-1" { + t.Errorf("model-string region should win over alias override: got %q", got) + } + + // No alias in ctx — falls through to key.Region. + emptyCtx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + if got := resolveBedrockRegion(emptyCtx, key, "anthropic.claude-v2"); got != keyRegion { + t.Errorf("no alias: should use key.Region: got %q, want %q", got, keyRegion) + } +} + +// TestResolveBedrockARN_AliasOverride verifies the BedrockAliasCfg +// InferenceProfileARN override takes precedence over key.BedrockKeyConfig.ARN. +func TestResolveBedrockARN_AliasOverride(t *testing.T) { + keyARN := "arn:aws:bedrock:us-east-1:1234567890:resource-config/default" + aliasARN := "arn:aws:bedrock:us-east-1:1234567890:inference-profile/us.anthropic.claude-3-7-sonnet" + key := schemas.Key{ + BedrockKeyConfig: &schemas.BedrockKeyConfig{ + ARN: schemas.NewEnvVar(keyARN), + }, + } + + // No alias — falls back to key.ARN. + if got := resolveBedrockARN(nil, key); got != keyARN { + t.Errorf("nil ctx: got %q, want key ARN %q", got, keyARN) + } + + // Alias with InferenceProfileARN override wins. + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + ctx.SetValue(schemas.BifrostContextKeyResolvedAlias, &schemas.ResolvedAlias{ + Key: "best-claude", + Config: &schemas.AliasConfig{ + ModelID: "anthropic.claude-3-7-sonnet-20250219-v1:0", + BedrockAliasCfg: &schemas.BedrockAliasCfg{ + InferenceProfileARN: schemas.NewEnvVar(aliasARN), + }, + }, + }) + if got := resolveBedrockARN(ctx, key); got != aliasARN { + t.Errorf("alias override should win: got %q, want %q", got, aliasARN) + } + + // Empty alias ARN — falls through to key.ARN. + ctx2 := schemas.NewBifrostContext(nil, schemas.NoDeadline) + ctx2.SetValue(schemas.BifrostContextKeyResolvedAlias, &schemas.ResolvedAlias{ + Key: "x", + Config: &schemas.AliasConfig{ + ModelID: "x", + BedrockAliasCfg: &schemas.BedrockAliasCfg{ + InferenceProfileARN: schemas.NewEnvVar(""), + }, + }, + }) + if got := resolveBedrockARN(ctx2, key); got != keyARN { + t.Errorf("empty alias ARN should fall through to key ARN: got %q, want %q", got, keyARN) + } +} + func TestResolveBedrockRegion(t *testing.T) { configuredRegion := "ap-southeast-1" key := schemas.Key{ @@ -119,7 +205,7 @@ func TestResolveBedrockRegion(t *testing.T) { } for _, tc := range cases { t.Run(tc.desc, func(t *testing.T) { - got := resolveBedrockRegion(tc.key, tc.model) + got := resolveBedrockRegion(nil, tc.key, tc.model) assert.Equal(t, tc.wantRegion, got) }) } diff --git a/core/providers/bedrock/rerank.go b/core/providers/bedrock/rerank.go index 3ba99441c6..82bbd4554c 100644 --- a/core/providers/bedrock/rerank.go +++ b/core/providers/bedrock/rerank.go @@ -5,7 +5,6 @@ import ( "sort" "strings" - providerUtils "github.com/maximhq/bifrost/core/providers/utils" "github.com/maximhq/bifrost/core/schemas" ) @@ -134,7 +133,7 @@ func (req *BedrockRerankRequest) ToBifrostRerankRequest(ctx *schemas.BifrostCont } modelARN := req.RerankingConfiguration.BedrockRerankingConfiguration.ModelConfiguration.ModelARN - provider, model := schemas.ParseModelString(modelARN, providerUtils.CheckAndSetDefaultProvider(ctx, schemas.Bedrock)) + provider, model := schemas.ParseModelString(modelARN, "") bifrostReq := &schemas.BifrostRerankRequest{ Provider: provider, diff --git a/core/providers/bedrock/rerank_test.go b/core/providers/bedrock/rerank_test.go index c1b7bb5480..5d55dccc69 100644 --- a/core/providers/bedrock/rerank_test.go +++ b/core/providers/bedrock/rerank_test.go @@ -196,7 +196,7 @@ func TestBedrockRerankRequestToBifrostRerankRequestNil(t *testing.T) { func TestResolveBedrockDeployment(t *testing.T) { key := schemas.Key{ Aliases: schemas.KeyAliases{ - "cohere-rerank": "arn:aws:bedrock:us-east-1::foundation-model/cohere.rerank-v3-5:0", + "cohere-rerank": {ModelID: "arn:aws:bedrock:us-east-1::foundation-model/cohere.rerank-v3-5:0"}, }, } @@ -211,7 +211,7 @@ func TestBedrockRerankRequiresARNModelIdentifier(t *testing.T) { ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) key := schemas.Key{ Aliases: schemas.KeyAliases{ - "cohere-rerank": "cohere.rerank-v3-5:0", + "cohere-rerank": {ModelID: "cohere.rerank-v3-5:0"}, }, } diff --git a/core/providers/bedrock/responses.go b/core/providers/bedrock/responses.go index 905deb0a19..2eb5bbe2cd 100644 --- a/core/providers/bedrock/responses.go +++ b/core/providers/bedrock/responses.go @@ -1850,7 +1850,7 @@ func (request *BedrockConverseRequest) ToBifrostResponsesRequest(ctx *schemas.Bi } // Extract provider from model ID (format: "bedrock/model-name") - provider, model := schemas.ParseModelString(request.ModelID, providerUtils.CheckAndSetDefaultProvider(ctx, schemas.Bedrock)) + provider, model := schemas.ParseModelString(request.ModelID, "") bifrostReq := &schemas.BifrostResponsesRequest{ Provider: provider, @@ -1921,7 +1921,7 @@ func (request *BedrockConverseRequest) ToBifrostResponsesRequest(ctx *schemas.Bi continue } bifrostReq.Params.Tools = append(bifrostReq.Params.Tools, schemas.ResponsesTool{Type: toolType}) - } else if tool.CachePoint != nil && !schemas.IsNovaModel(bifrostReq.Model) { + } else if tool.CachePoint != nil && !schemas.IsNovaModelFamily(ctx, bifrostReq.Model) { // add cache control to last tool in tools array if len(bifrostReq.Params.Tools) > 0 { bifrostReq.Params.Tools[len(bifrostReq.Params.Tools)-1].CacheControl = &schemas.CacheControl{ @@ -2018,7 +2018,7 @@ func (request *BedrockConverseRequest) ToBifrostResponsesRequest(ctx *schemas.Bi if request.InferenceConfig != nil && request.InferenceConfig.MaxTokens != nil { defaultMaxTokens = *request.InferenceConfig.MaxTokens } - if schemas.IsAnthropicModel(bifrostReq.Model) { + if schemas.IsAnthropicModelFamily(ctx, bifrostReq.Model) { minBudgetTokens = anthropic.MinimumReasoningMaxTokens } effort := providerUtils.GetReasoningEffortFromBudgetTokens(maxTokens, minBudgetTokens, defaultMaxTokens) @@ -2151,7 +2151,7 @@ func ToBedrockResponsesRequest(ctx *schemas.BifrostContext, bifrostReq *schemas. // map bifrost messages to bedrock messages using the new conversion method if bifrostReq.Input != nil { input := bifrostReq.Input - if schemas.IsAnthropicModel(bifrostReq.Model) && ctx.Value(schemas.BifrostContextKeySupportsAssistantPrefill) == false { + if schemas.IsAnthropicModelFamily(ctx, bifrostReq.Model) && ctx.Value(schemas.BifrostContextKeySupportsAssistantPrefill) == false { trimmed := len(input) for trimmed > 0 && input[trimmed-1].Role != nil && *input[trimmed-1].Role == schemas.ResponsesInputMessageRoleAssistant { trimmed-- @@ -2180,7 +2180,7 @@ func ToBedrockResponsesRequest(ctx *schemas.BifrostContext, bifrostReq *schemas. // Trim trailing whitespace from the last assistant message text blocks // (only for Anthropic models which use text-based prefill) lastMsgIndex := len(bedrockReq.Messages) - 1 - if schemas.IsAnthropicModel(bifrostReq.Model) && lastMsgIndex >= 0 && bedrockReq.Messages[lastMsgIndex].Role == BedrockMessageRoleAssistant { + if schemas.IsAnthropicModelFamily(ctx, bifrostReq.Model) && lastMsgIndex >= 0 && bedrockReq.Messages[lastMsgIndex].Role == BedrockMessageRoleAssistant { blocks := bedrockReq.Messages[lastMsgIndex].Content for j := len(blocks) - 1; j >= 0; j-- { if blocks[j].Text != nil { @@ -2217,15 +2217,26 @@ func ToBedrockResponsesRequest(ctx *schemas.BifrostContext, bifrostReq *schemas. // setting it to default max tokens tokenBudget = anthropic.MinimumReasoningMaxTokens } - if schemas.IsAnthropicModel(bifrostReq.Model) && tokenBudget < anthropic.MinimumReasoningMaxTokens { - return nil, fmt.Errorf("reasoning.max_tokens must be >= %d for anthropic", anthropic.MinimumReasoningMaxTokens) - } - if schemas.IsAnthropicModel(bifrostReq.Model) { - bedrockReq.AdditionalModelRequestFields.Set("thinking", map[string]any{ - "type": "enabled", - "budget_tokens": tokenBudget, - }) - } else if schemas.IsNovaModel(bifrostReq.Model) { + if schemas.IsAnthropicModelFamily(ctx, bifrostReq.Model) { + if anthropic.IsAdaptiveOnlyThinkingModel(bifrostReq.Model) { + bedrockReq.AdditionalModelRequestFields.Set("thinking", map[string]any{ + "type": "adaptive", + }) + // Preserve a co-present effort — these models support effort, + // and the budget is otherwise dropped. + if bifrostReq.Params.Reasoning.Effort != nil && *bifrostReq.Params.Reasoning.Effort != "none" { + setOutputConfigField(bedrockReq.AdditionalModelRequestFields, "effort", anthropic.MapBifrostEffortToAnthropic(*bifrostReq.Params.Reasoning.Effort)) + } + } else { + if tokenBudget < anthropic.MinimumReasoningMaxTokens { + return nil, fmt.Errorf("reasoning.max_tokens must be >= %d for anthropic", anthropic.MinimumReasoningMaxTokens) + } + bedrockReq.AdditionalModelRequestFields.Set("thinking", map[string]any{ + "type": "enabled", + "budget_tokens": tokenBudget, + }) + } + } else if schemas.IsNovaModelFamily(ctx, bifrostReq.Model) { minBudgetTokens := MinimumReasoningMaxTokens modelDefaultMaxTokens := providerUtils.GetMaxOutputTokensOrDefault(bifrostReq.Model, DefaultCompletionMaxTokens) defaultMaxTokens := modelDefaultMaxTokens @@ -2258,7 +2269,7 @@ func ToBedrockResponsesRequest(ctx *schemas.BifrostContext, bifrostReq *schemas. } } else { if bifrostReq.Params.Reasoning.Effort != nil && *bifrostReq.Params.Reasoning.Effort != "none" { - if schemas.IsNovaModel(bifrostReq.Model) { + if schemas.IsNovaModelFamily(ctx, bifrostReq.Model) { effort := *bifrostReq.Params.Reasoning.Effort typeStr := "enabled" switch effort { @@ -2283,7 +2294,7 @@ func ToBedrockResponsesRequest(ctx *schemas.BifrostContext, bifrostReq *schemas. } bedrockReq.AdditionalModelRequestFields.Set("reasoningConfig", config) - } else if schemas.IsAnthropicModel(bifrostReq.Model) { + } else if schemas.IsAnthropicModelFamily(ctx, bifrostReq.Model) { if anthropic.SupportsAdaptiveThinking(bifrostReq.Model) { // Opus 4.6+: adaptive thinking + output_config.effort effort := anthropic.MapBifrostEffortToAnthropic(*bifrostReq.Params.Reasoning.Effort) @@ -2297,7 +2308,7 @@ func ToBedrockResponsesRequest(ctx *schemas.BifrostContext, bifrostReq *schemas. } else { thinkingConfig["display"] = "summarized" } - } else if anthropic.IsOpus47Plus(bifrostReq.Model) { + } else if anthropic.IsAdaptiveOnlyThinkingModel(bifrostReq.Model) { thinkingConfig["display"] = "summarized" } bedrockReq.AdditionalModelRequestFields.Set("thinking", thinkingConfig) @@ -2338,11 +2349,15 @@ func ToBedrockResponsesRequest(ctx *schemas.BifrostContext, bifrostReq *schemas. }) } } else { - if schemas.IsAnthropicModel(bifrostReq.Model) { - bedrockReq.AdditionalModelRequestFields.Set("thinking", map[string]any{ - "type": "disabled", - }) - } else if schemas.IsNovaModel(bifrostReq.Model) { + if schemas.IsAnthropicModelFamily(ctx, bifrostReq.Model) { + if !anthropic.IsFableFamily(bifrostReq.Model) { + // Fable/Mythos reject thinking:{type:"disabled"}; omit it + // entirely (adaptive thinking is always on for that family). + bedrockReq.AdditionalModelRequestFields.Set("thinking", map[string]any{ + "type": "disabled", + }) + } + } else if schemas.IsNovaModelFamily(ctx, bifrostReq.Model) { bedrockReq.AdditionalModelRequestFields.Set("reasoningConfig", map[string]any{ "type": "disabled", }) @@ -2449,7 +2464,7 @@ func ToBedrockResponsesRequest(ctx *schemas.BifrostContext, bifrostReq *schemas. } bedrockTools = append(bedrockTools, bedrockTool) - if tool.CacheControl != nil && !schemas.IsNovaModel(bifrostReq.Model) { + if tool.CacheControl != nil && !schemas.IsNovaModelFamily(ctx, bifrostReq.Model) { bedrockTools = append(bedrockTools, BedrockTool{ CachePoint: &BedrockCachePoint{ Type: BedrockCachePointTypeDefault, @@ -2480,7 +2495,7 @@ func ToBedrockResponsesRequest(ctx *schemas.BifrostContext, bifrostReq *schemas. // behavior. See per-model support matrix at // https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html // (mirrors the gate in convertToolConfigFromFiltered for ChatCompletions). - if bedrockToolChoice != nil && bedrockToolChoice.Tool != nil && schemas.IsLlamaModel(bifrostReq.Model) { + if bedrockToolChoice != nil && bedrockToolChoice.Tool != nil && schemas.IsLlamaModelFamily(ctx, bifrostReq.Model) { bedrockToolChoice = nil } if bedrockToolChoice != nil { @@ -2510,7 +2525,7 @@ func ToBedrockResponsesRequest(ctx *schemas.BifrostContext, bifrostReq *schemas. thinkingEnabled := bifrostReq.Params.Reasoning != nil && (bifrostReq.Params.Reasoning.MaxTokens != nil || (bifrostReq.Params.Reasoning.Effort != nil && *bifrostReq.Params.Reasoning.Effort != "none")) - if !schemas.IsLlamaModel(bifrostReq.Model) && !thinkingEnabled { + if !schemas.IsLlamaModelFamily(ctx, bifrostReq.Model) && !thinkingEnabled { bedrockReq.ToolConfig.ToolChoice = &BedrockToolChoice{ Tool: &BedrockToolChoiceTool{ Name: responsesStructuredOutputTool.ToolSpec.Name, diff --git a/core/providers/bedrock/text.go b/core/providers/bedrock/text.go index d31d716ded..b019d2fee2 100644 --- a/core/providers/bedrock/text.go +++ b/core/providers/bedrock/text.go @@ -4,7 +4,6 @@ import ( "strings" "github.com/maximhq/bifrost/core/providers/anthropic" - "github.com/maximhq/bifrost/core/providers/utils" "github.com/maximhq/bifrost/core/schemas" ) @@ -80,7 +79,7 @@ func (request *BedrockTextCompletionRequest) ToBifrostTextCompletionRequest(ctx prompt = strings.Join(parts, "\n\n") } - provider, model := schemas.ParseModelString(request.ModelID, utils.CheckAndSetDefaultProvider(ctx, schemas.Bedrock)) + provider, model := schemas.ParseModelString(request.ModelID, "") bifrostReq := &schemas.BifrostTextCompletionRequest{ Provider: provider, @@ -126,8 +125,7 @@ func (response *BedrockAnthropicTextResponse) ToBifrostTextCompletionResponse() FinishReason: &response.StopReason, }, }, - ExtraFields: schemas.BifrostResponseExtraFields{ - }, + ExtraFields: schemas.BifrostResponseExtraFields{}, } } @@ -149,10 +147,9 @@ func (response *BedrockMistralTextResponse) ToBifrostTextCompletionResponse() *s } return &schemas.BifrostTextCompletionResponse{ - Object: "text_completion", - Choices: choices, - ExtraFields: schemas.BifrostResponseExtraFields{ - }, + Object: "text_completion", + Choices: choices, + ExtraFields: schemas.BifrostResponseExtraFields{}, } } diff --git a/core/providers/bedrock/usage_headers_test.go b/core/providers/bedrock/usage_headers_test.go new file mode 100644 index 0000000000..2cc870b8b0 --- /dev/null +++ b/core/providers/bedrock/usage_headers_test.go @@ -0,0 +1,52 @@ +package bedrock + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestInputTokensFromHeaders(t *testing.T) { + t.Run("reads canonical header", func(t *testing.T) { + n, ok := inputTokensFromHeaders(map[string]string{"X-Amzn-Bedrock-Input-Token-Count": "42"}) + require.True(t, ok) + assert.Equal(t, 42, n) + }) + + t.Run("is case-insensitive", func(t *testing.T) { + n, ok := inputTokensFromHeaders(map[string]string{"x-amzn-bedrock-input-token-count": "7"}) + require.True(t, ok) + assert.Equal(t, 7, n) + }) + + t.Run("trims surrounding whitespace", func(t *testing.T) { + n, ok := inputTokensFromHeaders(map[string]string{"X-Amzn-Bedrock-Input-Token-Count": " 15 "}) + require.True(t, ok) + assert.Equal(t, 15, n) + }) + + t.Run("returns false when header is absent", func(t *testing.T) { + n, ok := inputTokensFromHeaders(map[string]string{"Content-Type": "application/json"}) + assert.False(t, ok) + assert.Equal(t, 0, n) + }) + + t.Run("returns false for nil map", func(t *testing.T) { + n, ok := inputTokensFromHeaders(nil) + assert.False(t, ok) + assert.Equal(t, 0, n) + }) + + t.Run("returns false for non-numeric value", func(t *testing.T) { + n, ok := inputTokensFromHeaders(map[string]string{"X-Amzn-Bedrock-Input-Token-Count": "not-a-number"}) + assert.False(t, ok) + assert.Equal(t, 0, n) + }) + + t.Run("returns false for negative value", func(t *testing.T) { + n, ok := inputTokensFromHeaders(map[string]string{"X-Amzn-Bedrock-Input-Token-Count": "-3"}) + assert.False(t, ok) + assert.Equal(t, 0, n) + }) +} diff --git a/core/providers/bedrock/utils.go b/core/providers/bedrock/utils.go index 8ec4ee51cf..0f4a71c7f1 100644 --- a/core/providers/bedrock/utils.go +++ b/core/providers/bedrock/utils.go @@ -41,6 +41,43 @@ func parseBedrockRegionAndModel(model string) (region, bareModel string) { return "", model } +// resolveBedrockRegion returns the AWS region to use for a request. +// Priority: model-string region prefix > alias-level Region > key-level +// BedrockKeyConfig.Region > DefaultBedrockRegion. The model-string prefix +// stays highest since it's the most explicit signal — when an admin types a +// region into their model ID they expect that to win. +func resolveBedrockRegion(ctx *schemas.BifrostContext, key schemas.Key, model string) string { + if region, _ := parseBedrockRegionAndModel(model); region != "" { + return region + } + if ra := schemas.GetResolvedAlias(ctx); ra != nil && ra.Config != nil && ra.Config.Region != nil { + if v := ra.Config.Region.GetValue(); v != "" { + return v + } + } + if key.BedrockKeyConfig != nil && key.BedrockKeyConfig.Region != nil && key.BedrockKeyConfig.Region.GetValue() != "" { + return key.BedrockKeyConfig.Region.GetValue() + } + return DefaultBedrockRegion +} + +// resolveBedrockARN returns the inference-profile / resource ARN prepended +// to the Bedrock URL path. Priority: alias-level BedrockAliasCfg +// InferenceProfileARN > key-level BedrockKeyConfig.ARN. Returns empty when +// neither is set, in which case getModelPathAndRegion emits the bare model +// path. +func resolveBedrockARN(ctx *schemas.BifrostContext, key schemas.Key) string { + if ra := schemas.GetResolvedAlias(ctx); ra != nil && ra.Config != nil && ra.Config.BedrockAliasCfg != nil && ra.Config.BedrockAliasCfg.InferenceProfileARN != nil { + if v := ra.Config.BedrockAliasCfg.InferenceProfileARN.GetValue(); v != "" { + return v + } + } + if key.BedrockKeyConfig != nil && key.BedrockKeyConfig.ARN != nil { + return key.BedrockKeyConfig.ARN.GetValue() + } + return "" +} + var ( invalidCharRegex = regexp.MustCompile(`[^a-zA-Z0-9\s\-\(\)\[\]]`) multiSpaceRegex = regexp.MustCompile(`\s{2,}`) @@ -157,7 +194,7 @@ func bedrockAliasToolName(ctx context.Context, name string) string { } alias := hash + "_" + semanticName - if bifrostCtx, ok := ctx.(*schemas.BifrostContext); ok && alias != name { + if bifrostCtx, ok := ctx.(*schemas.BifrostContext); ok && bifrostCtx != nil && alias != name { aliases, _ := bifrostCtx.Value(bedrockToolNameAliasKey{}).(map[string]string) if aliases == nil { aliases = make(map[string]string) @@ -170,7 +207,7 @@ func bedrockAliasToolName(ctx context.Context, name string) string { // bedrockRestoreToolName maps a Bedrock wire-name alias back to the caller's tool name. func bedrockRestoreToolName(ctx context.Context, name string) string { - if bifrostCtx, ok := ctx.(*schemas.BifrostContext); ok { + if bifrostCtx, ok := ctx.(*schemas.BifrostContext); ok && bifrostCtx != nil { if aliases, _ := bifrostCtx.Value(bedrockToolNameAliasKey{}).(map[string]string); aliases != nil { if original, ok := aliases[name]; ok { return original @@ -254,15 +291,26 @@ func convertChatParameters(ctx *schemas.BifrostContext, bifrostReq *schemas.Bifr // setting it to default max tokens tokenBudget = anthropic.MinimumReasoningMaxTokens } - if schemas.IsAnthropicModel(bifrostReq.Model) { - if tokenBudget < anthropic.MinimumReasoningMaxTokens { - return fmt.Errorf("reasoning.max_tokens must be >= %d for anthropic", anthropic.MinimumReasoningMaxTokens) + if schemas.IsAnthropicModelFamily(ctx, bifrostReq.Model) { + if anthropic.IsAdaptiveOnlyThinkingModel(bifrostReq.Model) { + bedrockReq.AdditionalModelRequestFields.Set("thinking", map[string]any{ + "type": "adaptive", + }) + // Preserve a co-present effort — these models support effort, + // and the budget is otherwise dropped. + if bifrostReq.Params.Reasoning.Effort != nil && *bifrostReq.Params.Reasoning.Effort != "none" { + setOutputConfigField(bedrockReq.AdditionalModelRequestFields, "effort", anthropic.MapBifrostEffortToAnthropic(*bifrostReq.Params.Reasoning.Effort)) + } + } else { + if tokenBudget < anthropic.MinimumReasoningMaxTokens { + return fmt.Errorf("reasoning.max_tokens must be >= %d for anthropic", anthropic.MinimumReasoningMaxTokens) + } + bedrockReq.AdditionalModelRequestFields.Set("thinking", map[string]any{ + "type": "enabled", + "budget_tokens": tokenBudget, + }) } - bedrockReq.AdditionalModelRequestFields.Set("thinking", map[string]any{ - "type": "enabled", - "budget_tokens": tokenBudget, - }) - } else if schemas.IsNovaModel(bifrostReq.Model) { + } else if schemas.IsNovaModelFamily(ctx, bifrostReq.Model) { minBudgetTokens := MinimumReasoningMaxTokens modelDefaultMaxTokens := providerUtils.GetMaxOutputTokensOrDefault(bifrostReq.Model, DefaultCompletionMaxTokens) defaultMaxTokens := modelDefaultMaxTokens @@ -319,7 +367,7 @@ func convertChatParameters(ctx *schemas.BifrostContext, bifrostReq *schemas.Bifr } } } - if schemas.IsNovaModel(bifrostReq.Model) { + if schemas.IsNovaModelFamily(ctx, bifrostReq.Model) { effort := *bifrostReq.Params.Reasoning.Effort typeStr := "enabled" switch effort { @@ -343,7 +391,7 @@ func convertChatParameters(ctx *schemas.BifrostContext, bifrostReq *schemas.Bifr } bedrockReq.AdditionalModelRequestFields.Set("reasoningConfig", config) - } else if schemas.IsAnthropicModel(bifrostReq.Model) { + } else if schemas.IsAnthropicModelFamily(ctx, bifrostReq.Model) { if anthropic.SupportsAdaptiveThinking(bifrostReq.Model) { // Opus 4.6+: adaptive thinking + output_config.effort effort := anthropic.MapBifrostEffortToAnthropic(*bifrostReq.Params.Reasoning.Effort) @@ -352,8 +400,7 @@ func convertChatParameters(ctx *schemas.BifrostContext, bifrostReq *schemas.Bifr } if bifrostReq.Params.Reasoning.Display != nil { thinkingConfig["display"] = *bifrostReq.Params.Reasoning.Display - } else if anthropic.IsOpus47Plus(bifrostReq.Model) { - // Opus 4.7+ omits reasoning text by default; default to "summarized" + } else if anthropic.IsAdaptiveOnlyThinkingModel(bifrostReq.Model) { thinkingConfig["display"] = "summarized" } bedrockReq.AdditionalModelRequestFields.Set("thinking", thinkingConfig) @@ -371,11 +418,13 @@ func convertChatParameters(ctx *schemas.BifrostContext, bifrostReq *schemas.Bifr } } } else { - if schemas.IsAnthropicModel(bifrostReq.Model) { - bedrockReq.AdditionalModelRequestFields.Set("thinking", map[string]any{ - "type": "disabled", - }) - } else if schemas.IsNovaModel(bifrostReq.Model) { + if schemas.IsAnthropicModelFamily(ctx, bifrostReq.Model) { + if !anthropic.IsFableFamily(bifrostReq.Model) { + bedrockReq.AdditionalModelRequestFields.Set("thinking", map[string]any{ + "type": "disabled", + }) + } + } else if schemas.IsNovaModelFamily(ctx, bifrostReq.Model) { bedrockReq.AdditionalModelRequestFields.Set("reasoningConfig", map[string]any{ "type": "disabled", }) @@ -408,7 +457,7 @@ func convertChatParameters(ctx *schemas.BifrostContext, bifrostReq *schemas.Bifr thinkingEnabled := bifrostReq.Params.Reasoning != nil && (bifrostReq.Params.Reasoning.MaxTokens != nil || (bifrostReq.Params.Reasoning.Effort != nil && *bifrostReq.Params.Reasoning.Effort != "none")) - if !schemas.IsLlamaModel(bifrostReq.Model) && !thinkingEnabled { + if !schemas.IsLlamaModelFamily(ctx, bifrostReq.Model) && !thinkingEnabled { bedrockReq.ToolConfig.ToolChoice = &BedrockToolChoice{ Tool: &BedrockToolChoiceTool{ Name: responseFormatTool.ToolSpec.Name, @@ -1677,7 +1726,12 @@ func convertToolConfig(model string, params *schemas.ChatParameters) *BedrockToo // pre-filtered tool set. convertChatParameters uses this to avoid filtering // twice (once here, once in collectBedrockServerTools). The public // convertToolConfig entry point is a thin wrapper preserved for tests. -func convertToolConfigFromFiltered(ctx context.Context, model string, params *schemas.ChatParameters, filtered []schemas.ChatTool) *BedrockToolConfig { +// +// ctx is the BifrostContext (not context.Context) so the family gates inside +// this function can consult the resolved alias and honor explicit +// AliasConfig.ModelFamily overrides. Test paths may pass nil — family +// detection then falls back to substring matching on model. +func convertToolConfigFromFiltered(ctx *schemas.BifrostContext, model string, params *schemas.ChatParameters, filtered []schemas.ChatTool) *BedrockToolConfig { if params == nil { return nil } @@ -1717,7 +1771,7 @@ func convertToolConfigFromFiltered(ctx context.Context, model string, params *sc } bedrockTools = append(bedrockTools, bedrockTool) - if tool.CacheControl != nil && !schemas.IsNovaModel(model) { + if tool.CacheControl != nil && !schemas.IsNovaModelFamily(ctx, model) { bedrockTools = append(bedrockTools, BedrockTool{ CachePoint: &BedrockCachePoint{ Type: BedrockCachePointTypeDefault, @@ -1774,7 +1828,7 @@ func convertToolConfigFromFiltered(ctx context.Context, model string, params *sc // behavior. See per-model support matrix at // https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html // (mirrors the synthetic-tool gate in convertChatParameters). - if toolChoice != nil && toolChoice.Tool != nil && schemas.IsLlamaModel(model) { + if toolChoice != nil && toolChoice.Tool != nil && schemas.IsLlamaModelFamily(ctx, model) { toolChoice = nil } if toolChoice != nil { diff --git a/core/providers/cohere/chat.go b/core/providers/cohere/chat.go index 208d8fcaf1..9865516d91 100644 --- a/core/providers/cohere/chat.go +++ b/core/providers/cohere/chat.go @@ -250,7 +250,7 @@ func (req *CohereChatRequest) ToBifrostChatRequest(ctx *schemas.BifrostContext) return nil } - provider, model := schemas.ParseModelString(req.Model, providerUtils.CheckAndSetDefaultProvider(ctx, schemas.Cohere)) + provider, model := schemas.ParseModelString(req.Model, "") bifrostReq := &schemas.BifrostChatRequest{ Provider: provider, diff --git a/core/providers/cohere/count_tokens.go b/core/providers/cohere/count_tokens.go index 0a5e1b48e1..1ffa1a5005 100644 --- a/core/providers/cohere/count_tokens.go +++ b/core/providers/cohere/count_tokens.go @@ -5,7 +5,6 @@ import ( "strings" "unicode/utf8" - "github.com/maximhq/bifrost/core/providers/utils" "github.com/maximhq/bifrost/core/schemas" ) @@ -15,7 +14,7 @@ func (req *CohereCountTokensRequest) ToBifrostResponsesRequest(ctx *schemas.Bifr return nil } - provider, model := schemas.ParseModelString(req.Model, utils.CheckAndSetDefaultProvider(ctx, schemas.Cohere)) + provider, model := schemas.ParseModelString(req.Model, "") userRole := schemas.ResponsesInputMessageRoleUser return &schemas.BifrostResponsesRequest{ diff --git a/core/providers/cohere/embedding.go b/core/providers/cohere/embedding.go index a99ef14294..0f8976dcb5 100644 --- a/core/providers/cohere/embedding.go +++ b/core/providers/cohere/embedding.go @@ -1,7 +1,6 @@ package cohere import ( - "github.com/maximhq/bifrost/core/providers/utils" "github.com/maximhq/bifrost/core/schemas" ) @@ -74,7 +73,7 @@ func (req *CohereEmbeddingRequest) ToBifrostEmbeddingRequest(ctx *schemas.Bifros return nil } - provider, model := schemas.ParseModelString(req.Model, utils.CheckAndSetDefaultProvider(ctx, schemas.Cohere)) + provider, model := schemas.ParseModelString(req.Model, "") bifrostReq := &schemas.BifrostEmbeddingRequest{ Provider: provider, diff --git a/core/providers/cohere/models.go b/core/providers/cohere/models.go index 3b285f97b6..3312032888 100644 --- a/core/providers/cohere/models.go +++ b/core/providers/cohere/models.go @@ -45,7 +45,7 @@ type CohereRerankMeta struct { Tokens *CohereTokenUsage `json:"tokens,omitempty"` } -func (response *CohereListModelsResponse) ToBifrostListModelsResponse(providerKey schemas.ModelProvider, allowedModels schemas.WhiteList, blacklistedModels schemas.BlackList, aliases map[string]string, unfiltered bool) *schemas.BifrostListModelsResponse { +func (response *CohereListModelsResponse) ToBifrostListModelsResponse(providerKey schemas.ModelProvider, allowedModels schemas.WhiteList, blacklistedModels schemas.BlackList, aliases schemas.KeyAliases, unfiltered bool) *schemas.BifrostListModelsResponse { if response == nil { return nil } diff --git a/core/providers/cohere/rerank.go b/core/providers/cohere/rerank.go index b820e3796b..bf3f759c2d 100644 --- a/core/providers/cohere/rerank.go +++ b/core/providers/cohere/rerank.go @@ -4,7 +4,6 @@ import ( "sort" "github.com/bytedance/sonic" - "github.com/maximhq/bifrost/core/providers/utils" "github.com/maximhq/bifrost/core/schemas" "gopkg.in/yaml.v3" ) @@ -43,7 +42,7 @@ func (req *CohereRerankRequest) ToBifrostRerankRequest(ctx *schemas.BifrostConte return nil } - provider, model := schemas.ParseModelString(req.Model, utils.CheckAndSetDefaultProvider(ctx, schemas.Cohere)) + provider, model := schemas.ParseModelString(req.Model, "") bifrostReq := &schemas.BifrostRerankRequest{ Provider: provider, diff --git a/core/providers/elevenlabs/models.go b/core/providers/elevenlabs/models.go index f762d97ee8..7e3c1f8d34 100644 --- a/core/providers/elevenlabs/models.go +++ b/core/providers/elevenlabs/models.go @@ -7,7 +7,7 @@ import ( "github.com/maximhq/bifrost/core/schemas" ) -func (response *ElevenlabsListModelsResponse) ToBifrostListModelsResponse(providerKey schemas.ModelProvider, allowedModels schemas.WhiteList, blacklistedModels schemas.BlackList, aliases map[string]string, unfiltered bool) *schemas.BifrostListModelsResponse { +func (response *ElevenlabsListModelsResponse) ToBifrostListModelsResponse(providerKey schemas.ModelProvider, allowedModels schemas.WhiteList, blacklistedModels schemas.BlackList, aliases schemas.KeyAliases, unfiltered bool) *schemas.BifrostListModelsResponse { if response == nil { return nil } diff --git a/core/providers/gemini/batch.go b/core/providers/gemini/batch.go index 8f0405e524..be832ef52f 100644 --- a/core/providers/gemini/batch.go +++ b/core/providers/gemini/batch.go @@ -236,16 +236,6 @@ func parseGeminiTimestamp(timestamp string) int64 { return t.Unix() } -// extractBatchIDFromName extracts the batch ID from the full resource name. -// e.g., "batches/abc123" -> "abc123" -func extractBatchIDFromName(name string) string { - if name == "" { - return "" - } - parts := strings.Split(name, "/") - return parts[len(parts)-1] -} - // downloadBatchResultsFile downloads and parses a batch results file from Gemini. // Returns the parsed result items from the JSONL file and any parse errors encountered. func (provider *GeminiProvider) downloadBatchResultsFile(ctx context.Context, key schemas.Key, fileName string) ([]schemas.BatchResultItem, []schemas.BatchError, *schemas.BifrostError) { diff --git a/core/providers/gemini/embedding.go b/core/providers/gemini/embedding.go index 906b995c68..438f0dee65 100644 --- a/core/providers/gemini/embedding.go +++ b/core/providers/gemini/embedding.go @@ -1,7 +1,6 @@ package gemini import ( - "github.com/maximhq/bifrost/core/providers/utils" "github.com/maximhq/bifrost/core/schemas" ) @@ -189,7 +188,7 @@ func (request *GeminiGenerationRequest) ToBifrostEmbeddingRequest(ctx *schemas.B return nil } - provider, model := schemas.ParseModelString(request.Model, utils.CheckAndSetDefaultProvider(ctx, schemas.Gemini)) + provider, model := schemas.ParseModelString(request.Model, "") // Create the embedding request bifrostReq := &schemas.BifrostEmbeddingRequest{ diff --git a/core/providers/gemini/gemini.go b/core/providers/gemini/gemini.go index 83f5e7c870..742436e972 100644 --- a/core/providers/gemini/gemini.go +++ b/core/providers/gemini/gemini.go @@ -2754,7 +2754,8 @@ func (provider *GeminiProvider) batchListByKey(ctx *schemas.BifrostContext, key data := make([]schemas.BifrostBatchRetrieveResponse, 0, len(geminiResp.Operations)) for _, batch := range geminiResp.Operations { data = append(data, schemas.BifrostBatchRetrieveResponse{ - ID: extractBatchIDFromName(batch.Name), + // Full name (batches/), matching create/retrieve so the id is stable. + ID: batch.Name, Object: "batch", Status: ToBifrostBatchStatus(batch.Metadata.State), CreatedAt: parseGeminiTimestamp(batch.Metadata.CreateTime), diff --git a/core/providers/gemini/gemini_test.go b/core/providers/gemini/gemini_test.go index 56c9514327..c1e3728f66 100644 --- a/core/providers/gemini/gemini_test.go +++ b/core/providers/gemini/gemini_test.go @@ -2498,6 +2498,192 @@ func TestResponsesAPIParallelFunctionCalling(t *testing.T) { assert.Contains(t, responseStr, "Google", "Response should contain tab content") }, }, + { + name: "ResponsesAPI_FunctionCallOutput_MultimodalBlocks_ImagePreserved", + input: &schemas.BifrostResponsesRequest{ + Provider: schemas.Gemini, + Model: "gemini-3-pro-preview", + Input: []schemas.ResponsesMessage{ + { + Role: schemas.Ptr(schemas.ResponsesInputMessageRoleUser), + Type: schemas.Ptr(schemas.ResponsesMessageTypeMessage), + Content: &schemas.ResponsesMessageContent{ + ContentStr: schemas.Ptr("What color is the image the tool returned?"), + }, + }, + { + Type: schemas.Ptr(schemas.ResponsesMessageTypeFunctionCall), + ResponsesToolMessage: &schemas.ResponsesToolMessage{ + CallID: schemas.Ptr("c1"), + Name: schemas.Ptr("read_file"), + Arguments: schemas.Ptr(`{}`), + }, + }, + { + Type: schemas.Ptr(schemas.ResponsesMessageTypeFunctionCallOutput), + ResponsesToolMessage: &schemas.ResponsesToolMessage{ + CallID: schemas.Ptr("c1"), + Output: &schemas.ResponsesToolMessageOutputStruct{ + // Mixed text + image blocks (OpenAI Responses API format) + ResponsesFunctionToolCallOutputBlocks: []schemas.ResponsesMessageContentBlock{ + { + Type: schemas.ResponsesInputMessageContentBlockTypeText, + Text: schemas.Ptr("result:"), + }, + { + Type: schemas.ResponsesInputMessageContentBlockTypeImage, + ResponsesInputMessageContentBlockImage: &schemas.ResponsesInputMessageContentBlockImage{ + // 1x1 red PNG + ImageURL: schemas.Ptr("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="), + }, + }, + }, + }, + }, + }, + }, + }, + validate: func(t *testing.T, result *gemini.GeminiGenerationRequest) { + var fr *gemini.FunctionResponse + for i := range result.Contents { + for _, p := range result.Contents[i].Parts { + if p.FunctionResponse != nil { + fr = p.FunctionResponse + break + } + } + } + require.NotNil(t, fr, "Should have a functionResponse") + assert.Equal(t, "read_file", fr.Name) + + // Text block preserved in the structured response + responseStr := string(fr.Response) + assert.Contains(t, responseStr, "result:", "Text output should be preserved") + + // Image block preserved as a nested inlineData part (NOT dropped) on Gemini 3+ + require.Len(t, fr.Parts, 1, "Image block should be attached as a functionResponse part") + require.NotNil(t, fr.Parts[0].InlineData, "Media part must carry inlineData") + assert.Equal(t, "image/png", fr.Parts[0].InlineData.MIMEType) + assert.NotEmpty(t, fr.Parts[0].InlineData.Data, "Image base64 data must be present") + assert.NotEmpty(t, fr.Parts[0].InlineData.DisplayName, "Blob should have a displayName") + + // No $ref must be emitted: the Gemini Developer API rejects the $ref form + // ("does not match to a display_name"); Gemini 3 reads media directly from parts. + assert.NotContains(t, responseStr, "$ref", "Response must NOT contain a $ref placeholder") + }, + }, + { + name: "ResponsesAPI_FunctionCallOutput_MultimodalBlocks_DroppedForOlderModel", + input: &schemas.BifrostResponsesRequest{ + Provider: schemas.Gemini, + Model: "gemini-2.5-flash", // not Gemini 3 → multimodal tool output unsupported + Input: []schemas.ResponsesMessage{ + { + Type: schemas.Ptr(schemas.ResponsesMessageTypeFunctionCall), + ResponsesToolMessage: &schemas.ResponsesToolMessage{ + CallID: schemas.Ptr("c1"), + Name: schemas.Ptr("read_file"), + Arguments: schemas.Ptr(`{}`), + }, + }, + { + Type: schemas.Ptr(schemas.ResponsesMessageTypeFunctionCallOutput), + ResponsesToolMessage: &schemas.ResponsesToolMessage{ + CallID: schemas.Ptr("c1"), + Output: &schemas.ResponsesToolMessageOutputStruct{ + ResponsesFunctionToolCallOutputBlocks: []schemas.ResponsesMessageContentBlock{ + { + Type: schemas.ResponsesInputMessageContentBlockTypeText, + Text: schemas.Ptr("result:"), + }, + { + Type: schemas.ResponsesInputMessageContentBlockTypeImage, + ResponsesInputMessageContentBlockImage: &schemas.ResponsesInputMessageContentBlockImage{ + ImageURL: schemas.Ptr("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="), + }, + }, + }, + }, + }, + }, + }, + }, + validate: func(t *testing.T, result *gemini.GeminiGenerationRequest) { + var fr *gemini.FunctionResponse + for i := range result.Contents { + for _, p := range result.Contents[i].Parts { + if p.FunctionResponse != nil { + fr = p.FunctionResponse + break + } + } + } + require.NotNil(t, fr, "Should have a functionResponse") + // Text is still preserved, but the image is dropped (older models reject + // multimodal function responses with a hard 400), so no parts are emitted. + assert.Contains(t, string(fr.Response), "result:", "Text output should be preserved") + assert.Empty(t, fr.Parts, "Media must be dropped for non-Gemini-3 models (no 400)") + }, + }, + { + name: "ResponsesAPI_FunctionCallOutput_MultimodalBlocks_VertexEmitsRef", + input: &schemas.BifrostResponsesRequest{ + Provider: schemas.Vertex, // Vertex AI supports (and requires) the $ref form + Model: "gemini-3-pro-preview", + Input: []schemas.ResponsesMessage{ + { + Type: schemas.Ptr(schemas.ResponsesMessageTypeFunctionCall), + ResponsesToolMessage: &schemas.ResponsesToolMessage{ + CallID: schemas.Ptr("c1"), + Name: schemas.Ptr("read_file"), + Arguments: schemas.Ptr(`{}`), + }, + }, + { + Type: schemas.Ptr(schemas.ResponsesMessageTypeFunctionCallOutput), + ResponsesToolMessage: &schemas.ResponsesToolMessage{ + CallID: schemas.Ptr("c1"), + Output: &schemas.ResponsesToolMessageOutputStruct{ + ResponsesFunctionToolCallOutputBlocks: []schemas.ResponsesMessageContentBlock{ + { + Type: schemas.ResponsesInputMessageContentBlockTypeText, + Text: schemas.Ptr("result:"), + }, + { + Type: schemas.ResponsesInputMessageContentBlockTypeImage, + ResponsesInputMessageContentBlockImage: &schemas.ResponsesInputMessageContentBlockImage{ + ImageURL: schemas.Ptr("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="), + }, + }, + }, + }, + }, + }, + }, + }, + validate: func(t *testing.T, result *gemini.GeminiGenerationRequest) { + var fr *gemini.FunctionResponse + for i := range result.Contents { + for _, p := range result.Contents[i].Parts { + if p.FunctionResponse != nil { + fr = p.FunctionResponse + break + } + } + } + require.NotNil(t, fr, "Should have a functionResponse") + require.Len(t, fr.Parts, 1, "Image must be attached as a functionResponse part on Vertex") + require.NotNil(t, fr.Parts[0].InlineData) + dn := fr.Parts[0].InlineData.DisplayName + require.NotEmpty(t, dn, "Blob must have a displayName") + + // Vertex DOES emit the $ref, pointing at the blob's displayName. + responseStr := string(fr.Response) + assert.Contains(t, responseStr, "result:", "Text output should be preserved") + assert.Contains(t, responseStr, "$ref", "Vertex must emit a $ref into the response") + assert.Contains(t, responseStr, dn, "The $ref must point at the blob displayName") + }, + }, } for _, tt := range tests { @@ -3813,6 +3999,148 @@ func TestFunctionCallingConfigModeAny_RoundTrip(t *testing.T) { } } +// TestMultimodalFunctionResponse_RoundTrip verifies that an image returned by a tool +// inside functionResponse.parts survives the full +// GeminiGenerationRequest → BifrostResponsesRequest → GeminiGenerationRequest round-trip +// (Gemini 3 multimodal function responses), instead of being dropped or collapsed to text. +func TestMultimodalFunctionResponse_RoundTrip(t *testing.T) { + const redPNG = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" + + geminiReq := &gemini.GeminiGenerationRequest{ + Model: "gemini-3-flash-preview", + Contents: []gemini.Content{ + {Role: "user", Parts: []*gemini.Part{{Text: "What color is the tool image?"}}}, + {Role: "model", Parts: []*gemini.Part{{ + FunctionCall: &gemini.FunctionCall{ID: "c1", Name: "read_file", Args: json.RawMessage(`{}`)}, + }}}, + {Role: "user", Parts: []*gemini.Part{{ + FunctionResponse: &gemini.FunctionResponse{ + ID: "c1", + Name: "read_file", + Response: json.RawMessage(`{"media_0":{"$ref":"media_0"},"output":"result:"}`), + Parts: []*gemini.Part{{ + InlineData: &gemini.Blob{MIMEType: "image/png", DisplayName: "media_0", Data: redPNG}, + }}, + }, + }}}, + }, + } + + // --- Gemini -> Bifrost: image must be reconstructed as content blocks --- + bifrostCtx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + bifrostReq := geminiReq.ToBifrostResponsesRequest(bifrostCtx) + require.NotNil(t, bifrostReq) + + var outputMsg *schemas.ResponsesMessage + for i := range bifrostReq.Input { + m := &bifrostReq.Input[i] + if m.Type != nil && *m.Type == schemas.ResponsesMessageTypeFunctionCallOutput { + outputMsg = m + break + } + } + require.NotNil(t, outputMsg, "Should have a function_call_output message") + require.NotNil(t, outputMsg.ResponsesToolMessage.Output) + blocks := outputMsg.ResponsesToolMessage.Output.ResponsesFunctionToolCallOutputBlocks + require.NotEmpty(t, blocks, "Output must be reconstructed as content blocks (not collapsed to a string)") + + var hasText, hasImage bool + for _, b := range blocks { + if b.Type == schemas.ResponsesInputMessageContentBlockTypeText && b.Text != nil { + assert.Equal(t, "result:", *b.Text) + hasText = true + } + if b.Type == schemas.ResponsesInputMessageContentBlockTypeImage && + b.ResponsesInputMessageContentBlockImage != nil && + b.ResponsesInputMessageContentBlockImage.ImageURL != nil { + assert.Contains(t, *b.ResponsesInputMessageContentBlockImage.ImageURL, redPNG, "Image base64 must be preserved") + hasImage = true + } + } + assert.True(t, hasText, "Text block must be preserved") + assert.True(t, hasImage, "Image block must be preserved") + + // --- Bifrost -> Gemini: image must land back in functionResponse.parts --- + roundTrip, err := gemini.ToGeminiResponsesRequest(bifrostReq) + require.NoError(t, err) + require.NotNil(t, roundTrip) + + var fr *gemini.FunctionResponse + for i := range roundTrip.Contents { + for _, p := range roundTrip.Contents[i].Parts { + if p.FunctionResponse != nil { + fr = p.FunctionResponse + break + } + } + } + require.NotNil(t, fr, "Round-trip must still have a functionResponse") + require.Len(t, fr.Parts, 1, "Image must round-trip back into functionResponse.parts") + require.NotNil(t, fr.Parts[0].InlineData) + assert.Equal(t, "image/png", fr.Parts[0].InlineData.MIMEType) + assert.NotEmpty(t, fr.Parts[0].InlineData.Data, "Image data must survive the full round-trip") + assert.Contains(t, string(fr.Response), "result:", "Text output must survive the full round-trip") +} + +// TestMultimodalFunctionResponse_PreservesNonRefFields verifies that non-$ref fields in a +// multimodal functionResponse.response (the Gemini spec allows any keys, not just "output") +// survive the Gemini -> Bifrost -> Gemini round-trip instead of being dropped, while the +// $ref placeholders (which point at the media parts) are not re-emitted. +func TestMultimodalFunctionResponse_PreservesNonRefFields(t *testing.T) { + const redPNG = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" + + geminiReq := &gemini.GeminiGenerationRequest{ + Model: "gemini-3-flash-preview", + Contents: []gemini.Content{ + {Role: "user", Parts: []*gemini.Part{{Text: "weather?"}}}, + {Role: "model", Parts: []*gemini.Part{{ + FunctionCall: &gemini.FunctionCall{ID: "c1", Name: "get_weather", Args: json.RawMessage(`{}`)}, + }}}, + {Role: "user", Parts: []*gemini.Part{{ + FunctionResponse: &gemini.FunctionResponse{ + ID: "c1", + Name: "get_weather", + // Tool returns real data fields (temp, unit) plus an image reference. + Response: json.RawMessage(`{"temp":72,"unit":"F","chart_ref":{"$ref":"chart.png"}}`), + Parts: []*gemini.Part{{ + InlineData: &gemini.Blob{MIMEType: "image/png", DisplayName: "chart.png", Data: redPNG}, + }}, + }, + }}}, + }, + } + + bifrostCtx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + bifrostReq := geminiReq.ToBifrostResponsesRequest(bifrostCtx) + require.NotNil(t, bifrostReq) + + roundTrip, err := gemini.ToGeminiResponsesRequest(bifrostReq) + require.NoError(t, err) + require.NotNil(t, roundTrip) + + var fr *gemini.FunctionResponse + for i := range roundTrip.Contents { + for _, p := range roundTrip.Contents[i].Parts { + if p.FunctionResponse != nil { + fr = p.FunctionResponse + break + } + } + } + require.NotNil(t, fr, "Round-trip must still have a functionResponse") + + // Image survives as a part. + require.Len(t, fr.Parts, 1, "Image must round-trip into functionResponse.parts") + require.NotNil(t, fr.Parts[0].InlineData) + + // Non-$ref data fields survive; the $ref placeholder is not re-emitted. + responseStr := string(fr.Response) + assert.Contains(t, responseStr, "72", "temp must survive the round-trip") + assert.Contains(t, responseStr, `"F"`, "unit must survive the round-trip") + assert.NotContains(t, responseStr, "$ref", "the $ref placeholder must not be re-emitted (Gemini provider)") + assert.NotContains(t, responseStr, "chart_ref", "the media-ref key must not leak into the response") +} + // TestImageSizeRoundtrip verifies that imageSize and aspectRatio survive the // GeminiGenerationRequest → BifrostImageGenerationRequest → GeminiGenerationRequest round-trip // and that the outbound imageSize is always uppercase ("2K" not "2k"). diff --git a/core/providers/gemini/images.go b/core/providers/gemini/images.go index c94d33a327..7da0abb7db 100644 --- a/core/providers/gemini/images.go +++ b/core/providers/gemini/images.go @@ -20,7 +20,7 @@ func (request *GeminiGenerationRequest) ToBifrostImageGenerationRequest(ctx *sch // Parse provider from model string (e.g., "openai/gpt-image-1" -> provider="openai", model="gpt-image-1") // This allows cross-provider routing through the GenAI endpoint - provider, model := schemas.ParseModelString(request.Model, utils.CheckAndSetDefaultProvider(ctx, schemas.Gemini)) + provider, model := schemas.ParseModelString(request.Model, "") bifrostReq := &schemas.BifrostImageGenerationRequest{ Provider: provider, @@ -114,7 +114,7 @@ func (request *GeminiGenerationRequest) ToBifrostImageEditRequest(ctx *schemas.B // Parse provider from model string (e.g., "openai/gpt-image-1" -> provider="openai", model="gpt-image-1") // This allows cross-provider routing through the GenAI endpoint - provider, model := schemas.ParseModelString(request.Model, utils.CheckAndSetDefaultProvider(ctx, schemas.Gemini)) + provider, model := schemas.ParseModelString(request.Model, "") bifrostReq := &schemas.BifrostImageEditRequest{ Provider: provider, diff --git a/core/providers/gemini/models.go b/core/providers/gemini/models.go index 7b9f6410eb..e88387dc66 100644 --- a/core/providers/gemini/models.go +++ b/core/providers/gemini/models.go @@ -17,7 +17,7 @@ func toGeminiModelResourceName(modelID string) string { return "models/" + modelID } -func (response *GeminiListModelsResponse) ToBifrostListModelsResponse(providerKey schemas.ModelProvider, allowedModels schemas.WhiteList, blacklistedModels schemas.BlackList, aliases map[string]string, unfiltered bool) *schemas.BifrostListModelsResponse { +func (response *GeminiListModelsResponse) ToBifrostListModelsResponse(providerKey schemas.ModelProvider, allowedModels schemas.WhiteList, blacklistedModels schemas.BlackList, aliases schemas.KeyAliases, unfiltered bool) *schemas.BifrostListModelsResponse { if response == nil { return nil } diff --git a/core/providers/gemini/responses.go b/core/providers/gemini/responses.go index 67b09eb1d9..f594860c08 100644 --- a/core/providers/gemini/responses.go +++ b/core/providers/gemini/responses.go @@ -18,7 +18,7 @@ func (request *GeminiGenerationRequest) ToBifrostResponsesRequest(ctx *schemas.B return nil } - provider, model := schemas.ParseModelString(request.Model, providerUtils.CheckAndSetDefaultProvider(ctx, schemas.Gemini)) + provider, model := schemas.ParseModelString(request.Model, "") // Create the BifrostResponsesRequest bifrostReq := &schemas.BifrostResponsesRequest{ @@ -116,7 +116,7 @@ func ToGeminiResponsesRequest(bifrostReq *schemas.BifrostResponsesRequest) (*Gem // Convert ResponsesInput messages to Gemini contents if bifrostReq.Input != nil { - contents, systemInstruction, err := convertResponsesMessagesToGeminiContents(bifrostReq.Input) + contents, systemInstruction, err := convertResponsesMessagesToGeminiContents(bifrostReq.Input, bifrostReq.Model, bifrostReq.Provider) if err != nil { return nil, err } @@ -1942,6 +1942,45 @@ func convertGeminiSystemInstructionToResponsesMessage(systemInstruction *Content } } +// stripFunctionResponseMediaRefs returns the textual payload of a Gemini functionResponse.Response +// to carry alongside reconstructed media blocks. It drops top-level keys whose value is a +// {"$ref": ...} placeholder — those reference the media we materialize as content blocks, and +// re-emitting them would re-trigger the Gemini Developer API "$ref" bug — while preserving every +// other field so multimodal tool results are not lossy. The Gemini spec lets callers use any keys +// (output, result, error, ...), not just "output". When only the conventional "output" field +// remains it is unwrapped to keep the common round-trip shape; a media-only response yields "". +func stripFunctionResponseMediaRefs(response json.RawMessage) string { + if len(response) == 0 { + return "" + } + root := providerUtils.GetJSONField(response, "@this") + if !root.IsObject() { + return string(response) + } + + cleaned := []byte(response) + remaining := 0 + for key, value := range root.Map() { + if value.IsObject() && value.Get("$ref").Exists() { + if updated, err := providerUtils.DeleteJSONField(cleaned, key); err == nil { + cleaned = updated + } + continue + } + remaining++ + } + + if remaining == 0 { + return "" // media-only result; the forward path emits an empty "output" placeholder + } + if remaining == 1 { + if out := providerUtils.GetJSONField(cleaned, "output"); out.Exists() { + return out.String() + } + } + return string(cleaned) +} + func convertGeminiContentsToResponsesMessages(contents []Content) []schemas.ResponsesMessage { var messages []schemas.ResponsesMessage // Track function call IDs by name to match with responses @@ -2025,13 +2064,48 @@ func convertGeminiContentsToResponsesMessages(contents []Content) []schemas.Resp } } + output := &schemas.ResponsesToolMessageOutputStruct{} + if len(part.FunctionResponse.Parts) > 0 { + // Multimodal function response (Gemini 3 series): the tool returned images/files + // nested in functionResponse.parts. Reconstruct them as content blocks so the media + // is preserved on the way in, instead of being collapsed to the text "output" field. + // Mirrors the forward conversion in convertResponsesMessagesToGeminiContents. + var blocks []schemas.ResponsesMessageContentBlock + // Preserve the structured response text alongside the media. The Gemini spec allows + // any keys (output, result, error, ...), so keep the whole response object minus the + // {"$ref": ...} placeholders (those point at the media we materialize as blocks below). + if textPayload := stripFunctionResponseMediaRefs(part.FunctionResponse.Response); textPayload != "" { + blocks = append(blocks, schemas.ResponsesMessageContentBlock{ + Type: schemas.ResponsesInputMessageContentBlockTypeText, + Text: &textPayload, + }) + } + for _, p := range part.FunctionResponse.Parts { + var block *schemas.ResponsesMessageContentBlock + switch { + case p.InlineData != nil: + block = convertGeminiInlineDataToContentBlock(p.InlineData) + case p.FileData != nil: + block = convertGeminiFileDataToContentBlock(p.FileData) + } + if block != nil { + blocks = append(blocks, *block) + } + } + if len(blocks) > 0 { + output.ResponsesFunctionToolCallOutputBlocks = blocks + } else { + output.ResponsesToolCallOutputStr = &responseStr + } + } else { + output.ResponsesToolCallOutputStr = &responseStr + } + msg := schemas.ResponsesMessage{ Type: schemas.Ptr(schemas.ResponsesMessageTypeFunctionCallOutput), ResponsesToolMessage: &schemas.ResponsesToolMessage{ CallID: &responseID, - Output: &schemas.ResponsesToolMessageOutputStruct{ - ResponsesToolCallOutputStr: &responseStr, - }, + Output: output, }, } @@ -2967,8 +3041,13 @@ func convertResponsesToolChoiceToGemini(toolChoice *schemas.ResponsesToolChoice) return config } -// convertResponsesMessagesToGeminiContents converts Responses messages to Gemini contents -func convertResponsesMessagesToGeminiContents(messages []schemas.ResponsesMessage) ([]Content, *Content, error) { +// convertResponsesMessagesToGeminiContents converts Responses messages to Gemini contents. +// model is used to gate features that are only valid on Gemini 3+ (e.g. multimodal function +// responses, where a tool returns images/files nested in functionResponse.parts). provider +// distinguishes Vertex AI from the Gemini Developer API, which differ in how multimodal +// function responses must be referenced (see the FunctionCallOutput handling below). +func convertResponsesMessagesToGeminiContents(messages []schemas.ResponsesMessage, model string, provider schemas.ModelProvider) ([]Content, *Content, error) { + isVertex := provider == schemas.Vertex // if only system / developer message is there, convert it to user message (since openai allows it) if len(messages) == 1 && messages[0].Role != nil && (*messages[0].Role == schemas.ResponsesInputMessageRoleSystem || *messages[0].Role == schemas.ResponsesInputMessageRoleDeveloper) { content := Content{Role: "user"} @@ -3150,6 +3229,9 @@ func convertResponsesMessagesToGeminiContents(messages []schemas.ResponsesMessag // must be sent in a single message with only functionResponse parts (no text/content parts) if msg.ResponsesToolMessage.CallID != nil { responseMap := make(map[string]any) + // Multimodal blocks (images, files) returned by the function are collected here + // and attached to FunctionResponse.Parts (Gemini 3+ only). + var funcMediaParts []*Part // Extract output from ResponsesToolMessage.Output if msg.ResponsesToolMessage.Output != nil && msg.ResponsesToolMessage.Output.ResponsesToolCallOutputStr != nil { @@ -3160,12 +3242,48 @@ func convertResponsesMessagesToGeminiContents(messages []schemas.ResponsesMessag responseMap["output"] = output } } else if msg.ResponsesToolMessage.Output != nil && msg.ResponsesToolMessage.Output.ResponsesFunctionToolCallOutputBlocks != nil { - // Handle structured output blocks (e.g. from Anthropic Responses API format - // where output is an array of content blocks like [{"type":"input_text","text":"..."}]) + // Handle structured output blocks (e.g. from the OpenAI/Anthropic Responses API + // format where output is an array of content blocks like + // [{"type":"input_text","text":"..."}, {"type":"input_image","image_url":"..."}]). + // + // Text blocks go into responseMap["output"]. Multimodal blocks (images, files) + // cannot live inside the structured response; per the Gemini docs they must be + // nested as sibling FunctionResponse.Parts (inlineData/fileData). This is a + // Gemini 3+ feature, so for older models we drop the media and keep text only + // (sending parts to e.g. gemini-2.5 returns a hard "not supported" 400). + // + // Referencing the media from the structured response differs by provider: + // - Vertex AI: emit a "_ref": {"$ref": ""} entry into + // the response (the documented format; Vertex resolves the ref to the part). + // - Gemini Developer API: do NOT emit $ref — the API rejects it + // ("does not match to a display_name", a known upstream bug). The model + // still reads the media directly from parts. + supportsMultimodalToolOutput := isGemini3Plus(model) var textParts []string for _, block := range msg.ResponsesToolMessage.Output.ResponsesFunctionToolCallOutputBlocks { if block.Text != nil && *block.Text != "" { textParts = append(textParts, *block.Text) + continue + } + if !supportsMultimodalToolOutput { + continue // older models can't accept media in a function response + } + mediaPart, err := convertContentBlockToGeminiPart(block) + if err != nil { + return nil, nil, fmt.Errorf("failed to convert function output content block: %w", err) + } + if mediaPart == nil { + continue + } + displayName := fmt.Sprintf("media_%d", len(funcMediaParts)) + if mediaPart.InlineData != nil { + mediaPart.InlineData.DisplayName = displayName + } else if mediaPart.FileData != nil { + mediaPart.FileData.DisplayName = displayName + } + funcMediaParts = append(funcMediaParts, mediaPart) + if isVertex { + responseMap[displayName+"_ref"] = map[string]string{"$ref": displayName} } } if len(textParts) > 0 { @@ -3175,13 +3293,13 @@ func convertResponsesMessagesToGeminiContents(messages []schemas.ResponsesMessag } else { responseMap["output"] = combined } - } else { - // Fallback for non-text blocks (e.g. images, files): marshal the raw blocks - // so responseMap["output"] is never left empty when blocks are present - rawBlocks, err := providerUtils.MarshalSorted(msg.ResponsesToolMessage.Output.ResponsesFunctionToolCallOutputBlocks) - if err == nil && len(rawBlocks) > 0 { - responseMap["output"] = json.RawMessage(rawBlocks) - } + } else if len(funcMediaParts) > 0 { + // Media-only result: the content lives in parts. We intentionally emit + // {"output": ""} rather than leaving response as {} — an empty object would + // be treated by Gemini as the full (empty) function output. The reverse + // converter's stripFunctionResponseMediaRefs reads this "" back as no text + // block, so the media-only round-trip stays clean. + responseMap["output"] = "" } } else if msg.Content != nil && msg.Content.ContentStr != nil { // Fallback to Content.ContentStr for backward compatibility @@ -3209,6 +3327,7 @@ func convertResponsesMessagesToGeminiContents(messages []schemas.ResponsesMessag Name: funcName, Response: json.RawMessage(responseBytes), ID: *msg.ResponsesToolMessage.CallID, + Parts: funcMediaParts, }, } pendingFunctionResponseParts = append(pendingFunctionResponseParts, part) diff --git a/core/providers/gemini/speech.go b/core/providers/gemini/speech.go index d4683c250f..416c30239c 100644 --- a/core/providers/gemini/speech.go +++ b/core/providers/gemini/speech.go @@ -11,7 +11,7 @@ import ( // ToBifrostSpeechRequest converts a GeminiGenerationRequest to a BifrostSpeechRequest func (request *GeminiGenerationRequest) ToBifrostSpeechRequest(ctx *schemas.BifrostContext) *schemas.BifrostSpeechRequest { - provider, model := schemas.ParseModelString(request.Model, utils.CheckAndSetDefaultProvider(ctx, schemas.Gemini)) + provider, model := schemas.ParseModelString(request.Model, "") bifrostReq := &schemas.BifrostSpeechRequest{ Provider: provider, diff --git a/core/providers/gemini/transcription.go b/core/providers/gemini/transcription.go index 0548a3f512..7388b077c3 100644 --- a/core/providers/gemini/transcription.go +++ b/core/providers/gemini/transcription.go @@ -10,7 +10,7 @@ import ( // ToBifrostTranscriptionRequest converts a GeminiGenerationRequest to a BifrostTranscriptionRequest func (request *GeminiGenerationRequest) ToBifrostTranscriptionRequest(ctx *schemas.BifrostContext) (*schemas.BifrostTranscriptionRequest, error) { - provider, model := schemas.ParseModelString(request.Model, utils.CheckAndSetDefaultProvider(ctx, schemas.Gemini)) + provider, model := schemas.ParseModelString(request.Model, "") bifrostReq := &schemas.BifrostTranscriptionRequest{ Provider: provider, diff --git a/core/providers/gemini/types.go b/core/providers/gemini/types.go index eab2772ea4..2f0f4df7e4 100644 --- a/core/providers/gemini/types.go +++ b/core/providers/gemini/types.go @@ -1363,6 +1363,10 @@ func (p *Part) UnmarshalJSON(data []byte) error { FunctionCall *FunctionCall `json:"functionCall,omitempty"` FunctionResponse *FunctionResponse `json:"functionResponse,omitempty"` Text string `json:"text,omitempty"` + // snake_case fallbacks: the google-genai SDK serializes FunctionResponsePart + // (nested inside functionResponse.parts) with snake_case keys, unlike top-level parts. + InlineDataSnake *Blob `json:"inline_data,omitempty"` + FileDataSnake *FileData `json:"file_data,omitempty"` } var aux PartAlias @@ -1373,7 +1377,13 @@ func (p *Part) UnmarshalJSON(data []byte) error { p.VideoMetadata = aux.VideoMetadata p.Thought = aux.Thought p.InlineData = aux.InlineData + if p.InlineData == nil { + p.InlineData = aux.InlineDataSnake + } p.FileData = aux.FileData + if p.FileData == nil { + p.FileData = aux.FileDataSnake + } p.CodeExecutionResult = aux.CodeExecutionResult p.ExecutableCode = aux.ExecutableCode p.FunctionCall = aux.FunctionCall @@ -1415,12 +1425,16 @@ type Blob struct { MIMEType string `json:"mimeType,omitempty"` } -// UnmarshalJSON custom unmarshaler for Blob to handle URL-safe base64 +// UnmarshalJSON custom unmarshaler for Blob to handle URL-safe base64. +// Also accepts the snake_case keys (mime_type, display_name) the google-genai SDK +// emits inside functionResponse.parts (FunctionResponseBlob), preferring camelCase. func (b *Blob) UnmarshalJSON(data []byte) error { type BlobAlias struct { - DisplayName string `json:"displayName,omitempty"` - Data string `json:"data,omitempty"` - MIMEType string `json:"mimeType,omitempty"` + DisplayName string `json:"displayName,omitempty"` + DisplayNameSnake string `json:"display_name,omitempty"` + Data string `json:"data,omitempty"` + MIMEType string `json:"mimeType,omitempty"` + MIMETypeSnake string `json:"mime_type,omitempty"` } var aux BlobAlias @@ -1429,7 +1443,13 @@ func (b *Blob) UnmarshalJSON(data []byte) error { } b.DisplayName = aux.DisplayName + if b.DisplayName == "" { + b.DisplayName = aux.DisplayNameSnake + } b.MIMEType = aux.MIMEType + if b.MIMEType == "" { + b.MIMEType = aux.MIMETypeSnake + } if aux.Data != "" { // Convert URL-safe base64 to standard base64 @@ -1507,6 +1527,40 @@ type FileData struct { MIMEType string `json:"mimeType,omitempty"` } +// UnmarshalJSON custom unmarshaler for FileData. Also accepts the snake_case keys +// (mime_type, file_uri, display_name) the google-genai SDK emits inside +// functionResponse.parts (FunctionResponseFileData), preferring camelCase. +func (f *FileData) UnmarshalJSON(data []byte) error { + type FileDataAlias struct { + DisplayName string `json:"displayName,omitempty"` + DisplayNameSnake string `json:"display_name,omitempty"` + FileURI string `json:"fileUri,omitempty"` + FileURISnake string `json:"file_uri,omitempty"` + MIMEType string `json:"mimeType,omitempty"` + MIMETypeSnake string `json:"mime_type,omitempty"` + } + + var aux FileDataAlias + if err := sonic.Unmarshal(data, &aux); err != nil { + return err + } + + f.DisplayName = aux.DisplayName + if f.DisplayName == "" { + f.DisplayName = aux.DisplayNameSnake + } + f.FileURI = aux.FileURI + if f.FileURI == "" { + f.FileURI = aux.FileURISnake + } + f.MIMEType = aux.MIMEType + if f.MIMEType == "" { + f.MIMEType = aux.MIMETypeSnake + } + + return nil +} + // FunctionCall represents a function call. type FunctionCall struct { // Optional. The unique ID of the function call. If populated, the client to execute @@ -1543,6 +1597,10 @@ type FunctionResponse struct { // function output and "error" key to specify error details (if any). If "output" and // "error" keys are not specified, then whole "response" is treated as function output. Response json.RawMessage `json:"response,omitempty"` + // Optional. Multimodal content (images, files) returned by the function. Each part must + // contain inlineData or fileData with a displayName, referenced from `response` via + // {"$ref": ""}. Supported on Gemini 3 series models. + Parts []*Part `json:"parts,omitempty"` } // ==================== RESPONSE TYPES ==================== diff --git a/core/providers/gemini/videos.go b/core/providers/gemini/videos.go index 3f3c60802e..31b571b4d0 100644 --- a/core/providers/gemini/videos.go +++ b/core/providers/gemini/videos.go @@ -395,7 +395,7 @@ func (request *GeminiVideoGenerationRequest) ToBifrostVideoGenerationRequest(ctx // Use the first instance for the main input instance := request.Instances[0] - provider, model := schemas.ParseModelString(request.Model, providerUtils.CheckAndSetDefaultProvider(ctx, schemas.Gemini)) + provider, model := schemas.ParseModelString(request.Model, "") bifrostReq := &schemas.BifrostVideoGenerationRequest{ Provider: provider, diff --git a/core/providers/huggingface/models.go b/core/providers/huggingface/models.go index de615ccec2..3d04ce4936 100644 --- a/core/providers/huggingface/models.go +++ b/core/providers/huggingface/models.go @@ -14,7 +14,7 @@ const ( maxModelFetchLimit = 1000 ) -func (response *HuggingFaceListModelsResponse) ToBifrostListModelsResponse(providerKey schemas.ModelProvider, inferenceProvider inferenceProvider, allowedModels schemas.WhiteList, blacklistedModels schemas.BlackList, aliases map[string]string, unfiltered bool) *schemas.BifrostListModelsResponse { +func (response *HuggingFaceListModelsResponse) ToBifrostListModelsResponse(providerKey schemas.ModelProvider, inferenceProvider inferenceProvider, allowedModels schemas.WhiteList, blacklistedModels schemas.BlackList, aliases schemas.KeyAliases, unfiltered bool) *schemas.BifrostListModelsResponse { if response == nil { return nil } diff --git a/core/providers/mistral/models.go b/core/providers/mistral/models.go index 8d5fd7f3d6..7db5154aa6 100644 --- a/core/providers/mistral/models.go +++ b/core/providers/mistral/models.go @@ -7,7 +7,7 @@ import ( "github.com/maximhq/bifrost/core/schemas" ) -func (response *MistralListModelsResponse) ToBifrostListModelsResponse(allowedModels schemas.WhiteList, blacklistedModels schemas.BlackList, aliases map[string]string, unfiltered bool) *schemas.BifrostListModelsResponse { +func (response *MistralListModelsResponse) ToBifrostListModelsResponse(allowedModels schemas.WhiteList, blacklistedModels schemas.BlackList, aliases schemas.KeyAliases, unfiltered bool) *schemas.BifrostListModelsResponse { if response == nil { return nil } diff --git a/core/providers/openai/chat.go b/core/providers/openai/chat.go index af520e7d9c..6534c7d125 100644 --- a/core/providers/openai/chat.go +++ b/core/providers/openai/chat.go @@ -9,7 +9,7 @@ import ( // ToBifrostChatRequest converts an OpenAI chat request to Bifrost format func (req *OpenAIChatRequest) ToBifrostChatRequest(ctx *schemas.BifrostContext) *schemas.BifrostChatRequest { - provider, model := schemas.ParseModelString(req.Model, utils.CheckAndSetDefaultProvider(ctx, schemas.OpenAI)) + provider, model := schemas.ParseModelString(req.Model, "") return &schemas.BifrostChatRequest{ Provider: provider, @@ -29,6 +29,7 @@ func ToOpenAIChatRequest(ctx *schemas.BifrostContext, bifrostReq *schemas.Bifros openaiReq := &OpenAIChatRequest{ Model: bifrostReq.Model, Messages: ConvertBifrostMessagesToOpenAIMessages(bifrostReq.Input), + Provider: bifrostReq.Provider, } if bifrostReq.Params != nil { @@ -77,6 +78,14 @@ func ToOpenAIChatRequest(ctx *schemas.BifrostContext, bifrostReq *schemas.Bifros // Apply Mistral-specific transformations for Vertex Mistral models if schemas.IsMistralModel(bifrostReq.Model) { openaiReq.applyMistralCompatibility() + } else if openaiReq.Reasoning != nil && openaiReq.Reasoning.Effort != nil && + *openaiReq.Reasoning.Effort == "none" { + // Vertex Model Garden MaaS models (gpt-oss, Qwen3, kimi-k2-thinking, + // minimax-m2, ...) reject reasoning_effort "none" — only + // minimal/low/medium/high are accepted. Drop it so the model uses its + // default. (Mistral on Vertex does accept "none" and is handled above; + // // proprietary OpenAI/Azure GPT-5.1+ keep "none" via their own cases.) + openaiReq.Reasoning.Effort = nil } return openaiReq case schemas.Fireworks: diff --git a/core/providers/openai/chat_test.go b/core/providers/openai/chat_test.go index 027348002b..7e726c8e56 100644 --- a/core/providers/openai/chat_test.go +++ b/core/providers/openai/chat_test.go @@ -205,6 +205,80 @@ func TestToOpenAIChatRequest_NormalizesReasoningEffort(t *testing.T) { } } +// Vertex Model Garden MaaS models (gpt-oss, Qwen3, kimi-k2-thinking, minimax-m2) +// reject reasoning_effort "none"; only minimal/low/medium/high are accepted. The +// Vertex case should drop a "none" effort for these models while preserving it for +// Mistral on Vertex (which does accept "none"). +func TestToOpenAIChatRequest_VertexDropsNoneReasoningEffort(t *testing.T) { + tests := []struct { + name string + model string + keepsEffort bool + }{ + { + name: "MaaS model drops none effort", + model: "moonshotai/kimi-k2-thinking-maas", + keepsEffort: false, + }, + { + name: "minimax MaaS model drops none effort", + model: "minimaxai/minimax-m2-maas", + keepsEffort: false, + }, + { + name: "Mistral on Vertex keeps none effort", + model: "mistral-large", + keepsEffort: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := &schemas.BifrostChatRequest{ + Provider: schemas.Vertex, + Model: tt.model, + Input: []schemas.ChatMessage{{ + Role: schemas.ChatMessageRoleUser, + Content: &schemas.ChatMessageContent{ + ContentStr: schemas.Ptr("hello"), + }, + }}, + Params: &schemas.ChatParameters{ + Reasoning: &schemas.ChatReasoning{ + Effort: schemas.Ptr("none"), + }, + }, + } + + out := ToOpenAIChatRequest(schemas.NewBifrostContext(nil, schemas.NoDeadline), req) + if out == nil { + t.Fatal("expected OpenAI chat request") + } + + if tt.keepsEffort { + if out.Reasoning == nil || out.Reasoning.Effort == nil || *out.Reasoning.Effort != "none" { + t.Fatalf("expected reasoning effort to be preserved as \"none\", got %+v", out.Reasoning) + } + return + } + + // Effort must be dropped so reasoning_effort is omitted from the payload. + if out.Reasoning != nil && out.Reasoning.Effort != nil { + t.Fatalf("expected reasoning effort to be dropped, got %q", *out.Reasoning.Effort) + } + + // Verify the marshalled body does not contain reasoning_effort. + body, err := json.Marshal(out) + if err != nil { + t.Fatalf("failed to marshal request: %v", err) + } + if strings.Contains(string(body), "reasoning_effort") { + t.Fatalf("expected marshalled body to omit reasoning_effort, got %s", string(body)) + } + }) + } +} + func TestOpenAIChatRequest_FilterOpenAISpecificParameters_NormalizesReasoningEffort(t *testing.T) { tests := []struct { name string @@ -894,3 +968,85 @@ func TestApplyXAICompatibility(t *testing.T) { }) } } + +// TestToOpenAIChatRequest_CacheControl_OpenRouterOnly verifies that +// Anthropic-style cache_control breakpoints on message content blocks and on +// tools survive marshalling only when the originating provider is OpenRouter +// (which forwards them to the underlying Claude/Gemini model). For OpenAI and +// other OpenAI-format providers, cache_control is still stripped. +func TestToOpenAIChatRequest_CacheControl_OpenRouterOnly(t *testing.T) { + makeReq := func(provider schemas.ModelProvider) *schemas.BifrostChatRequest { + return &schemas.BifrostChatRequest{ + Provider: provider, + Model: "anthropic/claude-opus-4", + Input: []schemas.ChatMessage{ + { + Role: schemas.ChatMessageRoleSystem, + Content: &schemas.ChatMessageContent{ + ContentBlocks: []schemas.ChatContentBlock{ + { + Type: schemas.ChatContentBlockTypeText, + Text: schemas.Ptr("long cacheable system prompt"), + CacheControl: &schemas.CacheControl{Type: schemas.CacheControlTypeEphemeral}, + }, + }, + }, + }, + {Role: schemas.ChatMessageRoleUser, Content: &schemas.ChatMessageContent{ContentStr: schemas.Ptr("hello")}}, + }, + Params: &schemas.ChatParameters{ + Tools: []schemas.ChatTool{ + { + Type: schemas.ChatToolTypeFunction, + Function: &schemas.ChatToolFunction{ + Name: "lookup", + Description: schemas.Ptr("lookup something"), + Parameters: &schemas.ToolFunctionParameters{ + Type: "object", + Properties: schemas.NewOrderedMapFromPairs( + schemas.KV("q", map[string]interface{}{"type": "string"}), + ), + }, + }, + CacheControl: &schemas.CacheControl{Type: schemas.CacheControlTypeEphemeral}, + }, + }, + }, + } + } + + tests := []struct { + name string + provider schemas.ModelProvider + wantKept bool + }{ + {name: "openrouter preserves cache_control", provider: schemas.OpenRouter, wantKept: true}, + {name: "openai strips cache_control", provider: schemas.OpenAI, wantKept: false}, + {name: "gemini strips cache_control", provider: schemas.Gemini, wantKept: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := schemas.NewBifrostContextWithCancel(nil) + defer cancel() + + result := ToOpenAIChatRequest(ctx, makeReq(tt.provider)) + require.NotNil(t, result) + + wireBody, err := json.Marshal(result) + require.NoError(t, err) + s := string(wireBody) + + if tt.wantKept { + require.Contains(t, s, "cache_control", "cache_control must be preserved for OpenRouter: %s", s) + // Both the content-block breakpoint and the tool breakpoint must survive. + require.Equal(t, 2, strings.Count(s, "cache_control"), "expected cache_control on both content block and tool: %s", s) + } else { + require.NotContains(t, s, "cache_control", "cache_control must be stripped for %s: %s", tt.provider, s) + } + + // The tool identity must always survive regardless of stripping. + require.Contains(t, s, "lookup") + }) + } +} diff --git a/core/providers/openai/embedding.go b/core/providers/openai/embedding.go index fa243ac5b8..586d5f15eb 100644 --- a/core/providers/openai/embedding.go +++ b/core/providers/openai/embedding.go @@ -1,13 +1,12 @@ package openai import ( - "github.com/maximhq/bifrost/core/providers/utils" "github.com/maximhq/bifrost/core/schemas" ) // ToBifrostEmbeddingRequest converts an OpenAI embedding request to Bifrost format func (request *OpenAIEmbeddingRequest) ToBifrostEmbeddingRequest(ctx *schemas.BifrostContext) *schemas.BifrostEmbeddingRequest { - provider, model := schemas.ParseModelString(request.Model, utils.CheckAndSetDefaultProvider(ctx, schemas.OpenAI)) + provider, model := schemas.ParseModelString(request.Model, "") return &schemas.BifrostEmbeddingRequest{ Provider: provider, diff --git a/core/providers/openai/images.go b/core/providers/openai/images.go index 688df16831..799747fc80 100644 --- a/core/providers/openai/images.go +++ b/core/providers/openai/images.go @@ -56,7 +56,7 @@ func (request *OpenAIImageGenerationRequest) ToBifrostImageGenerationRequest(ctx return nil } - provider, model := schemas.ParseModelString(request.Model, providerUtils.CheckAndSetDefaultProvider(ctx, schemas.OpenAI)) + provider, model := schemas.ParseModelString(request.Model, "") return &schemas.BifrostImageGenerationRequest{ Provider: provider, @@ -74,7 +74,7 @@ func (request *OpenAIImageEditRequest) ToBifrostImageEditRequest(ctx *schemas.Bi return nil } - provider, model := schemas.ParseModelString(request.Model, providerUtils.CheckAndSetDefaultProvider(ctx, schemas.OpenAI)) + provider, model := schemas.ParseModelString(request.Model, "") return &schemas.BifrostImageEditRequest{ Provider: provider, @@ -90,7 +90,7 @@ func (request *OpenAIImageVariationRequest) ToBifrostImageVariationRequest(ctx * return nil } - provider, model := schemas.ParseModelString(request.Model, providerUtils.CheckAndSetDefaultProvider(ctx, schemas.OpenAI)) + provider, model := schemas.ParseModelString(request.Model, "") return &schemas.BifrostImageVariationRequest{ Provider: provider, diff --git a/core/providers/openai/models.go b/core/providers/openai/models.go index a76d350d28..f0fbea6057 100644 --- a/core/providers/openai/models.go +++ b/core/providers/openai/models.go @@ -1,6 +1,7 @@ package openai import ( + "encoding/json" "strings" providerUtils "github.com/maximhq/bifrost/core/providers/utils" @@ -8,7 +9,7 @@ import ( ) // ToBifrostListModelsResponse converts an OpenAI list models response to a Bifrost list models response -func (response *OpenAIListModelsResponse) ToBifrostListModelsResponse(providerKey schemas.ModelProvider, allowedModels schemas.WhiteList, blacklistedModels schemas.BlackList, aliases map[string]string, unfiltered bool) *schemas.BifrostListModelsResponse { +func (response *OpenAIListModelsResponse) ToBifrostListModelsResponse(providerKey schemas.ModelProvider, allowedModels schemas.WhiteList, blacklistedModels schemas.BlackList, aliases schemas.KeyAliases, unfiltered bool) *schemas.BifrostListModelsResponse { if response == nil { return nil } @@ -32,15 +33,18 @@ func (response *OpenAIListModelsResponse) ToBifrostListModelsResponse(providerKe included := make(map[string]bool) for _, model := range response.Data { - for _, result := range pipeline.FilterModel(model.ID) { - entry := schemas.Model{ - ID: string(providerKey) + "/" + result.ResolvedID, - Created: model.Created, - OwnedBy: schemas.Ptr(model.OwnedBy), - ContextLength: model.ContextWindow, - } + rawID := model.ID + if parsedProvider, parsedModel := schemas.ParseListModelString(rawID, ""); parsedProvider != "" && strings.EqualFold(string(parsedProvider), string(providerKey)) { + rawID = parsedModel + } + + for _, result := range pipeline.FilterModel(rawID) { + entry := model + entry.ID = string(providerKey) + "/" + result.ResolvedID if result.AliasValue != "" { entry.Alias = schemas.Ptr(result.AliasValue) + } else { + entry.Alias = nil } bifrostResponse.Data = append(bifrostResponse.Data, entry) included[strings.ToLower(result.ResolvedID)] = true @@ -59,22 +63,24 @@ func ToOpenAIListModelsResponse(response *schemas.BifrostListModelsResponse) *Op return nil } openaiResponse := &OpenAIListModelsResponse{ - Data: make([]OpenAIModel, 0, len(response.Data)), + Object: "list", + Data: make([]schemas.Model, 0, len(response.Data)), } for _, model := range response.Data { - openaiModel := OpenAIModel{ - ID: model.ID, - Object: "model", - } - if model.Created != nil { - openaiModel.Created = model.Created - } - if model.OwnedBy != nil { - openaiModel.OwnedBy = *model.OwnedBy + if len(model.RawModelJSON) == 0 { + model.RawModelJSON = json.RawMessage(`{"object":"model"}`) + } else { + payload := map[string]json.RawMessage{} + if err := json.Unmarshal(model.RawModelJSON, &payload); err == nil { + if _, ok := payload["object"]; !ok { + payload["object"] = json.RawMessage(`"model"`) + if raw, err := json.Marshal(payload); err == nil { + model.RawModelJSON = raw + } + } + } } - - openaiResponse.Data = append(openaiResponse.Data, openaiModel) - + openaiResponse.Data = append(openaiResponse.Data, model) } return openaiResponse } diff --git a/core/providers/openai/models_list_test.go b/core/providers/openai/models_list_test.go new file mode 100644 index 0000000000..7dc396ac7f --- /dev/null +++ b/core/providers/openai/models_list_test.go @@ -0,0 +1,147 @@ +package openai + +import ( + "encoding/json" + "testing" + + "github.com/maximhq/bifrost/core/schemas" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestToBifrostListModelsResponse_PreservesRichMetadata(t *testing.T) { + t.Parallel() + + raw := `{ + "object": "list", + "data": [{ + "id": "gpt-5.5", + "object": "model", + "created": 1754587413, + "owned_by": "openai", + "name": "GPT 5.5", + "description": "Rich metadata model", + "canonical_slug": "openai/gpt-5.5", + "context_length": 1050000, + "architecture": { + "modality": "text+image->text", + "input_modalities": ["text", "image"], + "output_modalities": ["text"] + }, + "default_parameters": { + "temperature": 0.7, + "top_p": 0.95 + }, + "supported_parameters": ["tools", "response_format"], + "top_provider": { + "is_moderated": true, + "context_length": 1050000, + "max_completion_tokens": 64000 + }, + "pricing": { + "prompt": "0.000001", + "completion": "0.000004", + "input_cache_read": "0.0000001" + }, + "knowledge_cutoff": "2025-01", + "expiration_date": "2026-12-31", + "aliases": ["gpt-5.5-latest"] + }] + }` + + var upstream OpenAIListModelsResponse + require.NoError(t, json.Unmarshal([]byte(raw), &upstream)) + + resp := upstream.ToBifrostListModelsResponse(schemas.OpenAI, nil, nil, nil, true) + require.Len(t, resp.Data, 1) + require.Equal(t, "openai/gpt-5.5", resp.Data[0].ID) + + payload, err := json.Marshal(resp) + require.NoError(t, err) + + var decoded map[string]any + require.NoError(t, json.Unmarshal(payload, &decoded)) + + data := decoded["data"].([]any) + model := data[0].(map[string]any) + assert.Equal(t, "openai/gpt-5.5", model["id"]) + assert.Equal(t, "model", model["object"]) + assert.Equal(t, "openai", model["owned_by"]) + assert.Equal(t, "GPT 5.5", model["name"]) + assert.Equal(t, "Rich metadata model", model["description"]) + assert.Equal(t, "openai/gpt-5.5", model["canonical_slug"]) + assert.Equal(t, "2025-01", model["knowledge_cutoff"]) + assert.Equal(t, "2026-12-31", model["expiration_date"]) + assert.Equal(t, float64(1050000), model["context_length"]) + + architecture := model["architecture"].(map[string]any) + assert.Equal(t, "text+image->text", architecture["modality"]) + assert.Equal(t, []any{"text", "image"}, architecture["input_modalities"]) + + pricing := model["pricing"].(map[string]any) + assert.Equal(t, "0.000001", pricing["prompt"]) + assert.Equal(t, "0.000004", pricing["completion"]) + assert.Equal(t, "0.0000001", pricing["input_cache_read"]) + + supportedParameters := model["supported_parameters"].([]any) + assert.Equal(t, []any{"tools", "response_format"}, supportedParameters) +} + +func TestToBifrostListModelsResponse_MinimalModelStillWorks(t *testing.T) { + t.Parallel() + + raw := `{"object":"list","data":[{"id":"gpt-4o-mini","object":"model","created":123,"owned_by":"openai"}]}` + var upstream OpenAIListModelsResponse + require.NoError(t, json.Unmarshal([]byte(raw), &upstream)) + + resp := upstream.ToBifrostListModelsResponse(schemas.OpenAI, nil, nil, nil, true) + require.Len(t, resp.Data, 1) + + payload, err := json.Marshal(resp) + require.NoError(t, err) + + var decoded map[string]any + require.NoError(t, json.Unmarshal(payload, &decoded)) + model := decoded["data"].([]any)[0].(map[string]any) + assert.Equal(t, "openai/gpt-4o-mini", model["id"]) + assert.Equal(t, "model", model["object"]) + assert.Equal(t, float64(123), model["created"]) + assert.Equal(t, "openai", model["owned_by"]) +} + +func TestToBifrostListModelsResponse_StripsCaseInsensitiveProviderPrefix(t *testing.T) { + t.Parallel() + + raw := `{"object":"list","data":[{"id":"OpenAI/gpt-4o","object":"model","owned_by":"openai"}]}` + var upstream OpenAIListModelsResponse + require.NoError(t, json.Unmarshal([]byte(raw), &upstream)) + + resp := upstream.ToBifrostListModelsResponse(schemas.OpenAI, nil, nil, nil, true) + require.Len(t, resp.Data, 1) + assert.Equal(t, "openai/gpt-4o", resp.Data[0].ID) +} + +func TestToOpenAIListModelsResponse_DefaultsItemObjectToModel(t *testing.T) { + t.Parallel() + + response := &schemas.BifrostListModelsResponse{ + Data: []schemas.Model{{ + ID: "openai/gpt-4o", + OwnedBy: schemas.Ptr("openai"), + }}, + } + + openaiResponse := ToOpenAIListModelsResponse(response) + require.NotNil(t, openaiResponse) + require.Len(t, openaiResponse.Data, 1) + assert.Equal(t, "list", openaiResponse.Object) + assert.Equal(t, "openai/gpt-4o", openaiResponse.Data[0].ID) + + payload, err := json.Marshal(openaiResponse) + require.NoError(t, err) + + var decoded map[string]any + require.NoError(t, json.Unmarshal(payload, &decoded)) + model := decoded["data"].([]any)[0].(map[string]any) + assert.Equal(t, "model", model["object"]) +} diff --git a/core/providers/openai/responses.go b/core/providers/openai/responses.go index a6216e0482..feaf4d1107 100644 --- a/core/providers/openai/responses.go +++ b/core/providers/openai/responses.go @@ -13,16 +13,7 @@ func (resp *OpenAIResponsesRequest) ToBifrostResponsesRequest(ctx *schemas.Bifro return nil } - defaultProvider := schemas.OpenAI - - // for requests coming from azure sdk without provider prefix, we need to set the default provider to azure - if ctx != nil { - if isAzureUser, ok := ctx.Value(schemas.BifrostContextKeyIsAzureUserAgent).(bool); ok && isAzureUser { - defaultProvider = schemas.Azure - } - } - - provider, model := schemas.ParseModelString(resp.Model, utils.CheckAndSetDefaultProvider(ctx, defaultProvider)) + provider, model := schemas.ParseModelString(resp.Model, "") input := resp.Input.OpenAIResponsesRequestInputArray if len(input) == 0 { @@ -463,16 +454,8 @@ func (r *OpenAICompactionRequest) ToBifrostCompactionRequest(ctx *schemas.Bifros if r == nil { return nil } - defaultProvider := schemas.OpenAI - - // for requests coming from azure sdk without provider prefix, we need to set the default provider to azure - if ctx != nil { - if isAzureUser, ok := ctx.Value(schemas.BifrostContextKeyIsAzureUserAgent).(bool); ok && isAzureUser { - defaultProvider = schemas.Azure - } - } - provider, model := schemas.ParseModelString(r.Model, utils.CheckAndSetDefaultProvider(ctx, defaultProvider)) + provider, model := schemas.ParseModelString(r.Model, "") input := r.Input.OpenAIResponsesRequestInputArray if len(input) == 0 && r.Input.OpenAIResponsesRequestInputStr != nil { input = []schemas.ResponsesMessage{ diff --git a/core/providers/openai/speech.go b/core/providers/openai/speech.go index 09c638fc5e..0a092e3e7d 100644 --- a/core/providers/openai/speech.go +++ b/core/providers/openai/speech.go @@ -1,13 +1,12 @@ package openai import ( - "github.com/maximhq/bifrost/core/providers/utils" "github.com/maximhq/bifrost/core/schemas" ) // ToBifrostSpeechRequest converts an OpenAI speech request to Bifrost format func (request *OpenAISpeechRequest) ToBifrostSpeechRequest(ctx *schemas.BifrostContext) *schemas.BifrostSpeechRequest { - provider, model := schemas.ParseModelString(request.Model, utils.CheckAndSetDefaultProvider(ctx, schemas.OpenAI)) + provider, model := schemas.ParseModelString(request.Model, "") return &schemas.BifrostSpeechRequest{ Provider: provider, diff --git a/core/providers/openai/text.go b/core/providers/openai/text.go index 07354a0263..e171050088 100644 --- a/core/providers/openai/text.go +++ b/core/providers/openai/text.go @@ -3,7 +3,6 @@ package openai import ( "maps" - "github.com/maximhq/bifrost/core/providers/utils" "github.com/maximhq/bifrost/core/schemas" ) @@ -63,7 +62,7 @@ func (req *OpenAITextCompletionRequest) ToBifrostTextCompletionRequest(ctx *sche return nil } - provider, model := schemas.ParseModelString(req.Model, utils.CheckAndSetDefaultProvider(ctx, schemas.OpenAI)) + provider, model := schemas.ParseModelString(req.Model, "") return &schemas.BifrostTextCompletionRequest{ Provider: provider, diff --git a/core/providers/openai/transcription.go b/core/providers/openai/transcription.go index 1bf419759a..cbfb130714 100644 --- a/core/providers/openai/transcription.go +++ b/core/providers/openai/transcription.go @@ -10,7 +10,7 @@ import ( // ToBifrostTranscriptionRequest converts an OpenAI transcription request to Bifrost format func (request *OpenAITranscriptionRequest) ToBifrostTranscriptionRequest(ctx *schemas.BifrostContext) *schemas.BifrostTranscriptionRequest { - provider, model := schemas.ParseModelString(request.Model, utils.CheckAndSetDefaultProvider(ctx, schemas.OpenAI)) + provider, model := schemas.ParseModelString(request.Model, "") return &schemas.BifrostTranscriptionRequest{ Provider: provider, diff --git a/core/providers/openai/types.go b/core/providers/openai/types.go index e0e4061944..566cb71c6f 100644 --- a/core/providers/openai/types.go +++ b/core/providers/openai/types.go @@ -88,6 +88,10 @@ type OpenAIChatRequest struct { // This Field is populated only for such providers and is NOT to be used externally. MaxTokens *int `json:"max_tokens,omitempty"` + // Provider is the originating provider, used for provider-specific marshalling + // (e.g. preserving cache_control for OpenRouter). Not serialized to wire. + Provider schemas.ModelProvider `json:"-"` + // Bifrost specific field (only parsed when converting from Provider -> Bifrost request) Fallbacks []string `json:"fallbacks,omitempty"` ExtraParams map[string]interface{} `json:"-"` // Optional: Extra parameters @@ -134,10 +138,17 @@ func (req *OpenAIChatRequest) MarshalJSON() ([]byte, error) { } type Alias OpenAIChatRequest + // OpenRouter forwards Anthropic-style cache_control breakpoints to the + // underlying model, so we must preserve cache_control (on content blocks + // and tools) when targeting OpenRouter. Everything else (citations, + // file types, Anthropic server tools, Anthropic-only tool flags) is still + // stripped — OpenRouter is otherwise an OpenAI-format endpoint. + keepCacheControl := req.Provider == schemas.OpenRouter + // First pass: check if we need to modify any messages needsCopy := false for _, msg := range req.Messages { - if hasFieldsToStripInChatMessage(msg) { + if hasFieldsToStripInChatMessage(msg, keepCacheControl) { needsCopy = true break } @@ -148,7 +159,7 @@ func (req *OpenAIChatRequest) MarshalJSON() ([]byte, error) { if needsCopy { processedMessages = make([]OpenAIMessage, len(req.Messages)) for i, msg := range req.Messages { - if !hasFieldsToStripInChatMessage(msg) { + if !hasFieldsToStripInChatMessage(msg, keepCacheControl) { // No modification needed, use original processedMessages[i] = msg continue @@ -162,10 +173,13 @@ func (req *OpenAIChatRequest) MarshalJSON() ([]byte, error) { contentCopy := *msg.Content contentCopy.ContentBlocks = make([]schemas.ChatContentBlock, len(msg.Content.ContentBlocks)) for j, block := range msg.Content.ContentBlocks { - needsBlockCopy := block.CacheControl != nil || block.Citations != nil || (block.File != nil && block.File.FileType != nil) + stripBlockCacheControl := block.CacheControl != nil && !keepCacheControl + needsBlockCopy := stripBlockCacheControl || block.Citations != nil || (block.File != nil && (block.File.FileType != nil || block.File.FileURL != nil)) if needsBlockCopy { blockCopy := block - blockCopy.CacheControl = nil + if stripBlockCacheControl { + blockCopy.CacheControl = nil + } blockCopy.Citations = nil // Strip FileType and FileURL from file block if blockCopy.File != nil && (blockCopy.File.FileType != nil || blockCopy.File.FileURL != nil) { @@ -197,7 +211,8 @@ func (req *OpenAIChatRequest) MarshalJSON() ([]byte, error) { if len(req.Tools) > 0 { needsToolChange := false for _, tool := range req.Tools { - if tool.CacheControl != nil || isAnthropicServerToolShape(tool) || hasAnthropicOnlyToolFlags(tool) { + stripToolCacheControl := tool.CacheControl != nil && !keepCacheControl + if stripToolCacheControl || isAnthropicServerToolShape(tool) || hasAnthropicOnlyToolFlags(tool) { needsToolChange = true break } @@ -211,12 +226,15 @@ func (req *OpenAIChatRequest) MarshalJSON() ([]byte, error) { if isAnthropicServerToolShape(tool) { continue } - if tool.CacheControl == nil && !hasAnthropicOnlyToolFlags(tool) { + stripToolCacheControl := tool.CacheControl != nil && !keepCacheControl + if !stripToolCacheControl && !hasAnthropicOnlyToolFlags(tool) { processedTools = append(processedTools, tool) continue } toolCopy := tool - toolCopy.CacheControl = nil + if stripToolCacheControl { + toolCopy.CacheControl = nil + } toolCopy.DeferLoading = nil toolCopy.AllowedCallers = nil toolCopy.InputExamples = nil @@ -575,10 +593,10 @@ func isAnthropicOnlyResponsesToolType(t schemas.ResponsesTool) bool { t.Type == schemas.ResponsesToolTypeMemory } -func hasFieldsToStripInChatMessage(msg OpenAIMessage) bool { +func hasFieldsToStripInChatMessage(msg OpenAIMessage, keepCacheControl bool) bool { if msg.Content != nil && msg.Content.ContentBlocks != nil { for _, block := range msg.Content.ContentBlocks { - if block.CacheControl != nil { + if block.CacheControl != nil && !keepCacheControl { return true } if block.Citations != nil { @@ -878,22 +896,10 @@ func (r *OpenAITranscriptionRequest) IsStreamingRequested() bool { return r.Stream != nil && *r.Stream } -// OpenAIModel represents an OpenAI model -type OpenAIModel struct { - ID string `json:"id"` - Object string `json:"object"` - OwnedBy string `json:"owned_by"` - Created *int64 `json:"created,omitempty"` - - // GROQ specific fields - Active *bool `json:"active,omitempty"` - ContextWindow *int `json:"context_window,omitempty"` -} - // OpenAIListModelsResponse represents an OpenAI list models response type OpenAIListModelsResponse struct { - Object string `json:"object"` - Data []OpenAIModel `json:"data"` + Object string `json:"object"` + Data []schemas.Model `json:"data"` } // OpenAIImageGenerationRequest is the struct for Image Generation requests by OpenAI. diff --git a/core/providers/openai/videos.go b/core/providers/openai/videos.go index 512306b7c7..1b794b8ad3 100644 --- a/core/providers/openai/videos.go +++ b/core/providers/openai/videos.go @@ -6,7 +6,6 @@ import ( "mime/multipart" "net/http" - "github.com/maximhq/bifrost/core/providers/utils" providerUtils "github.com/maximhq/bifrost/core/providers/utils" "github.com/maximhq/bifrost/core/schemas" ) @@ -101,16 +100,7 @@ func (req *OpenAIVideoGenerationRequest) ToBifrostVideoGenerationRequest(ctx *sc return nil } - defaultProvider := schemas.OpenAI - - // for requests coming from azure sdk without provider prefix, we need to set the default provider to azure - if ctx != nil { - if isAzureUser, ok := ctx.Value(schemas.BifrostContextKeyIsAzureUserAgent).(bool); ok && isAzureUser { - defaultProvider = schemas.Azure - } - } - - provider, model := schemas.ParseModelString(req.Model, utils.CheckAndSetDefaultProvider(ctx, defaultProvider)) + provider, model := schemas.ParseModelString(req.Model, "") input := &schemas.VideoGenerationInput{ Prompt: req.Prompt, @@ -132,30 +122,30 @@ func (req *OpenAIVideoGenerationRequest) ToBifrostVideoGenerationRequest(ctx *sc func parseVideoGenerationFormDataBodyFromRequest(writer *multipart.Writer, openaiReq *OpenAIVideoGenerationRequest, providerName schemas.ModelProvider) *schemas.BifrostError { // Add prompt field (required) if openaiReq.Prompt == "" { - return providerUtils.NewBifrostOperationError("prompt is required", nil) + return providerUtils.NewBifrostOperationError("prompt is required", nil) } if err := writer.WriteField("prompt", openaiReq.Prompt); err != nil { - return providerUtils.NewBifrostOperationError("failed to write prompt field", err) + return providerUtils.NewBifrostOperationError("failed to write prompt field", err) } // Add optional model field if openaiReq.Model != "" { if err := writer.WriteField("model", openaiReq.Model); err != nil { - return providerUtils.NewBifrostOperationError("failed to write model field", err) + return providerUtils.NewBifrostOperationError("failed to write model field", err) } } // Add optional seconds field if openaiReq.Seconds != nil { if err := writer.WriteField("seconds", *openaiReq.Seconds); err != nil { - return providerUtils.NewBifrostOperationError("failed to write seconds field", err) + return providerUtils.NewBifrostOperationError("failed to write seconds field", err) } } // Add optional size field if openaiReq.Size != "" { if err := writer.WriteField("size", openaiReq.Size); err != nil { - return providerUtils.NewBifrostOperationError("failed to write size field", err) + return providerUtils.NewBifrostOperationError("failed to write size field", err) } } @@ -196,16 +186,16 @@ func parseVideoGenerationFormDataBodyFromRequest(writer *multipart.Writer, opena "Content-Type": {mimeType}, }) if err != nil { - return providerUtils.NewBifrostOperationError("failed to create form part for input_reference", err) + return providerUtils.NewBifrostOperationError("failed to create form part for input_reference", err) } if _, err := part.Write(openaiReq.InputReference); err != nil { - return providerUtils.NewBifrostOperationError("failed to write input_reference file data", err) + return providerUtils.NewBifrostOperationError("failed to write input_reference file data", err) } } // Close the multipart writer if err := writer.Close(); err != nil { - return providerUtils.NewBifrostOperationError("failed to close multipart writer", err) + return providerUtils.NewBifrostOperationError("failed to close multipart writer", err) } return nil diff --git a/core/providers/openrouter/openrouter.go b/core/providers/openrouter/openrouter.go index eae6de2bf9..516f52477e 100644 --- a/core/providers/openrouter/openrouter.go +++ b/core/providers/openrouter/openrouter.go @@ -206,9 +206,11 @@ func (provider *OpenRouterProvider) listModelsByKey(ctx *schemas.BifrostContext, for _, m := range key.BlacklistedModels { normalizedBlacklist = append(normalizedBlacklist, stripPrefix(m)) } - normalizedAliases := make(map[string]string, len(key.Aliases)) + normalizedAliases := make(schemas.KeyAliases, len(key.Aliases)) for k, v := range key.Aliases { - normalizedAliases[stripPrefix(k)] = stripPrefix(v) + cfg := v + cfg.ModelID = stripPrefix(v.ModelID) + normalizedAliases[stripPrefix(k)] = cfg } pipeline := &providerUtils.ListModelsPipeline{ diff --git a/core/providers/replicate/models.go b/core/providers/replicate/models.go index 6c0c14dbf7..3d2c4b6081 100644 --- a/core/providers/replicate/models.go +++ b/core/providers/replicate/models.go @@ -14,7 +14,7 @@ func ToBifrostListModelsResponse( providerKey schemas.ModelProvider, allowedModels schemas.WhiteList, blacklistedModels schemas.BlackList, - aliases map[string]string, + aliases schemas.KeyAliases, unfiltered bool, ) *schemas.BifrostListModelsResponse { bifrostResponse := &schemas.BifrostListModelsResponse{ diff --git a/core/providers/replicate/replicate.go b/core/providers/replicate/replicate.go index aa6b75c9df..ee4292ae54 100644 --- a/core/providers/replicate/replicate.go +++ b/core/providers/replicate/replicate.go @@ -90,9 +90,20 @@ const ( pollingInterval = 2 * time.Second ) -// useDeploymentsEndpoint returns whether the key uses the deployments endpoint. -// Nil ReplicateKeyConfig is treated as false (default models/predictions behavior). -func useDeploymentsEndpoint(key schemas.Key) bool { +// useDeploymentsEndpoint returns whether the request should target the +// Replicate deployments endpoint vs the predictions endpoint. +// +// Priority: per-alias ReplicateAliasCfg.UseDeploymentsEndpoint (when set) > +// key-level ReplicateKeyConfig.UseDeploymentsEndpoint. The override lets one +// Replicate API token route some aliases through the deployments endpoint +// (e.g. production-pinned models) while others use the predictions endpoint +// (e.g. experimental versioned models). +// +// Nil ReplicateKeyConfig and missing alias both default to false (predictions). +func useDeploymentsEndpoint(ctx *schemas.BifrostContext, key schemas.Key) bool { + if ra := schemas.GetResolvedAlias(ctx); ra != nil && ra.Config != nil && ra.Config.ReplicateAliasCfg != nil && ra.Config.ReplicateAliasCfg.UseDeploymentsEndpoint != nil { + return *ra.Config.ReplicateAliasCfg.UseDeploymentsEndpoint + } return key.ReplicateKeyConfig != nil && key.ReplicateKeyConfig.UseDeploymentsEndpoint } @@ -284,7 +295,7 @@ func (provider *ReplicateProvider) listDeploymentsByKey(ctx *schemas.BifrostCont client := provider.client extraHeaders := provider.networkConfig.ExtraHeaders - if !useDeploymentsEndpoint(key) { + if !useDeploymentsEndpoint(ctx, key) { return ToBifrostListModelsResponse( &ReplicateDeploymentListResponse{}, providerName, @@ -439,7 +450,7 @@ func (provider *ReplicateProvider) TextCompletion(ctx *schemas.BifrostContext, k request.Model, provider.customProviderConfig, schemas.TextCompletionRequest, - useDeploymentsEndpoint(key), + useDeploymentsEndpoint(ctx, key), ) // create prediction @@ -531,7 +542,7 @@ func (provider *ReplicateProvider) TextCompletionStream(ctx *schemas.BifrostCont request.Model, provider.customProviderConfig, schemas.TextCompletionStreamRequest, - useDeploymentsEndpoint(key), + useDeploymentsEndpoint(ctx, key), ) startTime := time.Now() @@ -779,7 +790,7 @@ func (provider *ReplicateProvider) ChatCompletion(ctx *schemas.BifrostContext, k request.Model, provider.customProviderConfig, schemas.ChatCompletionRequest, - useDeploymentsEndpoint(key), + useDeploymentsEndpoint(ctx, key), ) // create prediction @@ -871,7 +882,7 @@ func (provider *ReplicateProvider) ChatCompletionStream(ctx *schemas.BifrostCont request.Model, provider.customProviderConfig, schemas.ChatCompletionStreamRequest, - useDeploymentsEndpoint(key), + useDeploymentsEndpoint(ctx, key), ) startTime := time.Now() @@ -1136,7 +1147,7 @@ func (provider *ReplicateProvider) Responses(ctx *schemas.BifrostContext, key sc request.Model, provider.customProviderConfig, schemas.ResponsesRequest, - useDeploymentsEndpoint(key), + useDeploymentsEndpoint(ctx, key), ) // create prediction @@ -1223,7 +1234,7 @@ func (provider *ReplicateProvider) ResponsesStream(ctx *schemas.BifrostContext, request.Model, provider.customProviderConfig, schemas.ResponsesStreamRequest, - useDeploymentsEndpoint(key), + useDeploymentsEndpoint(ctx, key), ) startTime := time.Now() @@ -1745,7 +1756,7 @@ func (provider *ReplicateProvider) ImageGeneration(ctx *schemas.BifrostContext, request.Model, provider.customProviderConfig, schemas.ImageGenerationRequest, - useDeploymentsEndpoint(key), + useDeploymentsEndpoint(ctx, key), ) // Create prediction with appropriate mode @@ -1839,7 +1850,7 @@ func (provider *ReplicateProvider) ImageGenerationStream(ctx *schemas.BifrostCon request.Model, provider.customProviderConfig, schemas.ImageGenerationStreamRequest, - useDeploymentsEndpoint(key), + useDeploymentsEndpoint(ctx, key), ) startTime := time.Now() // Create prediction @@ -2150,7 +2161,7 @@ func (provider *ReplicateProvider) ImageEdit(ctx *schemas.BifrostContext, key sc request.Model, provider.customProviderConfig, schemas.ImageEditRequest, - useDeploymentsEndpoint(key), + useDeploymentsEndpoint(ctx, key), ) // Create prediction with appropriate mode @@ -2244,7 +2255,7 @@ func (provider *ReplicateProvider) ImageEditStream(ctx *schemas.BifrostContext, request.Model, provider.customProviderConfig, schemas.ImageEditStreamRequest, - useDeploymentsEndpoint(key), + useDeploymentsEndpoint(ctx, key), ) startTime := time.Now() @@ -2538,7 +2549,7 @@ func (provider *ReplicateProvider) VideoGeneration(ctx *schemas.BifrostContext, request.Model, provider.customProviderConfig, schemas.VideoGenerationRequest, - useDeploymentsEndpoint(key), + useDeploymentsEndpoint(ctx, key), ) // Create prediction with appropriate mode diff --git a/core/providers/replicate/replicate_test.go b/core/providers/replicate/replicate_test.go index 855b9cb690..6d3f4bea55 100644 --- a/core/providers/replicate/replicate_test.go +++ b/core/providers/replicate/replicate_test.go @@ -1438,3 +1438,4 @@ func TestReplicateToBifrostResponsesResponse(t *testing.T) { }) } } + diff --git a/core/providers/replicate/use_deployments_endpoint_test.go b/core/providers/replicate/use_deployments_endpoint_test.go new file mode 100644 index 0000000000..f85b7ae4ff --- /dev/null +++ b/core/providers/replicate/use_deployments_endpoint_test.go @@ -0,0 +1,76 @@ +package replicate + +import ( + "context" + "testing" + + "github.com/maximhq/bifrost/core/schemas" +) + +// TestUseDeploymentsEndpoint_AliasOverride verifies the per-alias +// ReplicateAliasCfg.UseDeploymentsEndpoint override resolves correctly: +// alias value wins when set, else falls through to key-level config. +func TestUseDeploymentsEndpoint_AliasOverride(t *testing.T) { + keyDeployments := schemas.Key{ + ReplicateKeyConfig: &schemas.ReplicateKeyConfig{UseDeploymentsEndpoint: true}, + } + keyPredictions := schemas.Key{ + ReplicateKeyConfig: &schemas.ReplicateKeyConfig{UseDeploymentsEndpoint: false}, + } + + // No alias in ctx — falls back to key-level setting. + if got := useDeploymentsEndpoint(nil, keyDeployments); !got { + t.Errorf("nil ctx + key=deployments: want true, got false") + } + if got := useDeploymentsEndpoint(nil, keyPredictions); got { + t.Errorf("nil ctx + key=predictions: want false, got true") + } + if got := useDeploymentsEndpoint(nil, schemas.Key{}); got { + t.Errorf("nil ctx + nil ReplicateKeyConfig: want false, got true") + } + + // Alias override true wins over key=false. + ctxOverrideTrue := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + defer ctxOverrideTrue.Cancel() + trueVal := true + ctxOverrideTrue.SetValue(schemas.BifrostContextKeyResolvedAlias, &schemas.ResolvedAlias{ + Key: "prod-llm", + Config: &schemas.AliasConfig{ + ModelID: "owner/name:version", + ReplicateAliasCfg: &schemas.ReplicateAliasCfg{ + UseDeploymentsEndpoint: &trueVal, + }, + }, + }) + if got := useDeploymentsEndpoint(ctxOverrideTrue, keyPredictions); !got { + t.Errorf("alias=true should override key=false: got false") + } + + // Alias override false wins over key=true. + ctxOverrideFalse := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + defer ctxOverrideFalse.Cancel() + falseVal := false + ctxOverrideFalse.SetValue(schemas.BifrostContextKeyResolvedAlias, &schemas.ResolvedAlias{ + Key: "experimental-llm", + Config: &schemas.AliasConfig{ + ModelID: "owner/name:version", + ReplicateAliasCfg: &schemas.ReplicateAliasCfg{ + UseDeploymentsEndpoint: &falseVal, + }, + }, + }) + if got := useDeploymentsEndpoint(ctxOverrideFalse, keyDeployments); got { + t.Errorf("alias=false should override key=true: got true") + } + + // Alias present but ReplicateAliasCfg unset — falls through to key. + ctxNoCfg := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + defer ctxNoCfg.Cancel() + ctxNoCfg.SetValue(schemas.BifrostContextKeyResolvedAlias, &schemas.ResolvedAlias{ + Key: "x", + Config: &schemas.AliasConfig{ModelID: "x"}, + }) + if got := useDeploymentsEndpoint(ctxNoCfg, keyDeployments); !got { + t.Errorf("no alias cfg + key=deployments: want true, got false") + } +} diff --git a/core/providers/utils/models.go b/core/providers/utils/models.go index dbbbd8071a..22555f11ae 100644 --- a/core/providers/utils/models.go +++ b/core/providers/utils/models.go @@ -168,9 +168,9 @@ type FilterResult struct { type ListModelsPipeline struct { AllowedModels schemas.WhiteList BlacklistedModels schemas.BlackList - // Aliases maps user-facing alias keys to provider-specific model IDs. - // e.g. {"my-gpt4": "gpt-4-turbo-2024-04-09"} - Aliases map[string]string + // Aliases maps user-facing alias keys to their AliasConfig. The pipeline + // reads AliasConfig.ModelID for matching and Alias surfacing. + Aliases schemas.KeyAliases Unfiltered bool ProviderKey schemas.ModelProvider // MatchFns is the ordered list of equivalence functions used for every @@ -224,9 +224,9 @@ type aliasMatch struct { // → [{key:"gpt-3.5-turbo", value:""}] func (p *ListModelsPipeline) resolveModelID(modelID string) []aliasMatch { var candidates []aliasMatch - for aliasKey, providerID := range p.Aliases { - if matches(modelID, providerID, p.MatchFns) { - candidates = append(candidates, aliasMatch{key: aliasKey, value: providerID}) + for aliasKey, alias := range p.Aliases { + if matches(modelID, alias.ModelID, p.MatchFns) { + candidates = append(candidates, aliasMatch{key: aliasKey, value: alias.ModelID}) } } if len(candidates) == 0 { @@ -369,9 +369,9 @@ func (p *ListModelsPipeline) BackfillModels(included map[string]bool) []schemas. Name: schemas.Ptr(ToDisplayName(entry)), } // If this allowlist entry has an alias, surface the provider-specific ID. - for aliasKey, providerID := range p.Aliases { + for aliasKey, alias := range p.Aliases { if matches(entry, aliasKey, p.MatchFns) { - m.Alias = schemas.Ptr(providerID) + m.Alias = schemas.Ptr(alias.ModelID) break } } @@ -382,7 +382,7 @@ func (p *ListModelsPipeline) BackfillModels(included map[string]bool) []schemas. // Case B: wildcard allowlist — backfill only explicitly configured aliases. if !p.Unfiltered && len(p.Aliases) > 0 { - for aliasKey, providerID := range p.Aliases { + for aliasKey, alias := range p.Aliases { if included[strings.ToLower(aliasKey)] { continue } @@ -400,7 +400,7 @@ func (p *ListModelsPipeline) BackfillModels(included map[string]bool) []schemas. result = append(result, schemas.Model{ ID: string(p.ProviderKey) + "/" + aliasKey, Name: schemas.Ptr(ToDisplayName(aliasKey)), - Alias: schemas.Ptr(providerID), + Alias: schemas.Ptr(alias.ModelID), }) } } diff --git a/core/providers/utils/utils.go b/core/providers/utils/utils.go index f327412632..27f2ecb0d0 100644 --- a/core/providers/utils/utils.go +++ b/core/providers/utils/utils.go @@ -3056,36 +3056,6 @@ func completeDeferredSpan(ctx *schemas.BifrostContext, result *schemas.BifrostRe tracer.ClearDeferredSpan(traceID) } -// CheckAndSetDefaultProvider checks if the default provider should be used based on the context. -// It returns the default provider if it should be used, otherwise it returns an empty string. -// Checks if key selection is skipped, if a resolved provider was selected by routing, -// or if the available providers are set in the context and the default provider is in the list. -func CheckAndSetDefaultProvider(ctx *schemas.BifrostContext, defaultProvider schemas.ModelProvider) schemas.ModelProvider { - if ctx != nil { - if skip, ok := ctx.Value(schemas.BifrostContextKeySkipKeySelection).(bool); ok && skip { - return defaultProvider - } - if ctx.Value(schemas.BifrostContextKeyAvailableProviders) != nil { - availableProviders, ok := ctx.Value(schemas.BifrostContextKeyAvailableProviders).([]schemas.ModelProvider) - if !ok || len(availableProviders) == 0 { - return "" - } - if resolvedProvider, ok := ctx.Value(schemas.BifrostContextKeyResolvedProvider).(schemas.ModelProvider); ok && slices.Contains(availableProviders, resolvedProvider) { - getLogger().Debug("[Provider] Using routing-resolved provider: %s (available: %v)", resolvedProvider, availableProviders) - return resolvedProvider - } - getLogger().Debug("[Provider] Available providers: %v, checking %s", availableProviders, defaultProvider) - if slices.Contains(availableProviders, defaultProvider) { - return defaultProvider - } - // Return the first available provider - return availableProviders[0] - } - return defaultProvider - } - return defaultProvider -} - // ModelMatchesDenylist reports whether any of the candidate model IDs matches // an entry in denylist, using both exact and base-model (SameBaseModel) matching. // Empty candidates are skipped. Returns false immediately if denylist is empty. diff --git a/core/providers/utils/utils_test.go b/core/providers/utils/utils_test.go index 66db016271..517da6bbaf 100644 --- a/core/providers/utils/utils_test.go +++ b/core/providers/utils/utils_test.go @@ -1878,27 +1878,3 @@ func TestExtractPassthroughProviderResponseHeaders(t *testing.T) { t.Fatalf("benign header x-request-id was dropped: %v", headers) } } - -// TestCheckAndSetDefaultProviderUsesResolvedProvider verifies routing-selected -// providers take precedence over the route default when still allowed. -func TestCheckAndSetDefaultProviderUsesResolvedProvider(t *testing.T) { - ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) - ctx.SetValue(schemas.BifrostContextKeyAvailableProviders, []schemas.ModelProvider{schemas.Anthropic, schemas.Azure}) - ctx.SetValue(schemas.BifrostContextKeyResolvedProvider, schemas.Azure) - - if got := CheckAndSetDefaultProvider(ctx, schemas.Anthropic); got != schemas.Azure { - t.Fatalf("CheckAndSetDefaultProvider() = %s, want %s", got, schemas.Azure) - } -} - -// TestCheckAndSetDefaultProviderIgnoresDisallowedResolvedProvider verifies -// selected-provider context cannot bypass available-provider constraints. -func TestCheckAndSetDefaultProviderIgnoresDisallowedResolvedProvider(t *testing.T) { - ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) - ctx.SetValue(schemas.BifrostContextKeyAvailableProviders, []schemas.ModelProvider{schemas.Anthropic}) - ctx.SetValue(schemas.BifrostContextKeyResolvedProvider, schemas.Azure) - - if got := CheckAndSetDefaultProvider(ctx, schemas.Anthropic); got != schemas.Anthropic { - t.Fatalf("CheckAndSetDefaultProvider() = %s, want %s", got, schemas.Anthropic) - } -} diff --git a/core/providers/vertex/batch.go b/core/providers/vertex/batch.go new file mode 100644 index 0000000000..1d0521c3fb --- /dev/null +++ b/core/providers/vertex/batch.go @@ -0,0 +1,433 @@ +package vertex + +import ( + "bytes" + "fmt" + "strconv" + "strings" + "time" + + "github.com/bytedance/sonic" + providerUtils "github.com/maximhq/bifrost/core/providers/utils" + "github.com/maximhq/bifrost/core/schemas" +) + +// vertexBatchCustomIDLabel is the request label used to carry the Bifrost custom_id +// through a batch prediction job (Vertex JSONL has no native custom_id field; the +// request — labels included — is echoed back in each output line). +const vertexBatchCustomIDLabel = "bifrost_custom_id" + +// vertexJobStateToBatchStatus maps Vertex JOB_STATE_* values to Bifrost batch statuses. +func vertexJobStateToBatchStatus(state string) schemas.BatchStatus { + switch state { + case "JOB_STATE_QUEUED", "JOB_STATE_PENDING": + return schemas.BatchStatusValidating + case "JOB_STATE_RUNNING", "JOB_STATE_UPDATING": + return schemas.BatchStatusInProgress + case "JOB_STATE_SUCCEEDED", "JOB_STATE_PARTIALLY_SUCCEEDED": + return schemas.BatchStatusCompleted + case "JOB_STATE_FAILED": + return schemas.BatchStatusFailed + case "JOB_STATE_CANCELLING": + return schemas.BatchStatusCancelling + case "JOB_STATE_CANCELLED": + return schemas.BatchStatusCancelled + case "JOB_STATE_EXPIRED": + return schemas.BatchStatusExpired + default: + return schemas.BatchStatus(state) + } +} + +// vertexBatchJobsBaseURL returns ".../v1/projects/{project}/locations/{region}" for the +// key's configured project and region. Batch prediction requires a regional endpoint. +func vertexBatchJobsBaseURL(key schemas.Key) (string, *schemas.BifrostError) { + if key.VertexKeyConfig == nil { + return "", providerUtils.NewConfigurationError("vertex key config is not set") + } + projectID := strings.TrimSpace(key.VertexKeyConfig.ProjectID.GetValue()) + if projectID == "" { + return "", providerUtils.NewConfigurationError("project ID is not set") + } + region := key.VertexKeyConfig.Region.GetValue() + if region == "" { + return "", providerUtils.NewConfigurationError("region is required for batch prediction") + } + return getVertexProjectLocationURL(region, "v1", projectID), nil +} + +// vertexBatchJobURL resolves a Bifrost batch ID (bare job ID or full resource name) +// to the job's REST URL. +func vertexBatchJobURL(key schemas.Key, batchID string) (string, *schemas.BifrostError) { + if strings.HasPrefix(batchID, "projects/") { + // Full resource name: projects/{p}/locations/{r}/batchPredictionJobs/{id} + parts := strings.Split(batchID, "/") + if len(parts) >= 6 && parts[2] == "locations" { + return getVertexAPIBaseURL(parts[3], "v1") + "/" + batchID, nil + } + return "", providerUtils.NewBifrostOperationError(fmt.Sprintf("invalid Vertex batch ID %q", batchID), nil) + } + base, cfgErr := vertexBatchJobsBaseURL(key) + if cfgErr != nil { + return "", cfgErr + } + return base + "/batchPredictionJobs/" + batchID, nil +} + +// vertexBatchJobToBifrost maps a BatchPredictionJob resource to the Bifrost retrieve response. +func vertexBatchJobToBifrost(job *VertexBatchPredictionJob) schemas.BifrostBatchRetrieveResponse { + status := vertexJobStateToBatchStatus(job.State) + resp := schemas.BifrostBatchRetrieveResponse{ + ID: job.Name, + Object: "batch", + Status: status, + CreatedAt: gcsParseTime(job.CreateTime), + } + if job.DisplayName != "" { + resp.DisplayName = schemas.Ptr(job.DisplayName) + } + if job.InputConfig.GcsSource != nil && len(job.InputConfig.GcsSource.Uris) > 0 { + resp.InputFileID = job.InputConfig.GcsSource.Uris[0] + } + if job.OutputInfo != nil && job.OutputInfo.GcsOutputDirectory != "" { + resp.OutputFileID = schemas.Ptr(job.OutputInfo.GcsOutputDirectory) + } + if job.StartTime != "" { + resp.InProgressAt = schemas.Ptr(gcsParseTime(job.StartTime)) + } + if job.EndTime != "" { + endTime := gcsParseTime(job.EndTime) + switch status { + case schemas.BatchStatusCompleted: + resp.CompletedAt = &endTime + case schemas.BatchStatusFailed: + resp.FailedAt = &endTime + case schemas.BatchStatusCancelled: + resp.CancelledAt = &endTime + case schemas.BatchStatusExpired: + resp.ExpiredAt = &endTime + } + } + if job.CompletionStats != nil { + succeeded := gcsParseSize(job.CompletionStats.SuccessfulCount) + failed := gcsParseSize(job.CompletionStats.FailedCount) + incomplete := gcsParseSize(job.CompletionStats.IncompleteCount) + resp.RequestCounts = schemas.BatchRequestCounts{ + Total: int(succeeded + failed + incomplete), + Completed: int(succeeded), + Failed: int(failed), + } + } + if job.Error != nil && job.Error.Message != "" { + resp.Errors = &schemas.BatchErrors{ + Data: []schemas.BatchError{{Code: fmt.Sprintf("%d", job.Error.Code), Message: job.Error.Message}}, + } + } + return resp +} + +// parseVertexJobAPIError parses a Vertex AI error response (same envelope as GCS). +func parseVertexJobAPIError(body []byte, statusCode int, op string) *schemas.BifrostError { + var apiErr gcsErrorBody + _ = sonic.Unmarshal(body, &apiErr) + msg := apiErr.Error.Message + if msg == "" { + msg = fmt.Sprintf("Vertex %s failed with HTTP %d", op, statusCode) + } + return providerUtils.NewProviderAPIError(msg, nil, statusCode, nil, nil) +} + +// ToVertexBatchCreateRequest maps a Bifrost batch create request to a Vertex +// BatchPredictionJob request. The model, display name and input/output GCS config are +// mapped explicitly; every other field is taken from extra_params (e.g. modelParameters, +// labels, modelVersionId, encryptionSpec) and merged verbatim into the job body. +func ToVertexBatchCreateRequest(request *schemas.BifrostBatchCreateRequest, displayName, inputURI, outputURI string) *VertexBatchCreateRequest { + model := "" + if request.Model != nil { + model = *request.Model + } + if model != "" && !strings.Contains(model, "/") { + model = "publishers/google/models/" + model + } + + req := &VertexBatchCreateRequest{ + DisplayName: displayName, + Model: model, + InputConfig: VertexBatchInputConfig{ + InstancesFormat: "jsonl", + GcsSource: &VertexGcsSource{Uris: []string{inputURI}}, + }, + OutputConfig: VertexBatchOutputConfig{ + PredictionsFormat: "jsonl", + GcsDestination: &VertexGcsDestination{OutputUriPrefix: outputURI}, + }, + ExtraParams: request.ExtraParams, + } + + return req +} + +// vertexConvertRequestsToJSONL converts inline batch request items to Vertex batch JSONL. +// Bodies are passed through as-is (callers provide Gemini-native request bodies, mirroring +// the Anthropic/Bedrock providers); each custom_id is carried in request labels. +func vertexConvertRequestsToJSONL(requests []schemas.BatchRequestItem) ([]byte, error) { + var buf bytes.Buffer + for i, item := range requests { + body := item.Body + if body == nil { + body = item.Params + } + if body == nil { + return nil, fmt.Errorf("batch request item %d (custom_id %q) has no body", i, item.CustomID) + } + if item.CustomID != "" { + // Shallow-copy before injecting labels so the caller's map is not mutated. + withLabels := make(map[string]interface{}, len(body)+1) + for k, v := range body { + withLabels[k] = v + } + labels := map[string]interface{}{} + if existing, ok := withLabels["labels"].(map[string]interface{}); ok { + for k, v := range existing { + labels[k] = v + } + } + labels[vertexBatchCustomIDLabel] = item.CustomID + withLabels["labels"] = labels + body = withLabels + } + line, err := providerUtils.MarshalSorted(map[string]interface{}{"request": body}) + if err != nil { + return nil, fmt.Errorf("failed to marshal batch request item %d (custom_id %q): %w", i, item.CustomID, err) + } + buf.Write(line) + buf.WriteByte('\n') + } + return buf.Bytes(), nil +} + +// ============================ Integration Converters ============================ +// Convert between the native Vertex BatchPredictionJob wire shape (used by the aiplatform +// JobServiceClient) and Bifrost's neutral batch types, for the genai HTTP integration. +// Key/project selection happens in Bifrost from the vertex key config, so the project and +// location in the inbound request path are placeholders — only the job body is converted. + +// batchStatusToVertexJobState is the inverse of vertexJobStateToBatchStatus. +func batchStatusToVertexJobState(status schemas.BatchStatus) string { + switch status { + case schemas.BatchStatusValidating: + return "JOB_STATE_PENDING" + case schemas.BatchStatusInProgress, schemas.BatchStatusFinalizing: + return "JOB_STATE_RUNNING" + case schemas.BatchStatusCompleted, schemas.BatchStatusEnded: + return "JOB_STATE_SUCCEEDED" + case schemas.BatchStatusFailed: + return "JOB_STATE_FAILED" + case schemas.BatchStatusCancelling: + return "JOB_STATE_CANCELLING" + case schemas.BatchStatusCancelled: + return "JOB_STATE_CANCELLED" + case schemas.BatchStatusExpired: + return "JOB_STATE_EXPIRED" + default: + return "JOB_STATE_UNSPECIFIED" + } +} + +// formatVertexBatchTime renders a Unix timestamp as an RFC3339 string, empty when zero. +func formatVertexBatchTime(unix int64) string { + if unix <= 0 { + return "" + } + return time.Unix(unix, 0).UTC().Format(time.RFC3339) +} + +// vertexCompletionStatsFromCounts maps Bifrost request counts to Vertex completion stats. +func vertexCompletionStatsFromCounts(c schemas.BatchRequestCounts) *VertexBatchCompletionStats { + if c.Total == 0 && c.Completed == 0 && c.Failed == 0 { + return nil + } + incomplete := c.Total - c.Completed - c.Failed + if incomplete < 0 { + incomplete = 0 + } + return &VertexBatchCompletionStats{ + SuccessfulCount: strconv.Itoa(c.Completed), + FailedCount: strconv.Itoa(c.Failed), + IncompleteCount: strconv.Itoa(incomplete), + } +} + +// ToBifrostBatchCreateRequest maps an inbound native Vertex BatchPredictionJob (as sent by +// the aiplatform JobServiceClient) to a Bifrost batch create request. The model, GCS input +// URI and display name are mapped to typed Bifrost fields; the GCS output prefix and every +// other Vertex-native create-input field (modelParameters, labels, modelVersionId, +// encryptionSpec, instanceConfig, ...) are carried through ExtraParams keyed by their Vertex +// JSON names, so ToVertexBatchCreateRequest can merge them back into the job body verbatim +// for a lossless round trip. Server-populated, output-only fields (state, outputInfo, error, +// timestamps, completionStats, partialFailures, satisfiesPz*, ...) are intentionally omitted. +func ToBifrostBatchCreateRequest(job *VertexBatchPredictionJob) *schemas.BifrostBatchCreateRequest { + req := &schemas.BifrostBatchCreateRequest{Provider: schemas.Vertex} + if job == nil { + return req + } + if job.Model != "" { + req.Model = schemas.Ptr(job.Model) + } + if job.InputConfig.GcsSource != nil && len(job.InputConfig.GcsSource.Uris) > 0 { + req.InputFileID = job.InputConfig.GcsSource.Uris[0] + } + // Display name maps to the typed DisplayName field (read back by BatchCreate to set + // the outbound Vertex displayName) so it survives a full round trip. + if job.DisplayName != "" { + req.DisplayName = schemas.Ptr(job.DisplayName) + } + + // Output destination maps to the typed OutputFolder (read back by BatchCreate as the + // gs:// output prefix), so it survives a full round trip without going through extra_params. + if job.OutputConfig.GcsDestination != nil && job.OutputConfig.GcsDestination.OutputUriPrefix != "" { + req.OutputFolder = &schemas.BatchOutputFolder{URL: job.OutputConfig.GcsDestination.OutputUriPrefix} + } + + // Remaining create-input fields → ExtraParams, keyed by their Vertex JSON names so they + // merge cleanly into the outbound BatchPredictionJob body. Each is guarded so zero values + // are not forwarded (mirroring the native struct's omitempty tags). + extra := map[string]interface{}{} + if job.ModelVersionID != "" { + extra["modelVersionId"] = job.ModelVersionID + } + if job.UnmanagedContainerModel != nil { + extra["unmanagedContainerModel"] = job.UnmanagedContainerModel + } + if job.InstanceConfig != nil { + extra["instanceConfig"] = job.InstanceConfig + } + if job.ModelParameters != nil { + extra["modelParameters"] = job.ModelParameters + } + if job.DedicatedResources != nil { + extra["dedicatedResources"] = job.DedicatedResources + } + if job.ServiceAccount != "" { + extra["serviceAccount"] = job.ServiceAccount + } + if job.ManualBatchTuningParameters != nil { + extra["manualBatchTuningParameters"] = job.ManualBatchTuningParameters + } + if job.GenerateExplanation { + extra["generateExplanation"] = job.GenerateExplanation + } + if len(job.ExplanationSpec) > 0 { + extra["explanationSpec"] = job.ExplanationSpec + } + if len(job.Labels) > 0 { + extra["labels"] = job.Labels + } + if job.EncryptionSpec != nil { + extra["encryptionSpec"] = job.EncryptionSpec + } + if len(job.ModelMonitoringConfig) > 0 { + extra["modelMonitoringConfig"] = job.ModelMonitoringConfig + } + if job.DisableContainerLogging { + extra["disableContainerLogging"] = job.DisableContainerLogging + } + if len(extra) > 0 { + req.ExtraParams = extra + } + return req +} + +// vertexBatchJobShell builds the BatchPredictionJob fields shared by the create and retrieve +// response converters. name is whatever Bifrost returns (bare id or full resource name); +// displayName is the human-readable job name, kept distinct from name. +func vertexBatchJobShell(name, displayName string, status schemas.BatchStatus, createdAt int64, inputFileID string, outputFileID *string) *VertexBatchPredictionJob { + job := &VertexBatchPredictionJob{ + Name: name, + DisplayName: displayName, + State: batchStatusToVertexJobState(status), + CreateTime: formatVertexBatchTime(createdAt), + } + if inputFileID != "" { + job.InputConfig = VertexBatchInputConfig{ + InstancesFormat: "jsonl", + GcsSource: &VertexGcsSource{Uris: []string{inputFileID}}, + } + } + if outputFileID != nil && *outputFileID != "" { + job.OutputConfig = VertexBatchOutputConfig{ + PredictionsFormat: "jsonl", + GcsDestination: &VertexGcsDestination{OutputUriPrefix: *outputFileID}, + } + job.OutputInfo = &VertexBatchOutputInfo{GcsOutputDirectory: *outputFileID} + } + return job +} + +// ToVertexBatchCreateResponse maps a Bifrost batch create response to a native Vertex +// BatchPredictionJob. +func ToVertexBatchCreateResponse(resp *schemas.BifrostBatchCreateResponse) *VertexBatchPredictionJob { + if resp == nil { + return nil + } + displayName := "" + if resp.DisplayName != nil { + displayName = *resp.DisplayName + } + job := vertexBatchJobShell(resp.ID, displayName, resp.Status, resp.CreatedAt, resp.InputFileID, resp.OutputFileID) + job.CompletionStats = vertexCompletionStatsFromCounts(resp.RequestCounts) + return job +} + +// ToVertexBatchRetrieveResponse maps a Bifrost batch retrieve response to a native Vertex +// BatchPredictionJob, including timestamps, completion stats and any terminal error. +func ToVertexBatchRetrieveResponse(resp *schemas.BifrostBatchRetrieveResponse) *VertexBatchPredictionJob { + if resp == nil { + return nil + } + displayName := "" + if resp.DisplayName != nil { + displayName = *resp.DisplayName + } + job := vertexBatchJobShell(resp.ID, displayName, resp.Status, resp.CreatedAt, resp.InputFileID, resp.OutputFileID) + if resp.InProgressAt != nil { + job.StartTime = formatVertexBatchTime(*resp.InProgressAt) + } + switch { + case resp.CompletedAt != nil: + job.EndTime = formatVertexBatchTime(*resp.CompletedAt) + case resp.FailedAt != nil: + job.EndTime = formatVertexBatchTime(*resp.FailedAt) + case resp.CancelledAt != nil: + job.EndTime = formatVertexBatchTime(*resp.CancelledAt) + case resp.ExpiredAt != nil: + job.EndTime = formatVertexBatchTime(*resp.ExpiredAt) + } + job.CompletionStats = vertexCompletionStatsFromCounts(resp.RequestCounts) + if resp.Errors != nil && len(resp.Errors.Data) > 0 { + code := 0 + if c, err := strconv.Atoi(resp.Errors.Data[0].Code); err == nil { + code = c + } + job.Error = &VertexBatchJobError{Code: code, Message: resp.Errors.Data[0].Message} + } + return job +} + +// ToVertexBatchListResponse maps a Bifrost batch list response to the native Vertex +// batchPredictionJobs.list response envelope. +func ToVertexBatchListResponse(resp *schemas.BifrostBatchListResponse) *VertexBatchJobListResponse { + out := &VertexBatchJobListResponse{} + if resp == nil { + return out + } + for i := range resp.Data { + if job := ToVertexBatchRetrieveResponse(&resp.Data[i]); job != nil { + out.BatchPredictionJobs = append(out.BatchPredictionJobs, *job) + } + } + if resp.NextCursor != nil { + out.NextPageToken = *resp.NextCursor + } + return out +} diff --git a/core/providers/vertex/cachedcontents.go b/core/providers/vertex/cachedcontents.go index 73bf8d1dd8..05adb24bcb 100644 --- a/core/providers/vertex/cachedcontents.go +++ b/core/providers/vertex/cachedcontents.go @@ -115,13 +115,13 @@ func (provider *VertexProvider) CachedContentCreate(ctx *schemas.BifrostContext, return nil, providerUtils.NewBifrostOperationError("model is required for cached content create", nil) } - projectID := key.VertexKeyConfig.ProjectID.GetValue() + projectID := resolveVertexProjectID(ctx, key) if projectID == "" { - return nil, providerUtils.NewConfigurationError("project_id is not set in vertex key config") + return nil, providerUtils.NewConfigurationError("project_id is not set") } - region := key.VertexKeyConfig.Region.GetValue() + region := resolveVertexRegion(ctx, key) if region == "" { - return nil, providerUtils.NewConfigurationError("region is not set in vertex key config") + return nil, providerUtils.NewConfigurationError("region is not set") } model := expandVertexModelPath(request.Model, projectID, region) @@ -210,13 +210,13 @@ func (provider *VertexProvider) CachedContentCreate(ctx *schemas.BifrostContext, } func (provider *VertexProvider) cachedContentListByKey(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostCachedContentListRequest) (*schemas.BifrostCachedContentListResponse, time.Duration, *schemas.BifrostError) { - projectID := key.VertexKeyConfig.ProjectID.GetValue() + projectID := resolveVertexProjectID(ctx, key) if projectID == "" { - return nil, 0, providerUtils.NewConfigurationError("project_id is not set in vertex key config") + return nil, 0, providerUtils.NewConfigurationError("project_id is not set") } - region := key.VertexKeyConfig.Region.GetValue() + region := resolveVertexRegion(ctx, key) if region == "" { - return nil, 0, providerUtils.NewConfigurationError("region is not set in vertex key config") + return nil, 0, providerUtils.NewConfigurationError("region is not set") } req := fasthttp.AcquireRequest() @@ -292,13 +292,13 @@ func (provider *VertexProvider) CachedContentList(ctx *schemas.BifrostContext, k } func (provider *VertexProvider) cachedContentRetrieveByKey(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostCachedContentRetrieveRequest) (*schemas.BifrostCachedContentRetrieveResponse, time.Duration, *schemas.BifrostError) { - projectID := key.VertexKeyConfig.ProjectID.GetValue() + projectID := resolveVertexProjectID(ctx, key) if projectID == "" { - return nil, 0, providerUtils.NewConfigurationError("project_id is not set in vertex key config") + return nil, 0, providerUtils.NewConfigurationError("project_id is not set") } - region := key.VertexKeyConfig.Region.GetValue() + region := resolveVertexRegion(ctx, key) if region == "" { - return nil, 0, providerUtils.NewConfigurationError("region is not set in vertex key config") + return nil, 0, providerUtils.NewConfigurationError("region is not set") } req := fasthttp.AcquireRequest() @@ -372,13 +372,13 @@ func (provider *VertexProvider) CachedContentRetrieve(ctx *schemas.BifrostContex } func (provider *VertexProvider) cachedContentUpdateByKey(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostCachedContentUpdateRequest) (*schemas.BifrostCachedContentUpdateResponse, time.Duration, *schemas.BifrostError) { - projectID := key.VertexKeyConfig.ProjectID.GetValue() + projectID := resolveVertexProjectID(ctx, key) if projectID == "" { - return nil, 0, providerUtils.NewConfigurationError("project_id is not set in vertex key config") + return nil, 0, providerUtils.NewConfigurationError("project_id is not set") } - region := key.VertexKeyConfig.Region.GetValue() + region := resolveVertexRegion(ctx, key) if region == "" { - return nil, 0, providerUtils.NewConfigurationError("region is not set in vertex key config") + return nil, 0, providerUtils.NewConfigurationError("region is not set") } body := vertexCachedContent{} @@ -482,13 +482,13 @@ func (provider *VertexProvider) CachedContentUpdate(ctx *schemas.BifrostContext, } func (provider *VertexProvider) cachedContentDeleteByKey(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostCachedContentDeleteRequest) (*schemas.BifrostCachedContentDeleteResponse, time.Duration, *schemas.BifrostError) { - projectID := key.VertexKeyConfig.ProjectID.GetValue() + projectID := resolveVertexProjectID(ctx, key) if projectID == "" { - return nil, 0, providerUtils.NewConfigurationError("project_id is not set in vertex key config") + return nil, 0, providerUtils.NewConfigurationError("project_id is not set") } - region := key.VertexKeyConfig.Region.GetValue() + region := resolveVertexRegion(ctx, key) if region == "" { - return nil, 0, providerUtils.NewConfigurationError("region is not set in vertex key config") + return nil, 0, providerUtils.NewConfigurationError("region is not set") } req := fasthttp.AcquireRequest() diff --git a/core/providers/vertex/models.go b/core/providers/vertex/models.go index d373f58735..e27db45bd9 100644 --- a/core/providers/vertex/models.go +++ b/core/providers/vertex/models.go @@ -70,7 +70,7 @@ type vertexRerankOptions struct { // - If allowedModels is empty, all models are allowed // - If allowedModels is non-empty, only models/deployments with keys in allowedModels are included // - Deployments map is used to match model IDs to aliases and filter accordingly -func (response *VertexListModelsResponse) ToBifrostListModelsResponse(allowedModels schemas.WhiteList, blacklistedModels schemas.BlackList, aliases map[string]string, unfiltered bool) *schemas.BifrostListModelsResponse { +func (response *VertexListModelsResponse) ToBifrostListModelsResponse(allowedModels schemas.WhiteList, blacklistedModels schemas.BlackList, aliases schemas.KeyAliases, unfiltered bool) *schemas.BifrostListModelsResponse { if response == nil { return nil } @@ -140,7 +140,7 @@ func (response *VertexListModelsResponse) ToBifrostListModelsResponse(allowedMod // ToBifrostListModelsResponse converts a Vertex AI publisher models response to Bifrost's format. // This is for foundation models from the Model Garden (publishers.models.list endpoint). -func (response *VertexListPublisherModelsResponse) ToBifrostListModelsResponse(allowedModels schemas.WhiteList, blacklistedModels schemas.BlackList, aliases map[string]string, unfiltered bool) *schemas.BifrostListModelsResponse { +func (response *VertexListPublisherModelsResponse) ToBifrostListModelsResponse(allowedModels schemas.WhiteList, blacklistedModels schemas.BlackList, aliases schemas.KeyAliases, unfiltered bool) *schemas.BifrostListModelsResponse { if response == nil { return nil } diff --git a/core/providers/vertex/rerank.go b/core/providers/vertex/rerank.go index 257a1f8def..af2e68d25c 100644 --- a/core/providers/vertex/rerank.go +++ b/core/providers/vertex/rerank.go @@ -7,7 +7,6 @@ import ( "strings" "github.com/bytedance/sonic" - providerUtils "github.com/maximhq/bifrost/core/providers/utils" "github.com/maximhq/bifrost/core/schemas" ) @@ -157,9 +156,9 @@ func (req *VertexRankRequest) ToBifrostRerankRequest(ctx *schemas.BifrostContext var provider schemas.ModelProvider var model string if req.Model != nil { - provider, model = schemas.ParseModelString(*req.Model, providerUtils.CheckAndSetDefaultProvider(ctx, schemas.Vertex)) + provider, model = schemas.ParseModelString(*req.Model, schemas.Vertex) } else { - provider = providerUtils.CheckAndSetDefaultProvider(ctx, schemas.Vertex) + provider = schemas.Vertex } bifrostReq := &schemas.BifrostRerankRequest{ diff --git a/core/providers/vertex/types.go b/core/providers/vertex/types.go index 3a3db62f6b..f155426bde 100644 --- a/core/providers/vertex/types.go +++ b/core/providers/vertex/types.go @@ -246,3 +246,225 @@ type VertexCountTokensResponse struct { TotalTokens int32 `json:"totalTokens,omitempty"` CachedContentTokenCount int32 `json:"cachedContentTokenCount,omitempty"` } + +// ================================ Batch Prediction API Types ================================ + +// VertexGcsSource is the GCS input source for a batch prediction job. +type VertexGcsSource struct { + Uris []string `json:"uris"` +} + +// VertexBigQuerySource is the BigQuery input source for a batch prediction job. +type VertexBigQuerySource struct { + InputUri string `json:"inputUri"` +} + +// VertexBatchInputConfig is the input configuration for a batch prediction job. +type VertexBatchInputConfig struct { + InstancesFormat string `json:"instancesFormat"` + GcsSource *VertexGcsSource `json:"gcsSource,omitempty"` + BigquerySource *VertexBigQuerySource `json:"bigquerySource,omitempty"` +} + +// VertexBatchInstanceConfig controls how input instances are converted to prediction instances. +type VertexBatchInstanceConfig struct { + InstanceType string `json:"instanceType,omitempty"` + KeyField string `json:"keyField,omitempty"` + IncludedFields []string `json:"includedFields,omitempty"` + ExcludedFields []string `json:"excludedFields,omitempty"` +} + +// VertexGcsDestination is the GCS output destination for a batch prediction job. +type VertexGcsDestination struct { + OutputUriPrefix string `json:"outputUriPrefix"` +} + +// VertexBigQueryDestination is the BigQuery output destination for a batch prediction job. +type VertexBigQueryDestination struct { + OutputUri string `json:"outputUri"` +} + +// VertexBatchOutputConfig is the output configuration for a batch prediction job. +type VertexBatchOutputConfig struct { + PredictionsFormat string `json:"predictionsFormat"` + GcsDestination *VertexGcsDestination `json:"gcsDestination,omitempty"` + BigqueryDestination *VertexBigQueryDestination `json:"bigqueryDestination,omitempty"` +} + +// VertexBatchOutputInfo describes where a finished job wrote its output. +type VertexBatchOutputInfo struct { + GcsOutputDirectory string `json:"gcsOutputDirectory,omitempty"` + BigqueryOutputDataset string `json:"bigqueryOutputDataset,omitempty"` + BigqueryOutputTable string `json:"bigqueryOutputTable,omitempty"` +} + +// VertexBatchCompletionStats tracks per-request completion counts of a job. +type VertexBatchCompletionStats struct { + SuccessfulCount string `json:"successfulCount"` // int64 serialised as string + FailedCount string `json:"failedCount"` // int64 serialised as string + IncompleteCount string `json:"incompleteCount"` // int64 serialised as string + SuccessfulForecastPointCount string `json:"successfulForecastPointCount,omitempty"` // int64 serialised as string +} + +// VertexResourcesConsumed reports resources consumed by a batch prediction job. +type VertexResourcesConsumed struct { + ReplicaHours float64 `json:"replicaHours,omitempty"` +} + +// VertexManualBatchTuningParameters configures batch behaviour (only with dedicatedResources). +type VertexManualBatchTuningParameters struct { + BatchSize int `json:"batchSize,omitempty"` +} + +// VertexReservationAffinity configures the reservation a MachineSpec draws resources from. +type VertexReservationAffinity struct { + ReservationAffinityType string `json:"reservationAffinityType,omitempty"` + Key string `json:"key,omitempty"` + Values []string `json:"values,omitempty"` +} + +// VertexMachineSpec is the compute machine configuration for dedicated resources. +type VertexMachineSpec struct { + MachineType string `json:"machineType,omitempty"` + AcceleratorType string `json:"acceleratorType,omitempty"` + AcceleratorCount int `json:"acceleratorCount,omitempty"` + TpuTopology string `json:"tpuTopology,omitempty"` + ReservationAffinity *VertexReservationAffinity `json:"reservationAffinity,omitempty"` +} + +// VertexBatchDedicatedResources is the dedicated compute config used during batch prediction. +type VertexBatchDedicatedResources struct { + MachineSpec *VertexMachineSpec `json:"machineSpec,omitempty"` + StartingReplicaCount int `json:"startingReplicaCount,omitempty"` + MaxReplicaCount int `json:"maxReplicaCount,omitempty"` +} + +// VertexEncryptionSpec is the customer-managed encryption key configuration. +type VertexEncryptionSpec struct { + KmsKeyName string `json:"kmsKeyName,omitempty"` +} + +// VertexPredictSchemata describes the instance/parameter/prediction schemas of a model. +type VertexPredictSchemata struct { + InstanceSchemaUri string `json:"instanceSchemaUri,omitempty"` + ParametersSchemaUri string `json:"parametersSchemaUri,omitempty"` + PredictionSchemaUri string `json:"predictionSchemaUri,omitempty"` +} + +// VertexUnmanagedContainerModel describes a model used without registry upload. +type VertexUnmanagedContainerModel struct { + ArtifactUri string `json:"artifactUri,omitempty"` + PredictSchemata *VertexPredictSchemata `json:"predictSchemata,omitempty"` + // ContainerSpec (ModelContainerSpec) is deeply nested; kept generic for passthrough. + ContainerSpec map[string]interface{} `json:"containerSpec,omitempty"` +} + +// VertexBatchJobError mirrors google.rpc.Status. Used for the job's terminal error as well +// as partialFailures and modelMonitoringStatus. The details array holds google.protobuf.Any +// entries with no fixed schema, so it is kept generic. +type VertexBatchJobError struct { + Code int `json:"code"` + Message string `json:"message"` + Details []map[string]interface{} `json:"details,omitempty"` +} + +// VertexBatchPredictionJob is the BatchPredictionJob resource returned by the Vertex AI API. +// Fields Bifrost interprets are typed; the deeply-nested explanation/monitoring config trees +// (rarely used for Gemini batch) are kept generic for lossless passthrough. +type VertexBatchPredictionJob struct { + Name string `json:"name,omitempty"` + DisplayName string `json:"displayName"` + Model string `json:"model,omitempty"` + ModelVersionID string `json:"modelVersionId,omitempty"` + UnmanagedContainerModel *VertexUnmanagedContainerModel `json:"unmanagedContainerModel,omitempty"` + InputConfig VertexBatchInputConfig `json:"inputConfig"` + InstanceConfig *VertexBatchInstanceConfig `json:"instanceConfig,omitempty"` + ModelParameters interface{} `json:"modelParameters,omitempty"` + OutputConfig VertexBatchOutputConfig `json:"outputConfig"` + DedicatedResources *VertexBatchDedicatedResources `json:"dedicatedResources,omitempty"` + ServiceAccount string `json:"serviceAccount,omitempty"` + ManualBatchTuningParameters *VertexManualBatchTuningParameters `json:"manualBatchTuningParameters,omitempty"` + GenerateExplanation bool `json:"generateExplanation,omitempty"` + // ExplanationSpec is deeply nested; kept generic for passthrough. + ExplanationSpec map[string]interface{} `json:"explanationSpec,omitempty"` + OutputInfo *VertexBatchOutputInfo `json:"outputInfo,omitempty"` + State string `json:"state,omitempty"` + Error *VertexBatchJobError `json:"error,omitempty"` + PartialFailures []VertexBatchJobError `json:"partialFailures,omitempty"` + ResourcesConsumed *VertexResourcesConsumed `json:"resourcesConsumed,omitempty"` + CompletionStats *VertexBatchCompletionStats `json:"completionStats,omitempty"` + CreateTime string `json:"createTime,omitempty"` // RFC3339 + StartTime string `json:"startTime,omitempty"` // RFC3339 + EndTime string `json:"endTime,omitempty"` // RFC3339 + UpdateTime string `json:"updateTime,omitempty"` // RFC3339 + Labels map[string]string `json:"labels,omitempty"` + EncryptionSpec *VertexEncryptionSpec `json:"encryptionSpec,omitempty"` + // ModelMonitoringConfig / ModelMonitoringStatsAnomalies are deeply nested; kept generic. + ModelMonitoringConfig map[string]interface{} `json:"modelMonitoringConfig,omitempty"` + ModelMonitoringStatsAnomalies []map[string]interface{} `json:"modelMonitoringStatsAnomalies,omitempty"` + ModelMonitoringStatus *VertexBatchJobError `json:"modelMonitoringStatus,omitempty"` + DisableContainerLogging bool `json:"disableContainerLogging,omitempty"` + SatisfiesPzs bool `json:"satisfiesPzs,omitempty"` + SatisfiesPzi bool `json:"satisfiesPzi,omitempty"` +} + +// VertexBatchCreateRequest is the request body for creating a BatchPredictionJob. Only the +// fields Bifrost maps directly are typed; any other Vertex-native field (modelParameters, +// labels, modelVersionId, encryptionSpec, instanceConfig, ...) is passed through ExtraParams +// and merged into the body by CheckContextAndGetRequestBody. +type VertexBatchCreateRequest struct { + DisplayName string `json:"displayName"` + Model string `json:"model"` + InputConfig VertexBatchInputConfig `json:"inputConfig"` + OutputConfig VertexBatchOutputConfig `json:"outputConfig"` + + ExtraParams map[string]interface{} `json:"-"` +} + +// GetExtraParams implements the providerUtils.RequestBodyWithExtraParams interface. +func (r *VertexBatchCreateRequest) GetExtraParams() map[string]interface{} { + return r.ExtraParams +} + +// VertexBatchJobListResponse is the batchPredictionJobs.list response envelope. +type VertexBatchJobListResponse struct { + BatchPredictionJobs []VertexBatchPredictionJob `json:"batchPredictionJobs"` + NextPageToken string `json:"nextPageToken"` +} + +// VertexBatchOutputLine is one line of a predictions-*.jsonl batch output file. +// The original request is echoed back; labels carry the Bifrost custom_id. +type VertexBatchOutputLine struct { + Status string `json:"status,omitempty"` // error string for failed records, empty on success + Request struct { + Labels map[string]string `json:"labels"` + } `json:"request"` + Response map[string]interface{} `json:"response,omitempty"` +} + +// ================================ GCS File API Types ================================ + +// gcsObjectMetadata represents GCS object metadata as returned by the JSON API. +type gcsObjectMetadata struct { + Name string `json:"name"` + Bucket string `json:"bucket"` + Size string `json:"size"` // int64 serialised as string by GCS + ContentType string `json:"contentType"` + TimeCreated string `json:"timeCreated"` // RFC3339 + Updated string `json:"updated"` // RFC3339 + Metadata map[string]string `json:"metadata"` +} + +// gcsObjectListResponse is the GCS object list response envelope. +type gcsObjectListResponse struct { + NextPageToken string `json:"nextPageToken"` + Items []gcsObjectMetadata `json:"items"` +} + +// gcsErrorBody is the GCS API error response envelope. +type gcsErrorBody struct { + Error struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error"` +} diff --git a/core/providers/vertex/utils.go b/core/providers/vertex/utils.go index 7eb9161f6d..0fd3b3fcc1 100644 --- a/core/providers/vertex/utils.go +++ b/core/providers/vertex/utils.go @@ -10,6 +10,55 @@ import ( schemas "github.com/maximhq/bifrost/core/schemas" ) +// resolveVertexProjectID returns the GCP project ID for the current attempt. +// Priority: alias-level VertexAliasCfg.ProjectID > key-level +// VertexKeyConfig.ProjectID. Per-alias override lets one Vertex credential +// span deployments across distinct GCP projects (e.g. Anthropic models in +// one project, Gemini in another). +func resolveVertexProjectID(ctx *schemas.BifrostContext, key schemas.Key) string { + if ra := schemas.GetResolvedAlias(ctx); ra != nil && ra.Config != nil && ra.Config.VertexAliasCfg != nil && ra.Config.VertexAliasCfg.ProjectID != nil { + if v := ra.Config.VertexAliasCfg.ProjectID.GetValue(); v != "" { + return v + } + } + if key.VertexKeyConfig != nil { + return key.VertexKeyConfig.ProjectID.GetValue() + } + return "" +} + +// resolveVertexProjectNumber returns the GCP project number for the current +// attempt. Same precedence as resolveVertexProjectID. +func resolveVertexProjectNumber(ctx *schemas.BifrostContext, key schemas.Key) string { + if ra := schemas.GetResolvedAlias(ctx); ra != nil && ra.Config != nil && ra.Config.VertexAliasCfg != nil && ra.Config.VertexAliasCfg.ProjectNumber != nil { + if v := ra.Config.VertexAliasCfg.ProjectNumber.GetValue(); v != "" { + return v + } + } + if key.VertexKeyConfig != nil { + return key.VertexKeyConfig.ProjectNumber.GetValue() + } + return "" +} + +// resolveVertexRegion returns the Vertex region for the current attempt. +// Priority: alias-level AliasConfig.Region (top-level, shared with other +// providers) > key-level VertexKeyConfig.Region. Different Vertex model +// families publish in different regions (Anthropic on us-east5, Gemini on +// us-central1, …), so per-alias overrides let one credential reach all of +// them. +func resolveVertexRegion(ctx *schemas.BifrostContext, key schemas.Key) string { + if ra := schemas.GetResolvedAlias(ctx); ra != nil && ra.Config != nil && ra.Config.Region != nil { + if v := ra.Config.Region.GetValue(); v != "" { + return v + } + } + if key.VertexKeyConfig != nil { + return key.VertexKeyConfig.Region.GetValue() + } + return "" +} + // getRequestBodyForAnthropicResponses serializes a BifrostResponsesRequest into the Anthropic wire format for Vertex AI. // Compared to the native Anthropic path, it strips model/region fields, remaps tool versions, injects beta headers // into the request body (rather than HTTP headers), and pins the Anthropic API version to DefaultVertexAnthropicVersion. @@ -242,7 +291,7 @@ func vertexServiceTierHeaderValue(region string, model string, tier schemas.Bifr // buildResponseFromConfig builds a list models response from configured deployments and allowedModels. // This is used when the user has explicitly configured which models they want to use. -func buildResponseFromConfig(deployments map[string]string, allowedModels schemas.WhiteList, blacklistedModels schemas.BlackList) *schemas.BifrostListModelsResponse { +func buildResponseFromConfig(deployments schemas.KeyAliases, allowedModels schemas.WhiteList, blacklistedModels schemas.BlackList) *schemas.BifrostListModelsResponse { response := &schemas.BifrostListModelsResponse{ Data: make([]schemas.Model, 0), } @@ -272,7 +321,7 @@ func buildResponseFromConfig(deployments map[string]string, allowedModels schema modelEntry := schemas.Model{ ID: modelID, Name: schemas.Ptr(modelName), - Alias: schemas.Ptr(deploymentValue), + Alias: schemas.Ptr(deploymentValue.ModelID), } response.Data = append(response.Data, modelEntry) diff --git a/core/providers/vertex/utils_test.go b/core/providers/vertex/utils_test.go index cc2b209890..acf6a557ad 100644 --- a/core/providers/vertex/utils_test.go +++ b/core/providers/vertex/utils_test.go @@ -354,3 +354,146 @@ func TestVertexRegionToPool(t *testing.T) { }) } } + +// TestResolveVertexProjectID_AliasOverride verifies the per-alias ProjectID +// override lets one Vertex credential serve deployments across distinct GCP +// projects. +func TestResolveVertexProjectID_AliasOverride(t *testing.T) { + keyProject := "key-level-project" + aliasProject := "alias-level-project" + key := schemas.Key{ + VertexKeyConfig: &schemas.VertexKeyConfig{ + ProjectID: *schemas.NewEnvVar(keyProject), + }, + } + + if got := resolveVertexProjectID(nil, key); got != keyProject { + t.Errorf("nil ctx: got %q, want key-level %q", got, keyProject) + } + + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + if got := resolveVertexProjectID(ctx, key); got != keyProject { + t.Errorf("empty ctx: got %q, want key-level %q", got, keyProject) + } + + ctx.SetValue(schemas.BifrostContextKeyResolvedAlias, &schemas.ResolvedAlias{ + Key: "best-claude", + Config: &schemas.AliasConfig{ + ModelID: "claude-sonnet-4-5", + VertexAliasCfg: &schemas.VertexAliasCfg{ + ProjectID: schemas.NewEnvVar(aliasProject), + }, + }, + }) + if got := resolveVertexProjectID(ctx, key); got != aliasProject { + t.Errorf("alias override should win: got %q, want %q", got, aliasProject) + } + + // Empty alias ProjectID falls through to key-level. + ctx2 := schemas.NewBifrostContext(nil, schemas.NoDeadline) + ctx2.SetValue(schemas.BifrostContextKeyResolvedAlias, &schemas.ResolvedAlias{ + Key: "x", + Config: &schemas.AliasConfig{ + ModelID: "x", + VertexAliasCfg: &schemas.VertexAliasCfg{ + ProjectID: schemas.NewEnvVar(""), + }, + }, + }) + if got := resolveVertexProjectID(ctx2, key); got != keyProject { + t.Errorf("empty alias ProjectID should fall through: got %q, want %q", got, keyProject) + } +} + +// TestResolveVertexRegion_AliasOverride verifies the top-level +// AliasConfig.Region override for Vertex. +func TestResolveVertexRegion_AliasOverride(t *testing.T) { + keyRegion := "us-central1" + aliasRegion := "us-east5" + key := schemas.Key{ + VertexKeyConfig: &schemas.VertexKeyConfig{ + Region: *schemas.NewEnvVar(keyRegion), + }, + } + + if got := resolveVertexRegion(nil, key); got != keyRegion { + t.Errorf("nil ctx: got %q, want %q", got, keyRegion) + } + + ctx0 := schemas.NewBifrostContext(nil, schemas.NoDeadline) + if got := resolveVertexRegion(ctx0, key); got != keyRegion { + t.Errorf("empty ctx: got %q, want key-level %q", got, keyRegion) + } + + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + ctx.SetValue(schemas.BifrostContextKeyResolvedAlias, &schemas.ResolvedAlias{ + Key: "best-claude", + Config: &schemas.AliasConfig{ + ModelID: "claude-sonnet-4-5", + Region: schemas.NewEnvVar(aliasRegion), + }, + }) + if got := resolveVertexRegion(ctx, key); got != aliasRegion { + t.Errorf("alias Region should win: got %q, want %q", got, aliasRegion) + } + + ctx2 := schemas.NewBifrostContext(nil, schemas.NoDeadline) + ctx2.SetValue(schemas.BifrostContextKeyResolvedAlias, &schemas.ResolvedAlias{ + Key: "x", + Config: &schemas.AliasConfig{ + ModelID: "x", + Region: schemas.NewEnvVar(""), + }, + }) + if got := resolveVertexRegion(ctx2, key); got != keyRegion { + t.Errorf("empty alias Region should fall through: got %q, want %q", got, keyRegion) + } +} + +// TestResolveVertexProjectNumber_AliasOverride mirrors the ProjectID test. +func TestResolveVertexProjectNumber_AliasOverride(t *testing.T) { + keyNumber := "111111" + aliasNumber := "222222" + key := schemas.Key{ + VertexKeyConfig: &schemas.VertexKeyConfig{ + ProjectNumber: *schemas.NewEnvVar(keyNumber), + }, + } + + if got := resolveVertexProjectNumber(nil, key); got != keyNumber { + t.Errorf("nil ctx: got %q, want %q", got, keyNumber) + } + + ctx0 := schemas.NewBifrostContext(nil, schemas.NoDeadline) + if got := resolveVertexProjectNumber(ctx0, key); got != keyNumber { + t.Errorf("empty ctx: got %q, want key-level %q", got, keyNumber) + } + + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + ctx.SetValue(schemas.BifrostContextKeyResolvedAlias, &schemas.ResolvedAlias{ + Key: "x", + Config: &schemas.AliasConfig{ + ModelID: "x", + VertexAliasCfg: &schemas.VertexAliasCfg{ + ProjectNumber: schemas.NewEnvVar(aliasNumber), + }, + }, + }) + if got := resolveVertexProjectNumber(ctx, key); got != aliasNumber { + t.Errorf("alias ProjectNumber should win: got %q, want %q", got, aliasNumber) + } + + ctx2 := schemas.NewBifrostContext(nil, schemas.NoDeadline) + ctx2.SetValue(schemas.BifrostContextKeyResolvedAlias, &schemas.ResolvedAlias{ + Key: "x", + Config: &schemas.AliasConfig{ + ModelID: "x", + VertexAliasCfg: &schemas.VertexAliasCfg{ + ProjectNumber: schemas.NewEnvVar(""), + }, + }, + }) + if got := resolveVertexProjectNumber(ctx2, key); got != keyNumber { + t.Errorf("empty alias ProjectNumber should fall through: got %q, want %q", got, keyNumber) + } +} diff --git a/core/providers/vertex/vertex.go b/core/providers/vertex/vertex.go index 4656da7ac1..89451b435f 100644 --- a/core/providers/vertex/vertex.go +++ b/core/providers/vertex/vertex.go @@ -1,15 +1,19 @@ package vertex import ( + "bytes" "context" "crypto/sha256" "encoding/base64" "encoding/hex" "errors" "fmt" + "mime/multipart" "net/http" + "net/textproto" "net/url" "regexp" + "strconv" "strings" "sync" "time" @@ -19,6 +23,7 @@ import ( "golang.org/x/oauth2/google" "github.com/bytedance/sonic" + "github.com/google/uuid" "github.com/maximhq/bifrost/core/providers/anthropic" "github.com/maximhq/bifrost/core/providers/gemini" "github.com/maximhq/bifrost/core/providers/openai" @@ -96,13 +101,14 @@ func NewVertexProvider(config *schemas.ProviderConfig, logger schemas.Logger) (* config.CheckAndSetDefaults() requestTimeout := time.Second * time.Duration(config.NetworkConfig.DefaultRequestTimeoutInSeconds) client := &fasthttp.Client{ - ReadTimeout: requestTimeout, - WriteTimeout: requestTimeout, - MaxConnsPerHost: config.NetworkConfig.MaxConnsPerHost, - MaxIdleConnDuration: 30 * time.Second, - MaxConnWaitTimeout: requestTimeout, - MaxConnDuration: time.Second * time.Duration(schemas.DefaultMaxConnDurationInSeconds), - ConnPoolStrategy: fasthttp.FIFO, + ReadTimeout: requestTimeout, + WriteTimeout: requestTimeout, + MaxConnsPerHost: config.NetworkConfig.MaxConnsPerHost, + MaxIdleConnDuration: 30 * time.Second, + MaxConnWaitTimeout: requestTimeout, + MaxConnDuration: time.Second * time.Duration(schemas.DefaultMaxConnDurationInSeconds), + ConnPoolStrategy: fasthttp.FIFO, + DisablePathNormalizing: true, } client = providerUtils.ConfigureProxy(client, config.ProxyConfig, logger) client = providerUtils.ConfigureDialer(client, config.NetworkConfig.AllowPrivateNetwork) @@ -197,7 +203,7 @@ func (provider *VertexProvider) GetProviderKey() schemas.ModelProvider { // 1. If deployments or allowedModels are configured, return those (no API call needed) // 2. Otherwise, fetch from the publishers.models.list API endpoint (Model Garden) func (provider *VertexProvider) listModelsByKey(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostListModelsRequest) (*schemas.BifrostListModelsResponse, *schemas.BifrostError) { - region := key.VertexKeyConfig.Region.GetValue() + region := resolveVertexRegion(ctx, key) if region == "" { return nil, providerUtils.NewConfigurationError("region is not set in key config") } @@ -426,7 +432,7 @@ func (provider *VertexProvider) ChatCompletion(ctx *schemas.BifrostContext, key var extraParams map[string]interface{} var err error - if schemas.IsAnthropicModel(request.Model) { + if schemas.IsAnthropicModelFamily(ctx, request.Model) { // Anthropic-on-Vertex doesn't accept URL-source document blocks. // Inline any URL documents to base64 before the converter runs. if err := inlineDocumentURLs(ctx, request); err != nil { @@ -467,7 +473,7 @@ func (provider *VertexProvider) ChatCompletion(ctx *schemas.BifrostContext, key if err != nil { return nil, fmt.Errorf("failed to delete model field: %w", err) } - } else if schemas.IsGeminiModel(request.Model) || schemas.IsAllDigitsASCII(request.Model) || schemas.IsGemmaModel(request.Model) { + } else if schemas.IsGeminiModelFamily(ctx, request.Model) || schemas.IsAllDigitsASCII(request.Model) || schemas.IsGemmaModelFamily(ctx, request.Model) { reqBody, err := gemini.ToGeminiChatCompletionRequest(request) if err != nil { return nil, err @@ -508,25 +514,25 @@ func (provider *VertexProvider) ChatCompletion(ctx *schemas.BifrostContext, key if bifrostErr != nil { return nil, bifrostErr } - if schemas.IsGeminiModel(request.Model) || schemas.IsAllDigitsASCII(request.Model) || schemas.IsGemmaModel(request.Model) { + if schemas.IsGeminiModelFamily(ctx, request.Model) || schemas.IsAllDigitsASCII(request.Model) || schemas.IsGemmaModelFamily(ctx, request.Model) { if rawBody, ok := ctx.Value(schemas.BifrostContextKeyUseRawRequestBody).(bool); ok && rawBody { jsonBody = gemini.NormalizeRawGenerateContentRequestForCompatibility(jsonBody) } jsonBody = stripVertexGeminiUnsupportedFieldsRaw(jsonBody) } - projectID := key.VertexKeyConfig.ProjectID.GetValue() + projectID := resolveVertexProjectID(ctx, key) if projectID == "" { return nil, providerUtils.NewConfigurationError("project ID is not set") } - region := key.VertexKeyConfig.Region.GetValue() + region := resolveVertexRegion(ctx, key) if region == "" { return nil, providerUtils.NewConfigurationError("region is not set in key config") } // Remap unsupported tool versions for Vertex (handles raw passthrough bodies) - if schemas.IsAnthropicModel(request.Model) && jsonBody != nil { + if schemas.IsAnthropicModelFamily(ctx, request.Model) && jsonBody != nil { remappedBody, remapErr := anthropic.RemapRawToolVersionsForProvider(jsonBody, schemas.Vertex, request.Model) if remapErr != nil { return nil, providerUtils.NewBifrostOperationError(remapErr.Error(), nil) @@ -547,7 +553,7 @@ func (provider *VertexProvider) ChatCompletion(ctx *schemas.BifrostContext, key var completeURL string if schemas.IsAllDigitsASCII(request.Model) { // Custom Fine-tuned models use OpenAPI endpoint - projectNumber := key.VertexKeyConfig.ProjectNumber.GetValue() + projectNumber := resolveVertexProjectNumber(ctx, key) if projectNumber == "" { return nil, providerUtils.NewConfigurationError("project number is not set for fine-tuned models") } @@ -555,13 +561,13 @@ func (provider *VertexProvider) ChatCompletion(ctx *schemas.BifrostContext, key authQuery = fmt.Sprintf("key=%s", url.QueryEscape(key.Value.GetValue())) } completeURL = getVertexEndpointURL(region, "v1beta1", projectNumber, request.Model, ":generateContent") - } else if schemas.IsAnthropicModel(request.Model) { + } else if schemas.IsAnthropicModelFamily(ctx, request.Model) { // Claude models use Anthropic publisher — model-aware host for multi-region support completeURL = getVertexModelAwarePublisherModelURL(region, "v1", projectID, "anthropic", request.Model, ":rawPredict") - } else if schemas.IsMistralModel(request.Model) { + } else if schemas.IsMistralModelFamily(ctx, request.Model) { // Mistral models use mistralai publisher with rawPredict completeURL = getVertexPublisherModelURL(region, "v1", projectID, "mistralai", request.Model, ":rawPredict") - } else if schemas.IsGeminiModel(request.Model) || schemas.IsGemmaModel(request.Model) { + } else if schemas.IsGeminiModelFamily(ctx, request.Model) || schemas.IsGemmaModelFamily(ctx, request.Model) { // Gemini models support api key if key.Value.GetValue() != "" { authQuery = fmt.Sprintf("key=%s", url.QueryEscape(key.Value.GetValue())) @@ -584,7 +590,7 @@ func (provider *VertexProvider) ChatCompletion(ctx *schemas.BifrostContext, key req.Header.SetMethod(http.MethodPost) req.Header.SetContentType("application/json") - if (schemas.IsGeminiModel(request.Model) || schemas.IsAllDigitsASCII(request.Model)) && + if (schemas.IsGeminiModelFamily(ctx, request.Model) || schemas.IsAllDigitsASCII(request.Model)) && request.Params != nil && request.Params.ServiceTier != nil { if v := vertexServiceTierHeaderValue(region, request.Model, *request.Params.ServiceTier); v != "" { req.Header.Set(VertexServiceTierHeader, v) @@ -653,7 +659,7 @@ func (provider *VertexProvider) ChatCompletion(ctx *schemas.BifrostContext, key }, nil } - if schemas.IsAnthropicModel(request.Model) { + if schemas.IsAnthropicModelFamily(ctx, request.Model) { // Create response object from pool anthropicResponse := anthropic.AcquireAnthropicMessageResponse() defer anthropic.ReleaseAnthropicMessageResponse(anthropicResponse) @@ -682,7 +688,7 @@ func (provider *VertexProvider) ChatCompletion(ctx *schemas.BifrostContext, key } return response, nil - } else if schemas.IsGeminiModel(request.Model) || schemas.IsAllDigitsASCII(request.Model) || schemas.IsGemmaModel(request.Model) { + } else if schemas.IsGeminiModelFamily(ctx, request.Model) || schemas.IsAllDigitsASCII(request.Model) || schemas.IsGemmaModelFamily(ctx, request.Model) { geminiResponse := gemini.GenerateContentResponse{} rawRequest, rawResponse, bifrostErr := providerUtils.HandleProviderResponse(responseBody, &geminiResponse, jsonBody, providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest), providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse)) @@ -734,17 +740,17 @@ func (provider *VertexProvider) ChatCompletion(ctx *schemas.BifrostContext, key // Returns a channel of BifrostStreamChunk objects for streaming results or an error if the request fails. func (provider *VertexProvider) ChatCompletionStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostChatRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { providerName := provider.GetProviderKey() - projectID := key.VertexKeyConfig.ProjectID.GetValue() + projectID := resolveVertexProjectID(ctx, key) if projectID == "" { return nil, providerUtils.NewConfigurationError("project ID is not set") } - region := key.VertexKeyConfig.Region.GetValue() + region := resolveVertexRegion(ctx, key) if region == "" { return nil, providerUtils.NewConfigurationError("region is not set in key config") } - if schemas.IsAnthropicModel(request.Model) { + if schemas.IsAnthropicModelFamily(ctx, request.Model) { // Use Anthropic-style streaming for Claude models jsonData, bifrostErr := providerUtils.CheckContextAndGetRequestBody( ctx, @@ -859,7 +865,7 @@ func (provider *VertexProvider) ChatCompletionStream(ctx *schemas.BifrostContext provider.logger, postHookSpanFinalizer, ) - } else if schemas.IsGeminiModel(request.Model) || schemas.IsAllDigitsASCII(request.Model) || schemas.IsGemmaModel(request.Model) { + } else if schemas.IsGeminiModelFamily(ctx, request.Model) || schemas.IsAllDigitsASCII(request.Model) || schemas.IsGemmaModelFamily(ctx, request.Model) { // Use Gemini-style streaming for Gemini models jsonData, bifrostErr := providerUtils.CheckContextAndGetRequestBody( ctx, @@ -888,7 +894,7 @@ func (provider *VertexProvider) ChatCompletionStream(ctx *schemas.BifrostContext } // For custom/fine-tuned models, validate projectNumber is set - projectNumber := key.VertexKeyConfig.ProjectNumber.GetValue() + projectNumber := resolveVertexProjectNumber(ctx, key) if schemas.IsAllDigitsASCII(request.Model) && projectNumber == "" { return nil, providerUtils.NewConfigurationError("project number is not set for fine-tuned models") } @@ -909,7 +915,7 @@ func (provider *VertexProvider) ChatCompletionStream(ctx *schemas.BifrostContext "Cache-Control": "no-cache", } - if schemas.IsGeminiModel(request.Model) || schemas.IsAllDigitsASCII(request.Model) { + if schemas.IsGeminiModelFamily(ctx, request.Model) || schemas.IsAllDigitsASCII(request.Model) { if _, overridden := provider.networkConfig.ExtraHeaders[VertexServiceTierHeader]; !overridden { if request.Params != nil && request.Params.ServiceTier != nil { if v := vertexServiceTierHeaderValue(region, request.Model, *request.Params.ServiceTier); v != "" { @@ -955,7 +961,7 @@ func (provider *VertexProvider) ChatCompletionStream(ctx *schemas.BifrostContext authQuery := "" // Determine the URL based on model type var completeURL string - if schemas.IsMistralModel(request.Model) { + if schemas.IsMistralModelFamily(ctx, request.Model) { // Mistral models use mistralai publisher with streamRawPredict completeURL = getVertexPublisherModelURL(region, "v1", projectID, "mistralai", request.Model, ":streamRawPredict") } else { @@ -1009,17 +1015,17 @@ func (provider *VertexProvider) ChatCompletionStream(ctx *schemas.BifrostContext // Responses performs a responses request to the Vertex API. func (provider *VertexProvider) Responses(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostResponsesRequest) (*schemas.BifrostResponsesResponse, *schemas.BifrostError) { - if schemas.IsAnthropicModel(request.Model) { + if schemas.IsAnthropicModelFamily(ctx, request.Model) { jsonBody, bifrostErr := getRequestBodyForAnthropicResponses(ctx, request, request.Model, false, false, provider.networkConfig.BetaHeaderOverrides, provider.networkConfig.ExtraHeaders, provider.sendBackRawRequest, provider.sendBackRawResponse) if bifrostErr != nil { return nil, bifrostErr } - projectID := key.VertexKeyConfig.ProjectID.GetValue() + projectID := resolveVertexProjectID(ctx, key) if projectID == "" { return nil, providerUtils.NewConfigurationError("project ID is not set") } - region := key.VertexKeyConfig.Region.GetValue() + region := resolveVertexRegion(ctx, key) if region == "" { return nil, providerUtils.NewConfigurationError("region is not set in key config") } @@ -1128,7 +1134,7 @@ func (provider *VertexProvider) Responses(ctx *schemas.BifrostContext, key schem } return response, nil - } else if schemas.IsGeminiModel(request.Model) || schemas.IsAllDigitsASCII(request.Model) || schemas.IsGemmaModel(request.Model) { + } else if schemas.IsGeminiModelFamily(ctx, request.Model) || schemas.IsAllDigitsASCII(request.Model) || schemas.IsGemmaModelFamily(ctx, request.Model) { jsonBody, bifrostErr := providerUtils.CheckContextAndGetRequestBody( ctx, request, @@ -1153,12 +1159,12 @@ func (provider *VertexProvider) Responses(ctx *schemas.BifrostContext, key schem } jsonBody = stripVertexGeminiUnsupportedFieldsRaw(jsonBody) - projectID := key.VertexKeyConfig.ProjectID.GetValue() + projectID := resolveVertexProjectID(ctx, key) if projectID == "" { return nil, providerUtils.NewConfigurationError("project ID is not set") } - region := key.VertexKeyConfig.Region.GetValue() + region := resolveVertexRegion(ctx, key) if region == "" { return nil, providerUtils.NewConfigurationError("region is not set in key config") } @@ -1169,7 +1175,7 @@ func (provider *VertexProvider) Responses(ctx *schemas.BifrostContext, key schem } // For custom/fine-tuned models, validate projectNumber is set - projectNumber := key.VertexKeyConfig.ProjectNumber.GetValue() + projectNumber := resolveVertexProjectNumber(ctx, key) if schemas.IsAllDigitsASCII(request.Model) && projectNumber == "" { return nil, providerUtils.NewConfigurationError("project number is not set for fine-tuned models") } @@ -1189,7 +1195,7 @@ func (provider *VertexProvider) Responses(ctx *schemas.BifrostContext, key schem req.Header.SetMethod(http.MethodPost) req.Header.SetContentType("application/json") - if (schemas.IsGeminiModel(request.Model) || schemas.IsAllDigitsASCII(request.Model)) && + if (schemas.IsGeminiModelFamily(ctx, request.Model) || schemas.IsAllDigitsASCII(request.Model)) && request.Params != nil && request.Params.ServiceTier != nil { if v := vertexServiceTierHeaderValue(region, request.Model, *request.Params.ServiceTier); v != "" { req.Header.Set(VertexServiceTierHeader, v) @@ -1290,13 +1296,13 @@ func (provider *VertexProvider) Responses(ctx *schemas.BifrostContext, key schem // ResponsesStream performs a streaming responses request to the Vertex API. func (provider *VertexProvider) ResponsesStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostResponsesRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { - if schemas.IsAnthropicModel(request.Model) { - region := key.VertexKeyConfig.Region.GetValue() + if schemas.IsAnthropicModelFamily(ctx, request.Model) { + region := resolveVertexRegion(ctx, key) if region == "" { return nil, providerUtils.NewConfigurationError("region is not set in key config") } - projectID := key.VertexKeyConfig.ProjectID.GetValue() + projectID := resolveVertexProjectID(ctx, key) if projectID == "" { return nil, providerUtils.NewConfigurationError("project ID is not set") } @@ -1344,13 +1350,13 @@ func (provider *VertexProvider) ResponsesStream(ctx *schemas.BifrostContext, pos provider.logger, postHookSpanFinalizer, ) - } else if schemas.IsGeminiModel(request.Model) || schemas.IsAllDigitsASCII(request.Model) || schemas.IsGemmaModel(request.Model) { - region := key.VertexKeyConfig.Region.GetValue() + } else if schemas.IsGeminiModelFamily(ctx, request.Model) || schemas.IsAllDigitsASCII(request.Model) || schemas.IsGemmaModelFamily(ctx, request.Model) { + region := resolveVertexRegion(ctx, key) if region == "" { return nil, providerUtils.NewConfigurationError("region is not set in key config") } - projectID := key.VertexKeyConfig.ProjectID.GetValue() + projectID := resolveVertexProjectID(ctx, key) if projectID == "" { return nil, providerUtils.NewConfigurationError("project ID is not set") } @@ -1387,7 +1393,7 @@ func (provider *VertexProvider) ResponsesStream(ctx *schemas.BifrostContext, pos } // For custom/fine-tuned models, validate projectNumber is set - projectNumber := key.VertexKeyConfig.ProjectNumber.GetValue() + projectNumber := resolveVertexProjectNumber(ctx, key) if schemas.IsAllDigitsASCII(request.Model) && projectNumber == "" { return nil, providerUtils.NewConfigurationError("project number is not set for fine-tuned models") } @@ -1407,7 +1413,7 @@ func (provider *VertexProvider) ResponsesStream(ctx *schemas.BifrostContext, pos "Cache-Control": "no-cache", } - if schemas.IsGeminiModel(request.Model) || schemas.IsAllDigitsASCII(request.Model) { + if schemas.IsGeminiModelFamily(ctx, request.Model) || schemas.IsAllDigitsASCII(request.Model) { if _, overridden := provider.networkConfig.ExtraHeaders[VertexServiceTierHeader]; !overridden { if request.Params != nil && request.Params.ServiceTier != nil { if v := vertexServiceTierHeaderValue(region, request.Model, *request.Params.ServiceTier); v != "" { @@ -1463,12 +1469,12 @@ func (provider *VertexProvider) ResponsesStream(ctx *schemas.BifrostContext, pos // All Vertex AI embedding models use the same response format regardless of the model type. // Returns a BifrostResponse containing the embedding(s) and any error that occurred. func (provider *VertexProvider) Embedding(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostEmbeddingRequest) (*schemas.BifrostEmbeddingResponse, *schemas.BifrostError) { - projectID := key.VertexKeyConfig.ProjectID.GetValue() + projectID := resolveVertexProjectID(ctx, key) if projectID == "" { return nil, providerUtils.NewConfigurationError("project ID is not set") } - region := key.VertexKeyConfig.Region.GetValue() + region := resolveVertexRegion(ctx, key) if region == "" { return nil, providerUtils.NewConfigurationError("region is not set in key config") } @@ -1485,13 +1491,17 @@ func (provider *VertexProvider) Embedding(ctx *schemas.BifrostContext, key schem } // For custom/fine-tuned models, validate projectNumber is set - projectNumber := key.VertexKeyConfig.ProjectNumber.GetValue() + projectNumber := resolveVertexProjectNumber(ctx, key) if schemas.IsAllDigitsASCII(request.Model) && projectNumber == "" { return nil, providerUtils.NewConfigurationError("project number is not set for fine-tuned models") } // Build the native Vertex embedding API endpoint - url := getCompleteURLForGeminiEndpoint(request.Model, region, projectID, projectNumber, ":predict") + authQuery := "" + if key.Value.GetValue() != "" { + authQuery = fmt.Sprintf("key=%s", url.QueryEscape(key.Value.GetValue())) + } + completeURL := getCompleteURLForGeminiEndpoint(request.Model, region, projectID, projectNumber, ":predict") // Create HTTP request for streaming req := fasthttp.AcquireRequest() @@ -1505,22 +1515,28 @@ func (provider *VertexProvider) Embedding(ctx *schemas.BifrostContext, key schem }() req.Header.SetMethod(http.MethodPost) - req.SetRequestURI(url) req.Header.SetContentType("application/json") // Set any extra headers from network config providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) - // Getting oauth2 token - tokenSource, err := getAuthTokenSource(key) - if err != nil { - return nil, providerUtils.NewBifrostOperationError("error creating auth token source", err) - } - token, err := tokenSource.Token() - if err != nil { - return nil, providerUtils.NewBifrostOperationError("error getting token", err) + // If auth query is set, add it to the URL + // Otherwise, get the oauth2 token and set the Authorization header + if authQuery != "" { + completeURL = fmt.Sprintf("%s?%s", completeURL, authQuery) + } else { + tokenSource, err := getAuthTokenSource(key) + if err != nil { + return nil, providerUtils.NewBifrostOperationError("error creating auth token source", err) + } + token, err := tokenSource.Token() + if err != nil { + return nil, providerUtils.NewBifrostOperationError("error getting token", err) + } + req.Header.Set("Authorization", "Bearer "+token.AccessToken) } - req.Header.Set("Authorization", "Bearer "+token.AccessToken) + + req.SetRequestURI(completeURL) usedLargePayloadBody := providerUtils.ApplyLargePayloadRequestBody(ctx, req) if !usedLargePayloadBody { @@ -1617,7 +1633,7 @@ func (provider *VertexProvider) Speech(ctx *schemas.BifrostContext, key schemas. // Rerank performs a rerank request using Vertex Discovery Engine ranking API. func (provider *VertexProvider) Rerank(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostRerankRequest) (*schemas.BifrostRerankResponse, *schemas.BifrostError) { - projectID := strings.TrimSpace(key.VertexKeyConfig.ProjectID.GetValue()) + projectID := strings.TrimSpace(resolveVertexProjectID(ctx, key)) if projectID == "" { return nil, providerUtils.NewConfigurationError("project ID is not set") } @@ -1771,7 +1787,7 @@ func (provider *VertexProvider) TranscriptionStream(ctx *schemas.BifrostContext, func (provider *VertexProvider) ImageGeneration(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostImageGenerationRequest) (*schemas.BifrostImageGenerationResponse, *schemas.BifrostError) { // Validate model type before processing - if !schemas.IsGeminiModel(request.Model) && !schemas.IsAllDigitsASCII(request.Model) && !schemas.IsImagenModel(request.Model) { + if !schemas.IsGeminiModelFamily(ctx, request.Model) && !schemas.IsAllDigitsASCII(request.Model) && !schemas.IsImagenModelFamily(ctx, request.Model) { return nil, providerUtils.NewConfigurationError(fmt.Sprintf("image generation is only supported for Gemini and Imagen models, got: %s", request.Model)) } @@ -1783,7 +1799,7 @@ func (provider *VertexProvider) ImageGeneration(ctx *schemas.BifrostContext, key var extraParams map[string]interface{} var err error - if schemas.IsGeminiModel(request.Model) || schemas.IsAllDigitsASCII(request.Model) { + if schemas.IsGeminiModelFamily(ctx, request.Model) || schemas.IsAllDigitsASCII(request.Model) { reqBody := gemini.ToGeminiImageGenerationRequest(request) if reqBody == nil { return nil, fmt.Errorf("image generation input is not provided") @@ -1796,7 +1812,7 @@ func (provider *VertexProvider) ImageGeneration(ctx *schemas.BifrostContext, key if err != nil { return nil, fmt.Errorf("failed to marshal request body: %w", err) } - } else if schemas.IsImagenModel(request.Model) { + } else if schemas.IsImagenModelFamily(ctx, request.Model) { reqBody := gemini.ToImagenImageGenerationRequest(request) if reqBody == nil { return nil, fmt.Errorf("image generation input is not provided") @@ -1821,12 +1837,12 @@ func (provider *VertexProvider) ImageGeneration(ctx *schemas.BifrostContext, key return nil, bifrostErr } - projectID := key.VertexKeyConfig.ProjectID.GetValue() + projectID := resolveVertexProjectID(ctx, key) if projectID == "" { return nil, providerUtils.NewConfigurationError("project ID is not set") } - region := key.VertexKeyConfig.Region.GetValue() + region := resolveVertexRegion(ctx, key) if region == "" { return nil, providerUtils.NewConfigurationError("region is not set in key config") } @@ -1837,7 +1853,7 @@ func (provider *VertexProvider) ImageGeneration(ctx *schemas.BifrostContext, key var completeURL string if schemas.IsAllDigitsASCII(request.Model) { // Custom Fine-tuned models use OpenAPI endpoint - projectNumber := key.VertexKeyConfig.ProjectNumber.GetValue() + projectNumber := resolveVertexProjectNumber(ctx, key) if projectNumber == "" { return nil, providerUtils.NewConfigurationError("project number is not set for fine-tuned models") } @@ -1846,13 +1862,13 @@ func (provider *VertexProvider) ImageGeneration(ctx *schemas.BifrostContext, key } completeURL = getVertexEndpointURL(region, "v1beta1", projectNumber, request.Model, ":generateContent") - } else if schemas.IsImagenModel(request.Model) { + } else if schemas.IsImagenModelFamily(ctx, request.Model) { // Imagen models are published models, use publishers/google/models path if value := key.Value.GetValue(); value != "" { authQuery = fmt.Sprintf("key=%s", url.QueryEscape(value)) } completeURL = getVertexPublisherModelURL(region, "v1", projectID, "google", gemini.NormalizeModelName(request.Model), ":predict") - } else if schemas.IsGeminiModel(request.Model) { + } else if schemas.IsGeminiModelFamily(ctx, request.Model) { if value := key.Value.GetValue(); value != "" { authQuery = fmt.Sprintf("key=%s", url.QueryEscape(value)) } @@ -1932,7 +1948,7 @@ func (provider *VertexProvider) ImageGeneration(ctx *schemas.BifrostContext, key }, nil } - if schemas.IsGeminiModel(request.Model) || schemas.IsAllDigitsASCII(request.Model) { + if schemas.IsGeminiModelFamily(ctx, request.Model) || schemas.IsAllDigitsASCII(request.Model) { geminiResponse := gemini.GenerateContentResponse{} rawRequest, rawResponse, bifrostErr := providerUtils.HandleProviderResponse(responseBody, &geminiResponse, jsonBody, providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest), providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse)) @@ -1991,7 +2007,7 @@ func (provider *VertexProvider) ImageGenerationStream(ctx *schemas.BifrostContex // Returns a BifrostResponse containing the images and any error that occurred. func (provider *VertexProvider) ImageEdit(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostImageEditRequest) (*schemas.BifrostImageGenerationResponse, *schemas.BifrostError) { // Validate model type before processing - if !schemas.IsGeminiModel(request.Model) && !schemas.IsAllDigitsASCII(request.Model) && !schemas.IsImagenModel(request.Model) { + if !schemas.IsGeminiModelFamily(ctx, request.Model) && !schemas.IsAllDigitsASCII(request.Model) && !schemas.IsImagenModelFamily(ctx, request.Model) { return nil, providerUtils.NewConfigurationError(fmt.Sprintf("image edit is only supported for Gemini and Imagen models, got: %s", request.Model)) } @@ -2003,7 +2019,7 @@ func (provider *VertexProvider) ImageEdit(ctx *schemas.BifrostContext, key schem var extraParams map[string]interface{} var err error - if schemas.IsGeminiModel(request.Model) || schemas.IsAllDigitsASCII(request.Model) { + if schemas.IsGeminiModelFamily(ctx, request.Model) || schemas.IsAllDigitsASCII(request.Model) { reqBody := gemini.ToGeminiImageEditRequest(request) if reqBody == nil { return nil, fmt.Errorf("image edit input is not provided") @@ -2016,7 +2032,7 @@ func (provider *VertexProvider) ImageEdit(ctx *schemas.BifrostContext, key schem if err != nil { return nil, fmt.Errorf("failed to marshal request body: %w", err) } - } else if schemas.IsImagenModel(request.Model) { + } else if schemas.IsImagenModelFamily(ctx, request.Model) { reqBody := gemini.ToImagenImageEditRequest(request) if reqBody == nil { return nil, fmt.Errorf("image edit input is not provided") @@ -2041,12 +2057,12 @@ func (provider *VertexProvider) ImageEdit(ctx *schemas.BifrostContext, key schem return nil, bifrostErr } - projectID := key.VertexKeyConfig.ProjectID.GetValue() + projectID := resolveVertexProjectID(ctx, key) if projectID == "" { return nil, providerUtils.NewConfigurationError("project ID is not set") } - region := key.VertexKeyConfig.Region.GetValue() + region := resolveVertexRegion(ctx, key) if region == "" { return nil, providerUtils.NewConfigurationError("region is not set in key config") } @@ -2058,14 +2074,14 @@ func (provider *VertexProvider) ImageEdit(ctx *schemas.BifrostContext, key schem var completeURL string if schemas.IsAllDigitsASCII(request.Model) { - projectNumber := key.VertexKeyConfig.ProjectNumber.GetValue() + projectNumber := resolveVertexProjectNumber(ctx, key) if projectNumber == "" { return nil, providerUtils.NewConfigurationError("project number is not set for fine-tuned models") } completeURL = getVertexEndpointURL(region, "v1beta1", projectNumber, gemini.NormalizeModelName(request.Model), ":generateContent") - } else if schemas.IsImagenModel(request.Model) { + } else if schemas.IsImagenModelFamily(ctx, request.Model) { completeURL = getVertexPublisherModelURL(region, "v1", projectID, "google", gemini.NormalizeModelName(request.Model), ":predict") - } else if schemas.IsGeminiModel(request.Model) { + } else if schemas.IsGeminiModelFamily(ctx, request.Model) { completeURL = getVertexPublisherModelURL(region, "v1", projectID, "google", gemini.NormalizeModelName(request.Model), ":generateContent") } @@ -2140,7 +2156,7 @@ func (provider *VertexProvider) ImageEdit(ctx *schemas.BifrostContext, key schem }, nil } - if schemas.IsGeminiModel(request.Model) || schemas.IsAllDigitsASCII(request.Model) { + if schemas.IsGeminiModelFamily(ctx, request.Model) || schemas.IsAllDigitsASCII(request.Model) { geminiResponse := gemini.GenerateContentResponse{} rawRequest, rawResponse, bifrostErr := providerUtils.HandleProviderResponse(responseBody, &geminiResponse, jsonBody, providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest), providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse)) @@ -2206,7 +2222,7 @@ func (provider *VertexProvider) VideoGeneration(ctx *schemas.BifrostContext, key providerName := provider.GetProviderKey() // Only Gemini models support video generation in Vertex - if !schemas.IsVeoModel(bifrostReq.Model) && !schemas.IsAllDigitsASCII(bifrostReq.Model) { + if !schemas.IsVeoModelFamily(ctx, bifrostReq.Model) && !schemas.IsAllDigitsASCII(bifrostReq.Model) { return nil, providerUtils.NewConfigurationError(fmt.Sprintf("video generation is only supported for Veo models in Vertex, got: %s", bifrostReq.Model)) } @@ -2222,12 +2238,12 @@ func (provider *VertexProvider) VideoGeneration(ctx *schemas.BifrostContext, key return nil, bifrostErr } - projectID := key.VertexKeyConfig.ProjectID.GetValue() + projectID := resolveVertexProjectID(ctx, key) if projectID == "" { return nil, providerUtils.NewConfigurationError("project ID is not set") } - region := key.VertexKeyConfig.Region.GetValue() + region := resolveVertexRegion(ctx, key) if region == "" { return nil, providerUtils.NewConfigurationError("region is not set in key config") } @@ -2239,7 +2255,7 @@ func (provider *VertexProvider) VideoGeneration(ctx *schemas.BifrostContext, key } // For custom/fine-tuned models, validate projectNumber is set - projectNumber := key.VertexKeyConfig.ProjectNumber.GetValue() + projectNumber := resolveVertexProjectNumber(ctx, key) if schemas.IsAllDigitsASCII(bifrostReq.Model) && projectNumber == "" { return nil, providerUtils.NewConfigurationError("project number is not set for fine-tuned models") } @@ -2328,7 +2344,7 @@ func (provider *VertexProvider) VideoRetrieve(ctx *schemas.BifrostContext, key s sendBackRawResponse := providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse) sendBackRawRequest := providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest) - region := key.VertexKeyConfig.Region.GetValue() + region := resolveVertexRegion(ctx, key) if region == "" { return nil, providerUtils.NewConfigurationError("region is not set in key config") } @@ -2618,60 +2634,1336 @@ func stripVertexGeminiUnsupportedFieldsRaw(jsonBody []byte) []byte { return out } -// BatchCreate is not supported by Vertex AI provider. +// BatchCreate creates a Vertex AI batch prediction job. +// +// Input modes (mutually exclusive, mirroring the Gemini provider): +// - InputFileID: a gs:// URI of an existing Vertex-format JSONL file. +// - Requests: inline items converted to JSONL and uploaded to GCS via FileUpload. +// +// The output destination is taken from the typed output_folder.url (a gs:// prefix). func (provider *VertexProvider) BatchCreate(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostBatchCreateRequest) (*schemas.BifrostBatchCreateResponse, *schemas.BifrostError) { - return nil, providerUtils.NewUnsupportedOperationError(schemas.BatchCreateRequest, provider.GetProviderKey()) + baseURL, cfgErr := vertexBatchJobsBaseURL(key) + if cfgErr != nil { + return nil, cfgErr + } + + rawBody, hasRawBody := providerUtils.CheckAndGetRawRequestBody(ctx, request) + hasRawBody = hasRawBody && len(rawBody) > 0 + + inputFileID := request.InputFileID + jobName := "" + outputURI := "" + if !hasRawBody { + if request.Model == nil || *request.Model == "" { + return nil, providerUtils.NewBifrostOperationError("model is required for Vertex batch API", nil) + } + hasFileInput := request.InputFileID != "" + hasInlineRequests := len(request.Requests) > 0 + if hasFileInput && hasInlineRequests { + return nil, providerUtils.NewBifrostOperationError("cannot specify both input_file_id and requests", nil) + } + if !hasFileInput && !hasInlineRequests { + return nil, providerUtils.NewBifrostOperationError("either input_file_id (gs:// JSONL URI) or requests is required for Vertex batch API", nil) + } + + // Output destination is the typed output_folder.url (a gs:// prefix). Vertex writes + // results into its own subdirectory under this prefix. + if request.OutputFolder != nil { + outputURI = strings.TrimSpace(request.OutputFolder.URL) + } + if outputURI == "" { + return nil, providerUtils.NewBifrostOperationError("output_folder.url (gs:// prefix) is required for Vertex batch API", nil) + } + + jobName = fmt.Sprintf("bifrost-batch-%d", time.Now().Unix()) + if request.DisplayName != nil && *request.DisplayName != "" { + jobName = *request.DisplayName + } else if request.Metadata != nil { + // Back-compat: OpenAI-compatible clients may pass the job name via metadata. + if name, ok := request.Metadata["job_name"]; ok && name != "" { + jobName = name + } + } + + // Inline mode: convert to JSONL and upload next to the output location (Bedrock pattern). + if inputFileID == "" { + jsonlData, err := vertexConvertRequestsToJSONL(request.Requests) + if err != nil { + return nil, providerUtils.NewBifrostOperationError("failed to convert requests to Vertex JSONL", err) + } + outBucket, outKey, parseErr := parseGCSURI(outputURI) + if parseErr != nil { + return nil, providerUtils.NewBifrostOperationError(parseErr.Error(), nil) + } + // Place the input alongside the output directory (sibling, not child) so the + // generated JSONL does not live inside the directory Vertex writes results to. + inputPrefix := "vertex-batches-input" + if trimmed := strings.Trim(outKey, "/"); trimmed != "" { + if idx := strings.LastIndexByte(trimmed, '/'); idx >= 0 { + inputPrefix = trimmed[:idx] + "/input" + } else { + inputPrefix = "input" + } + } + uploadResp, uploadErr := provider.FileUpload(ctx, key, &schemas.BifrostFileUploadRequest{ + Provider: schemas.Vertex, + File: jsonlData, + Filename: jobName + "-input.jsonl", + Purpose: schemas.FilePurposeBatch, + ContentType: schemas.Ptr("application/jsonl"), + StorageConfig: &schemas.FileStorageConfig{ + GCS: &schemas.GCSStorageConfig{Bucket: outBucket, Prefix: inputPrefix}, + }, + }) + if uploadErr != nil { + return nil, uploadErr + } + inputFileID = uploadResp.ID + } + } + + jsonData, bodyErr := providerUtils.CheckContextAndGetRequestBody( + ctx, + request, + func() (providerUtils.RequestBodyWithExtraParams, error) { + return ToVertexBatchCreateRequest(request, jobName, inputFileID, outputURI), nil + }, + ) + if bodyErr != nil { + return nil, bodyErr + } + + authHeader, authErr := gcsGetAuthHeader(key) + if authErr != nil { + return nil, providerUtils.NewBifrostOperationError(authErr.Error(), nil) + } + + req := fasthttp.AcquireRequest() + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseRequest(req) + defer fasthttp.ReleaseResponse(resp) + + req.SetRequestURI(baseURL + "/batchPredictionJobs") + req.Header.SetMethod(http.MethodPost) + req.Header.SetContentType("application/json") + providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) + req.Header.Set("Authorization", authHeader) + req.SetBody(jsonData) + + sendBackRawRequest := providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest) + sendBackRawResponse := providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse) + + startTime := time.Now() + _, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, provider.client, req, resp) + defer wait() + if bifrostErr != nil { + return nil, providerUtils.EnrichError(ctx, bifrostErr, jsonData, nil, provider.sendBackRawRequest, provider.sendBackRawResponse) + } + + if resp.StatusCode() != fasthttp.StatusOK { + if resp.StatusCode() == fasthttp.StatusUnauthorized || resp.StatusCode() == fasthttp.StatusForbidden { + removeVertexClient(key.VertexKeyConfig.AuthCredentials.GetValue()) + } + return nil, providerUtils.EnrichError(ctx, parseVertexJobAPIError(resp.Body(), resp.StatusCode(), "batch create"), jsonData, resp.Body(), provider.sendBackRawRequest, provider.sendBackRawResponse) + } + + var created VertexBatchPredictionJob + rawRequest, rawResponse, parseErr := providerUtils.HandleProviderResponse(resp.Body(), &created, jsonData, sendBackRawRequest, sendBackRawResponse) + if parseErr != nil { + return nil, providerUtils.EnrichError(ctx, parseErr, jsonData, resp.Body(), provider.sendBackRawRequest, provider.sendBackRawResponse) + } + + // In raw-passthrough mode inputFileID may be empty (e.g. BigQuery or multi-URI inputs the + // typed path never sees); fall back to the GCS source the created job echoes back. + if inputFileID == "" && created.InputConfig.GcsSource != nil && len(created.InputConfig.GcsSource.Uris) > 0 { + inputFileID = created.InputConfig.GcsSource.Uris[0] + } + + result := &schemas.BifrostBatchCreateResponse{ + ID: created.Name, + Object: "batch", + InputFileID: inputFileID, + Status: vertexJobStateToBatchStatus(created.State), + CreatedAt: gcsParseTime(created.CreateTime), + Metadata: request.Metadata, + ExtraFields: schemas.BifrostResponseExtraFields{ + Latency: time.Since(startTime).Milliseconds(), + }, + } + if created.DisplayName != "" { + result.DisplayName = schemas.Ptr(created.DisplayName) + } + if sendBackRawRequest { + result.ExtraFields.RawRequest = rawRequest + } + if sendBackRawResponse { + result.ExtraFields.RawResponse = rawResponse + } + return result, nil } -// BatchList is not supported by Vertex AI provider. -func (provider *VertexProvider) BatchList(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostBatchListRequest) (*schemas.BifrostBatchListResponse, *schemas.BifrostError) { - return nil, providerUtils.NewUnsupportedOperationError(schemas.BatchListRequest, provider.GetProviderKey()) +// BatchList lists Vertex AI batch prediction jobs across all keys, paginating one key +// at a time. Each Vertex key carries its own project/region, and batch jobs are scoped to +// that project/region, so the serial helper walks every key (exhausting all of its pages +// before advancing) to avoid hiding jobs created under any key but the first. +func (provider *VertexProvider) BatchList(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostBatchListRequest) (*schemas.BifrostBatchListResponse, *schemas.BifrostError) { + if len(keys) == 0 { + return nil, providerUtils.NewBifrostOperationError("no keys provided for Vertex BatchList", nil) + } + + // The OpenAI-compatible /v1/batches route feeds the cursor back via After. + helper, err := providerUtils.NewSerialListHelper(keys, request.After, provider.logger, true) + if err != nil { + return nil, providerUtils.NewBifrostOperationError("invalid pagination cursor", err) + } + + key, nativeCursor, ok := helper.GetCurrentKey() + if !ok { + // All keys exhausted. + return &schemas.BifrostBatchListResponse{ + Object: "list", + Data: []schemas.BifrostBatchRetrieveResponse{}, + }, nil + } + + // Query the current key with its native Vertex page token. + modifiedRequest := *request + if nativeCursor != "" { + modifiedRequest.PageToken = &nativeCursor + } else { + modifiedRequest.PageToken = nil + } + + resp, latency, bifrostErr := provider.batchListByKey(ctx, key, &modifiedRequest) + if bifrostErr != nil { + return nil, bifrostErr + } + + nativeNextCursor := "" + if resp.NextCursor != nil { + nativeNextCursor = *resp.NextCursor + } + nextCursor, hasMore := helper.BuildNextCursor(resp.HasMore, nativeNextCursor) + + resp.HasMore = hasMore + if nextCursor != "" { + resp.NextCursor = &nextCursor + } else { + resp.NextCursor = nil + } + resp.ExtraFields.Latency = latency.Milliseconds() + return resp, nil } -// BatchRetrieve is not supported by Vertex AI provider. -func (provider *VertexProvider) BatchRetrieve(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostBatchRetrieveRequest) (*schemas.BifrostBatchRetrieveResponse, *schemas.BifrostError) { - return nil, providerUtils.NewUnsupportedOperationError(schemas.BatchRetrieveRequest, provider.GetProviderKey()) +// batchListByKey lists batch prediction jobs for a single Vertex key/project/region. +// The native Vertex page token (if any) is taken from request.PageToken; the returned +// NextCursor carries Vertex's nextPageToken verbatim for the caller to re-encode. +func (provider *VertexProvider) batchListByKey(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostBatchListRequest) (*schemas.BifrostBatchListResponse, time.Duration, *schemas.BifrostError) { + baseURL, cfgErr := vertexBatchJobsBaseURL(key) + if cfgErr != nil { + return nil, 0, cfgErr + } + + params := url.Values{} + pageSize := request.PageSize + if pageSize <= 0 { + pageSize = request.Limit + } + if pageSize <= 0 { + pageSize = 20 + } + params.Set("pageSize", fmt.Sprintf("%d", pageSize)) + if request.PageToken != nil && *request.PageToken != "" { + params.Set("pageToken", *request.PageToken) + } + + authHeader, authErr := gcsGetAuthHeader(key) + if authErr != nil { + return nil, 0, providerUtils.NewBifrostOperationError(authErr.Error(), nil) + } + + req := fasthttp.AcquireRequest() + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseRequest(req) + defer fasthttp.ReleaseResponse(resp) + + req.SetRequestURI(baseURL + "/batchPredictionJobs?" + params.Encode()) + req.Header.SetMethod(http.MethodGet) + providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) + req.Header.Set("Authorization", authHeader) + + sendBackRawResponse := providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse) + + startTime := time.Now() + _, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, provider.client, req, resp) + defer wait() + if bifrostErr != nil { + return nil, 0, providerUtils.EnrichError(ctx, bifrostErr, nil, nil, provider.sendBackRawRequest, provider.sendBackRawResponse) + } + + if resp.StatusCode() != fasthttp.StatusOK { + if resp.StatusCode() == fasthttp.StatusUnauthorized || resp.StatusCode() == fasthttp.StatusForbidden { + removeVertexClient(key.VertexKeyConfig.AuthCredentials.GetValue()) + } + return nil, 0, providerUtils.EnrichError(ctx, parseVertexJobAPIError(resp.Body(), resp.StatusCode(), "batch list"), nil, resp.Body(), provider.sendBackRawRequest, provider.sendBackRawResponse) + } + + // GET request: no request body, so raw request capture is skipped by HandleProviderResponse. + var listResp VertexBatchJobListResponse + _, rawResponse, parseErr := providerUtils.HandleProviderResponse(resp.Body(), &listResp, nil, false, sendBackRawResponse) + if parseErr != nil { + return nil, 0, providerUtils.EnrichError(ctx, parseErr, nil, resp.Body(), provider.sendBackRawRequest, provider.sendBackRawResponse) + } + + data := make([]schemas.BifrostBatchRetrieveResponse, 0, len(listResp.BatchPredictionJobs)) + for i := range listResp.BatchPredictionJobs { + data = append(data, vertexBatchJobToBifrost(&listResp.BatchPredictionJobs[i])) + } + + var nextCursor *string + if listResp.NextPageToken != "" { + nextCursor = &listResp.NextPageToken + } + + result := &schemas.BifrostBatchListResponse{ + Object: "list", + Data: data, + HasMore: listResp.NextPageToken != "", + NextCursor: nextCursor, + } + if sendBackRawResponse { + result.ExtraFields.RawResponse = rawResponse + } + return result, time.Since(startTime), nil } -// BatchCancel is not supported by Vertex AI provider. -func (provider *VertexProvider) BatchCancel(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostBatchCancelRequest) (*schemas.BifrostBatchCancelResponse, *schemas.BifrostError) { - return nil, providerUtils.NewUnsupportedOperationError(schemas.BatchCancelRequest, provider.GetProviderKey()) +// BatchRetrieve fetches a Vertex AI batch prediction job by ID (bare or full resource name). +func (provider *VertexProvider) BatchRetrieve(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostBatchRetrieveRequest) (*schemas.BifrostBatchRetrieveResponse, *schemas.BifrostError) { + if len(keys) == 0 { + return nil, providerUtils.NewBifrostOperationError("no keys provided for Vertex BatchRetrieve", nil) + } + + // A job ID is scoped to the project/region of the key that created it, so try each key + // until one resolves the job; return the last error only if all keys fail. + var lastErr *schemas.BifrostError + for _, key := range keys { + startTime := time.Now() + job, rawResponse, bifrostErr := provider.vertexGetBatchJob(ctx, key, request.BatchID) + if bifrostErr != nil { + lastErr = bifrostErr + continue + } + + result := vertexBatchJobToBifrost(job) + result.ExtraFields = schemas.BifrostResponseExtraFields{ + Latency: time.Since(startTime).Milliseconds(), + } + if providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse) { + result.ExtraFields.RawResponse = rawResponse + } + return &result, nil + } + + return nil, lastErr } -// BatchDelete is not supported by Vertex AI provider. -func (provider *VertexProvider) BatchDelete(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostBatchDeleteRequest) (*schemas.BifrostBatchDeleteResponse, *schemas.BifrostError) { - return nil, providerUtils.NewUnsupportedOperationError(schemas.BatchDeleteRequest, provider.GetProviderKey()) +// vertexGetBatchJob fetches a BatchPredictionJob resource. The returned rawResponse is +// the raw response payload when raw-response capture is enabled (nil otherwise); it is a +// GET, so there is no raw request to capture. +func (provider *VertexProvider) vertexGetBatchJob(ctx *schemas.BifrostContext, key schemas.Key, batchID string) (*VertexBatchPredictionJob, interface{}, *schemas.BifrostError) { + jobURL, cfgErr := vertexBatchJobURL(key, batchID) + if cfgErr != nil { + return nil, nil, cfgErr + } + + authHeader, authErr := gcsGetAuthHeader(key) + if authErr != nil { + return nil, nil, providerUtils.NewBifrostOperationError(authErr.Error(), nil) + } + + req := fasthttp.AcquireRequest() + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseRequest(req) + defer fasthttp.ReleaseResponse(resp) + + req.SetRequestURI(jobURL) + req.Header.SetMethod(http.MethodGet) + providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) + req.Header.Set("Authorization", authHeader) + + _, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, provider.client, req, resp) + defer wait() + if bifrostErr != nil { + return nil, nil, providerUtils.EnrichError(ctx, bifrostErr, nil, nil, provider.sendBackRawRequest, provider.sendBackRawResponse) + } + + if resp.StatusCode() != fasthttp.StatusOK { + if resp.StatusCode() == fasthttp.StatusUnauthorized || resp.StatusCode() == fasthttp.StatusForbidden { + removeVertexClient(key.VertexKeyConfig.AuthCredentials.GetValue()) + } + return nil, nil, providerUtils.EnrichError(ctx, parseVertexJobAPIError(resp.Body(), resp.StatusCode(), "batch retrieve"), nil, resp.Body(), provider.sendBackRawRequest, provider.sendBackRawResponse) + } + + var job VertexBatchPredictionJob + _, rawResponse, parseErr := providerUtils.HandleProviderResponse(resp.Body(), &job, nil, false, providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse)) + if parseErr != nil { + return nil, nil, providerUtils.EnrichError(ctx, parseErr, nil, resp.Body(), provider.sendBackRawRequest, provider.sendBackRawResponse) + } + return &job, rawResponse, nil } -// BatchResults is not supported by Vertex AI provider. -func (provider *VertexProvider) BatchResults(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostBatchResultsRequest) (*schemas.BifrostBatchResultsResponse, *schemas.BifrostError) { - return nil, providerUtils.NewUnsupportedOperationError(schemas.BatchResultsRequest, provider.GetProviderKey()) +// BatchCancel cancels a running Vertex AI batch prediction job. +func (provider *VertexProvider) BatchCancel(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostBatchCancelRequest) (*schemas.BifrostBatchCancelResponse, *schemas.BifrostError) { + if len(keys) == 0 { + return nil, providerUtils.NewBifrostOperationError("no keys provided for Vertex BatchCancel", nil) + } + + // A job ID is scoped to the project/region of the key that created it, so try each key + // until the cancel succeeds; return the last error only if all keys fail. + var lastErr *schemas.BifrostError + for _, key := range keys { + resp, bifrostErr := provider.batchCancelByKey(ctx, key, request) + if bifrostErr == nil { + return resp, nil + } + lastErr = bifrostErr + } + return nil, lastErr } -// FileUpload is not yet implemented for Vertex AI provider. -// Vertex AI uses Google Cloud Storage (GCS) for batch input/output files. -func (provider *VertexProvider) FileUpload(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostFileUploadRequest) (*schemas.BifrostFileUploadResponse, *schemas.BifrostError) { - return nil, providerUtils.NewUnsupportedOperationError(schemas.FileUploadRequest, provider.GetProviderKey()) +// batchCancelByKey cancels a batch prediction job using a single Vertex key. +func (provider *VertexProvider) batchCancelByKey(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostBatchCancelRequest) (*schemas.BifrostBatchCancelResponse, *schemas.BifrostError) { + jobURL, cfgErr := vertexBatchJobURL(key, request.BatchID) + if cfgErr != nil { + return nil, cfgErr + } + + authHeader, authErr := gcsGetAuthHeader(key) + if authErr != nil { + return nil, providerUtils.NewBifrostOperationError(authErr.Error(), nil) + } + + req := fasthttp.AcquireRequest() + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseRequest(req) + defer fasthttp.ReleaseResponse(resp) + + req.SetRequestURI(jobURL + ":cancel") + req.Header.SetMethod(http.MethodPost) + req.Header.SetContentType("application/json") + providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) + req.Header.Set("Authorization", authHeader) + + startTime := time.Now() + _, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, provider.client, req, resp) + defer wait() + if bifrostErr != nil { + return nil, providerUtils.EnrichError(ctx, bifrostErr, nil, nil, provider.sendBackRawRequest, provider.sendBackRawResponse) + } + + if resp.StatusCode() != fasthttp.StatusOK { + if resp.StatusCode() == fasthttp.StatusUnauthorized || resp.StatusCode() == fasthttp.StatusForbidden { + removeVertexClient(key.VertexKeyConfig.AuthCredentials.GetValue()) + } + return nil, providerUtils.EnrichError(ctx, parseVertexJobAPIError(resp.Body(), resp.StatusCode(), "batch cancel"), nil, resp.Body(), provider.sendBackRawRequest, provider.sendBackRawResponse) + } + + return &schemas.BifrostBatchCancelResponse{ + // Echo the caller's id so it stays stable across create/retrieve/cancel. + ID: request.BatchID, + Object: "batch", + Status: schemas.BatchStatusCancelling, + CancellingAt: schemas.Ptr(startTime.Unix()), + ExtraFields: schemas.BifrostResponseExtraFields{ + Latency: time.Since(startTime).Milliseconds(), + }, + }, nil } -// FileList is not yet implemented for Vertex AI provider. -func (provider *VertexProvider) FileList(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostFileListRequest) (*schemas.BifrostFileListResponse, *schemas.BifrostError) { - return nil, providerUtils.NewUnsupportedOperationError(schemas.FileListRequest, provider.GetProviderKey()) +// BatchDelete deletes a finished Vertex AI batch prediction job. +func (provider *VertexProvider) BatchDelete(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostBatchDeleteRequest) (*schemas.BifrostBatchDeleteResponse, *schemas.BifrostError) { + if len(keys) == 0 { + return nil, providerUtils.NewBifrostOperationError("no keys provided for Vertex BatchDelete", nil) + } + + // A job ID is scoped to the project/region of the key that created it, so try each key + // until the delete succeeds; return the last error only if all keys fail. + var lastErr *schemas.BifrostError + for _, key := range keys { + resp, bifrostErr := provider.batchDeleteByKey(ctx, key, request) + if bifrostErr == nil { + return resp, nil + } + lastErr = bifrostErr + } + return nil, lastErr } -// FileRetrieve is not yet implemented for Vertex AI provider. -func (provider *VertexProvider) FileRetrieve(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostFileRetrieveRequest) (*schemas.BifrostFileRetrieveResponse, *schemas.BifrostError) { - return nil, providerUtils.NewUnsupportedOperationError(schemas.FileRetrieveRequest, provider.GetProviderKey()) +// batchDeleteByKey deletes a batch prediction job using a single Vertex key. +func (provider *VertexProvider) batchDeleteByKey(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostBatchDeleteRequest) (*schemas.BifrostBatchDeleteResponse, *schemas.BifrostError) { + jobURL, cfgErr := vertexBatchJobURL(key, request.BatchID) + if cfgErr != nil { + return nil, cfgErr + } + + authHeader, authErr := gcsGetAuthHeader(key) + if authErr != nil { + return nil, providerUtils.NewBifrostOperationError(authErr.Error(), nil) + } + + req := fasthttp.AcquireRequest() + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseRequest(req) + defer fasthttp.ReleaseResponse(resp) + + req.SetRequestURI(jobURL) + req.Header.SetMethod(http.MethodDelete) + providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) + req.Header.Set("Authorization", authHeader) + + startTime := time.Now() + _, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, provider.client, req, resp) + defer wait() + if bifrostErr != nil { + return nil, providerUtils.EnrichError(ctx, bifrostErr, nil, nil, provider.sendBackRawRequest, provider.sendBackRawResponse) + } + + if resp.StatusCode() != fasthttp.StatusOK { + if resp.StatusCode() == fasthttp.StatusUnauthorized || resp.StatusCode() == fasthttp.StatusForbidden { + removeVertexClient(key.VertexKeyConfig.AuthCredentials.GetValue()) + } + return nil, providerUtils.EnrichError(ctx, parseVertexJobAPIError(resp.Body(), resp.StatusCode(), "batch delete"), nil, resp.Body(), provider.sendBackRawRequest, provider.sendBackRawResponse) + } + + return &schemas.BifrostBatchDeleteResponse{ + // Echo the caller's id so it stays stable across create/retrieve/delete. + ID: request.BatchID, + Object: "batch", + Status: schemas.BatchStatusDeleted, + ExtraFields: schemas.BifrostResponseExtraFields{ + Latency: time.Since(startTime).Milliseconds(), + }, + }, nil } -// FileDelete is not yet implemented for Vertex AI provider. -func (provider *VertexProvider) FileDelete(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostFileDeleteRequest) (*schemas.BifrostFileDeleteResponse, *schemas.BifrostError) { - return nil, providerUtils.NewUnsupportedOperationError(schemas.FileDeleteRequest, provider.GetProviderKey()) +// BatchResults reads the predictions-*.jsonl files a finished job wrote to its GCS +// output directory and maps each line to a Bifrost batch result item. The custom_id +// is recovered from the echoed request labels. +func (provider *VertexProvider) BatchResults(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostBatchResultsRequest) (*schemas.BifrostBatchResultsResponse, *schemas.BifrostError) { + if len(keys) == 0 { + return nil, providerUtils.NewBifrostOperationError("no keys provided for Vertex BatchResults", nil) + } + + // A job ID is scoped to the project/region of the key that created it, so try each key + // until one resolves the job and reads its results; return the last error if all fail. + var lastErr *schemas.BifrostError + for _, key := range keys { + resp, bifrostErr := provider.batchResultsByKey(ctx, key, request) + if bifrostErr == nil { + return resp, nil + } + lastErr = bifrostErr + } + return nil, lastErr } -// FileContent is not yet implemented for Vertex AI provider. -func (provider *VertexProvider) FileContent(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostFileContentRequest) (*schemas.BifrostFileContentResponse, *schemas.BifrostError) { - return nil, providerUtils.NewUnsupportedOperationError(schemas.FileContentRequest, provider.GetProviderKey()) +// batchResultsByKey reads a finished job's GCS output using a single Vertex key. +func (provider *VertexProvider) batchResultsByKey(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostBatchResultsRequest) (*schemas.BifrostBatchResultsResponse, *schemas.BifrostError) { + startTime := time.Now() + job, _, bifrostErr := provider.vertexGetBatchJob(ctx, key, request.BatchID) + if bifrostErr != nil { + return nil, bifrostErr + } + if job.OutputInfo == nil || job.OutputInfo.GcsOutputDirectory == "" { + return nil, providerUtils.NewBifrostOperationError(fmt.Sprintf("batch output is not available yet (job state: %s)", job.State), nil) + } + + bucket, dirKey, parseErr := parseGCSURI(job.OutputInfo.GcsOutputDirectory) + if parseErr != nil { + return nil, providerUtils.NewBifrostOperationError(parseErr.Error(), nil) + } + + authHeader, authErr := gcsGetAuthHeader(key) + if authErr != nil { + return nil, providerUtils.NewBifrostOperationError(authErr.Error(), nil) + } + + objects, listErr := provider.gcsListAllObjects(ctx, authHeader, bucket, strings.Trim(dirKey, "/")+"/") + if listErr != nil { + return nil, listErr + } + + results := []schemas.BatchResultItem{} + for _, obj := range objects { + name := obj.Name + if idx := strings.LastIndexByte(name, '/'); idx >= 0 { + name = name[idx+1:] + } + if !strings.HasPrefix(name, "predictions") { + continue + } + + content, downloadErr := provider.gcsDownloadObject(ctx, authHeader, bucket, obj.Name) + if downloadErr != nil { + return nil, downloadErr + } + + for _, rawLine := range bytes.Split(content, []byte("\n")) { + if len(bytes.TrimSpace(rawLine)) == 0 { + continue + } + var line VertexBatchOutputLine + if err := sonic.Unmarshal(rawLine, &line); err != nil { + continue // skip malformed lines rather than failing the whole result set + } + item := schemas.BatchResultItem{ + CustomID: line.Request.Labels[vertexBatchCustomIDLabel], + } + if line.Response != nil { + item.Response = &schemas.BatchResultResponse{ + StatusCode: 200, + Body: line.Response, + } + } else { + item.Error = &schemas.BatchResultError{Message: line.Status} + } + results = append(results, item) + } + } + + return &schemas.BifrostBatchResultsResponse{ + BatchID: request.BatchID, + Results: results, + ExtraFields: schemas.BifrostResponseExtraFields{ + Latency: time.Since(startTime).Milliseconds(), + }, + }, nil +} + +// gcsListAllObjects lists every object under a prefix, following pagination. +func (provider *VertexProvider) gcsListAllObjects(ctx *schemas.BifrostContext, authHeader, bucket, prefix string) ([]gcsObjectMetadata, *schemas.BifrostError) { + var objects []gcsObjectMetadata + pageToken := "" + for { + params := url.Values{} + params.Set("prefix", prefix) + params.Set("maxResults", "1000") + if pageToken != "" { + params.Set("pageToken", pageToken) + } + + req := fasthttp.AcquireRequest() + resp := fasthttp.AcquireResponse() + + req.SetRequestURI(fmt.Sprintf("%s/b/%s/o?%s", gcsStorageBase, url.PathEscape(bucket), params.Encode())) + req.Header.SetMethod(http.MethodGet) + providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) + req.Header.Set("Authorization", authHeader) + + _, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, provider.client, req, resp) + if bifrostErr != nil { + wait() + fasthttp.ReleaseRequest(req) + fasthttp.ReleaseResponse(resp) + return nil, bifrostErr + } + + statusCode := resp.StatusCode() + var listResp gcsObjectListResponse + var unmarshalErr error + if statusCode == fasthttp.StatusOK { + unmarshalErr = sonic.Unmarshal(resp.Body(), &listResp) + } + var apiErr *schemas.BifrostError + if statusCode != fasthttp.StatusOK { + apiErr = parseGCSAPIError(resp.Body(), statusCode, "list") + } + wait() + fasthttp.ReleaseRequest(req) + fasthttp.ReleaseResponse(resp) + + if apiErr != nil { + return nil, apiErr + } + if unmarshalErr != nil { + return nil, providerUtils.NewBifrostOperationError("failed to parse GCS list response", unmarshalErr) + } + + objects = append(objects, listResp.Items...) + if listResp.NextPageToken == "" { + return objects, nil + } + pageToken = listResp.NextPageToken + } +} + +// gcsDownloadObject downloads the raw bytes of a GCS object. +func (provider *VertexProvider) gcsDownloadObject(ctx *schemas.BifrostContext, authHeader, bucket, objectKey string) ([]byte, *schemas.BifrostError) { + req := fasthttp.AcquireRequest() + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseRequest(req) + defer fasthttp.ReleaseResponse(resp) + + req.SetRequestURI(fmt.Sprintf("%s/b/%s/o/%s?alt=media", gcsStorageBase, url.PathEscape(bucket), gcsEncodeObjectName(objectKey))) + req.Header.SetMethod(http.MethodGet) + providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) + req.Header.Set("Authorization", authHeader) + + _, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, provider.client, req, resp) + defer wait() + if bifrostErr != nil { + return nil, bifrostErr + } + + if resp.StatusCode() != fasthttp.StatusOK { + return nil, parseGCSAPIError(resp.Body(), resp.StatusCode(), "content download") + } + + content := make([]byte, len(resp.Body())) + copy(content, resp.Body()) + return content, nil +} + +const ( + gcsStorageBase = "https://storage.googleapis.com/storage/v1" + gcsUploadBase = "https://storage.googleapis.com/upload/storage/v1" +) + +// --- GCS helpers --- + +func gcsObjectKey(prefix, filename string) string { + key := "vertex-files/" + uuid.New().String() + "/" + filename + if prefix != "" { + key = strings.Trim(prefix, "/") + "/" + key + } + return key +} + +// gcsEncodeObjectName percent-encodes a GCS object name for the URL path, +// encoding slashes as %2F (url.PathEscape preserves them). +func gcsEncodeObjectName(name string) string { + return strings.ReplaceAll(url.PathEscape(name), "/", "%2F") +} + +func parseGCSURI(uri string) (bucket, objectKey string, err error) { + if !strings.HasPrefix(uri, "gs://") { + return "", "", fmt.Errorf("invalid GCS URI %q: must start with gs://", uri) + } + rest := strings.TrimPrefix(uri, "gs://") + idx := strings.IndexByte(rest, '/') + if idx < 0 { + return rest, "", nil + } + return rest[:idx], rest[idx+1:], nil +} + +func gcsParseTime(s string) int64 { + if s == "" { + return 0 + } + t, err := time.Parse(time.RFC3339Nano, s) + if err != nil { + return 0 + } + return t.Unix() +} + +func gcsParseSize(s string) int64 { + var n int64 + fmt.Sscanf(s, "%d", &n) + return n +} + +func gcsMetadataToFileObject(bucket string, obj gcsObjectMetadata) schemas.FileObject { + filename := obj.Metadata["bifrost_filename"] + if filename == "" { + // Fall back to last path segment of the object key. + if idx := strings.LastIndexByte(obj.Name, '/'); idx >= 0 { + filename = obj.Name[idx+1:] + } else { + filename = obj.Name + } + } + return schemas.FileObject{ + ID: "gs://" + bucket + "/" + obj.Name, + Object: "file", + Bytes: gcsParseSize(obj.Size), + CreatedAt: gcsParseTime(obj.TimeCreated), + UpdatedAt: gcsParseTime(obj.Updated), + Filename: filename, + Purpose: schemas.FilePurpose(obj.Metadata["bifrost_purpose"]), + Status: schemas.FileStatusProcessed, + } +} + +func gcsGetAuthHeader(key schemas.Key) (string, error) { + tokenSrc, err := getAuthTokenSource(key) + if err != nil { + return "", fmt.Errorf("failed to get GCS auth token source: %w", err) + } + tok, err := tokenSrc.Token() + if err != nil { + removeVertexClient(key.VertexKeyConfig.AuthCredentials.GetValue()) + return "", fmt.Errorf("failed to acquire GCS access token: %w", err) + } + return "Bearer " + tok.AccessToken, nil +} + +func parseGCSAPIError(body []byte, statusCode int, op string) *schemas.BifrostError { + var gcsErr gcsErrorBody + _ = sonic.Unmarshal(body, &gcsErr) + msg := gcsErr.Error.Message + if msg == "" { + msg = fmt.Sprintf("GCS %s failed with HTTP %d", op, statusCode) + } + return providerUtils.NewProviderAPIError(msg, nil, statusCode, nil, nil) +} + +// FileUpload uploads a file to GCS for use with Vertex AI inference. +// +// Two modes based on whether file bytes are provided: +// - Direct (request.File non-empty): uploads bytes via GCS multipart upload. +// - Resumable (request.File empty): mints a GCS resumable upload session URL. +// The client uploads bytes directly to GCS; Bifrost stays out of the data path. +func (provider *VertexProvider) FileUpload(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostFileUploadRequest) (*schemas.BifrostFileUploadResponse, *schemas.BifrostError) { + var bucket, prefix string + if request.StorageConfig != nil && request.StorageConfig.GCS != nil { + bucket = request.StorageConfig.GCS.Bucket + prefix = request.StorageConfig.GCS.Prefix + } + if bucket == "" { + return nil, providerUtils.NewBifrostOperationError("gcs_bucket is required for Vertex FileUpload (provide in storage_config.gcs)", nil) + } + + filename := request.Filename + if filename == "" { + filename = "file-" + uuid.New().String() + } + + objectKey := gcsObjectKey(prefix, filename) + gcsURI := "gs://" + bucket + "/" + objectKey + + contentType := "application/octet-stream" + if request.ContentType != nil && *request.ContentType != "" { + contentType = *request.ContentType + } + + authHeader, authErr := gcsGetAuthHeader(key) + if authErr != nil { + return nil, providerUtils.NewBifrostOperationError(authErr.Error(), nil) + } + + gcsMeta := map[string]string{ + "bifrost_filename": filename, + "bifrost_purpose": string(request.Purpose), + "bifrost_content_type": contentType, + } + + // GCS object metadata JSON, shared by both upload modes (multipart part 1 + // for direct uploads, session body for resumable uploads). + metaJSON, err := sonic.Marshal(map[string]interface{}{ + "name": objectKey, + "contentType": contentType, + "metadata": gcsMeta, + }) + if err != nil { + return nil, providerUtils.NewBifrostOperationError("failed to marshal GCS object metadata", err) + } + + startTime := time.Now() + + if len(request.File) == 0 { + return provider.gcsFileUploadResumable(ctx, key, authHeader, bucket, contentType, gcsURI, metaJSON, request, filename, startTime) + } + return provider.gcsFileUploadDirect(ctx, key, authHeader, bucket, contentType, gcsURI, metaJSON, request, filename, startTime) +} + +func (provider *VertexProvider) gcsFileUploadDirect( + ctx *schemas.BifrostContext, + key schemas.Key, + authHeader, bucket, contentType, gcsURI string, + metaJSON []byte, + request *schemas.BifrostFileUploadRequest, + filename string, + startTime time.Time, +) (*schemas.BifrostFileUploadResponse, *schemas.BifrostError) { + // Build GCS multipart/related body: part 1 = JSON object metadata, part 2 = file bytes. + var buf bytes.Buffer + mw := multipart.NewWriter(&buf) + + metaPartHeader := textproto.MIMEHeader{} + metaPartHeader.Set("Content-Type", "application/json; charset=UTF-8") + metaPart, err := mw.CreatePart(metaPartHeader) + if err != nil { + return nil, providerUtils.NewBifrostOperationError("failed to create GCS metadata part", err) + } + if _, err := metaPart.Write(metaJSON); err != nil { + return nil, providerUtils.NewBifrostOperationError("failed to write GCS metadata part", err) + } + + filePartHeader := textproto.MIMEHeader{} + filePartHeader.Set("Content-Type", contentType) + filePart, err := mw.CreatePart(filePartHeader) + if err != nil { + return nil, providerUtils.NewBifrostOperationError("failed to create GCS file part", err) + } + if _, err := filePart.Write(request.File); err != nil { + return nil, providerUtils.NewBifrostOperationError("failed to write file bytes", err) + } + if err := mw.Close(); err != nil { + return nil, providerUtils.NewBifrostOperationError("failed to finalise GCS multipart body", err) + } + + req := fasthttp.AcquireRequest() + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseRequest(req) + defer fasthttp.ReleaseResponse(resp) + + req.SetRequestURI(fmt.Sprintf("%s/b/%s/o?uploadType=multipart", gcsUploadBase, url.PathEscape(bucket))) + req.Header.SetMethod(http.MethodPost) + req.Header.SetContentType("multipart/related; boundary=" + mw.Boundary()) + providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) + req.Header.Set("Authorization", authHeader) + req.SetBody(buf.Bytes()) + + _, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, provider.client, req, resp) + defer wait() + if bifrostErr != nil { + return nil, bifrostErr + } + + if resp.StatusCode() != fasthttp.StatusOK && resp.StatusCode() != fasthttp.StatusCreated { + if resp.StatusCode() == fasthttp.StatusUnauthorized || resp.StatusCode() == fasthttp.StatusForbidden { + removeVertexClient(key.VertexKeyConfig.AuthCredentials.GetValue()) + } + return nil, parseGCSAPIError(resp.Body(), resp.StatusCode(), "upload") + } + + return &schemas.BifrostFileUploadResponse{ + ID: gcsURI, + Object: "file", + Bytes: int64(len(request.File)), + CreatedAt: startTime.Unix(), + Filename: filename, + Purpose: request.Purpose, + Status: schemas.FileStatusProcessed, + StorageBackend: schemas.FileStorageGCS, + StorageURI: gcsURI, + ExtraFields: schemas.BifrostResponseExtraFields{ + Latency: time.Since(startTime).Milliseconds(), + ProviderResponseHeaders: providerUtils.ExtractProviderResponseHeaders(resp), + }, + }, nil +} + +func (provider *VertexProvider) gcsFileUploadResumable( + ctx *schemas.BifrostContext, + key schemas.Key, + authHeader, bucket, contentType, gcsURI string, + metaJSON []byte, + request *schemas.BifrostFileUploadRequest, + filename string, + startTime time.Time, +) (*schemas.BifrostFileUploadResponse, *schemas.BifrostError) { + req := fasthttp.AcquireRequest() + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseRequest(req) + defer fasthttp.ReleaseResponse(resp) + + req.SetRequestURI(fmt.Sprintf("%s/b/%s/o?uploadType=resumable", gcsUploadBase, url.PathEscape(bucket))) + req.Header.SetMethod(http.MethodPost) + req.Header.SetContentType("application/json; charset=UTF-8") + providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) + req.Header.Set("Authorization", authHeader) + req.Header.Set("X-Upload-Content-Type", contentType) + req.SetBody(metaJSON) + + // content_length is optional but helps GCS validate the upload size. Depending + // on the transport it may arrive as a JSON number (float64) or a form-field + // string, so accept both numeric and string forms. + if request.ExtraParams != nil { + var contentLength int64 + switch cl := request.ExtraParams["content_length"].(type) { + case float64: + contentLength = int64(cl) + case int: + contentLength = int64(cl) + case int64: + contentLength = cl + case string: + if parsed, err := strconv.ParseInt(strings.TrimSpace(cl), 10, 64); err == nil { + contentLength = parsed + } + } + if contentLength > 0 { + req.Header.Set("X-Upload-Content-Length", fmt.Sprintf("%d", contentLength)) + } + } + + _, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, provider.client, req, resp) + defer wait() + if bifrostErr != nil { + return nil, bifrostErr + } + + if resp.StatusCode() != fasthttp.StatusOK { + if resp.StatusCode() == fasthttp.StatusUnauthorized || resp.StatusCode() == fasthttp.StatusForbidden { + removeVertexClient(key.VertexKeyConfig.AuthCredentials.GetValue()) + } + return nil, parseGCSAPIError(resp.Body(), resp.StatusCode(), "resumable session initiation") + } + + sessionURL := string(resp.Header.Peek("Location")) + if sessionURL == "" { + return nil, providerUtils.NewBifrostOperationError("GCS did not return a Location header for the resumable session", nil) + } + + return &schemas.BifrostFileUploadResponse{ + ID: gcsURI, + Object: "file", + Bytes: 0, + CreatedAt: startTime.Unix(), + Filename: filename, + Purpose: request.Purpose, + Status: schemas.FileStatusPendingUpload, + StorageBackend: schemas.FileStorageGCS, + StorageURI: gcsURI, + UploadURL: &sessionURL, + ExtraFields: schemas.BifrostResponseExtraFields{ + Latency: time.Since(startTime).Milliseconds(), + ProviderResponseHeaders: providerUtils.ExtractProviderResponseHeaders(resp), + }, + }, nil +} + +// FileList lists GCS objects under the configured prefix. +// Bucket must be provided via storage_config.gcs. +// Pagination is serial across keys: each key's GCS pages are exhausted (via the +// native pageToken) before moving to the next key. +func (provider *VertexProvider) FileList(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostFileListRequest) (*schemas.BifrostFileListResponse, *schemas.BifrostError) { + if len(keys) == 0 { + return nil, providerUtils.NewBifrostOperationError("no keys provided for Vertex FileList", nil) + } + var bucket, prefix string + if request.StorageConfig != nil && request.StorageConfig.GCS != nil { + bucket = request.StorageConfig.GCS.Bucket + prefix = request.StorageConfig.GCS.Prefix + } + if bucket == "" { + return nil, providerUtils.NewBifrostOperationError("gcs_bucket is required for Vertex FileList (provide in storage_config.gcs)", nil) + } + + // Serial pagination across keys: exhaust one key's pages before moving to the next. + helper, err := providerUtils.NewSerialListHelper(keys, request.After, provider.logger, true) + if err != nil { + return nil, providerUtils.NewBifrostOperationError("invalid pagination cursor", err) + } + + key, nativeCursor, ok := helper.GetCurrentKey() + if !ok { + // All keys exhausted + return &schemas.BifrostFileListResponse{ + Object: "list", + Data: []schemas.FileObject{}, + HasMore: false, + }, nil + } + + authHeader, authErr := gcsGetAuthHeader(key) + if authErr != nil { + return nil, providerUtils.NewBifrostOperationError(authErr.Error(), nil) + } + + params := url.Values{} + if prefix != "" { + params.Set("prefix", prefix) + } + limit := request.Limit + if limit <= 0 { + limit = 20 + } + params.Set("maxResults", fmt.Sprintf("%d", limit)) + if nativeCursor != "" { + params.Set("pageToken", nativeCursor) + } + + req := fasthttp.AcquireRequest() + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseRequest(req) + defer fasthttp.ReleaseResponse(resp) + + req.SetRequestURI(fmt.Sprintf("%s/b/%s/o?%s", gcsStorageBase, url.PathEscape(bucket), params.Encode())) + req.Header.SetMethod(http.MethodGet) + providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) + req.Header.Set("Authorization", authHeader) + + startTime := time.Now() + _, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, provider.client, req, resp) + defer wait() + if bifrostErr != nil { + return nil, bifrostErr + } + + if resp.StatusCode() != fasthttp.StatusOK { + if resp.StatusCode() == fasthttp.StatusUnauthorized || resp.StatusCode() == fasthttp.StatusForbidden { + removeVertexClient(key.VertexKeyConfig.AuthCredentials.GetValue()) + } + return nil, parseGCSAPIError(resp.Body(), resp.StatusCode(), "list") + } + + var listResp gcsObjectListResponse + if err := sonic.Unmarshal(resp.Body(), &listResp); err != nil { + return nil, providerUtils.NewBifrostOperationError("failed to parse GCS list response", err) + } + + files := make([]schemas.FileObject, 0, len(listResp.Items)) + for _, item := range listResp.Items { + files = append(files, gcsMetadataToFileObject(bucket, item)) + } + + // Build cursor for next request: stay on this key while it has more pages, + // then advance to the next key. + nextCursor, hasMore := helper.BuildNextCursor(listResp.NextPageToken != "", listResp.NextPageToken) + + result := &schemas.BifrostFileListResponse{ + Object: "list", + Data: files, + HasMore: hasMore, + ExtraFields: schemas.BifrostResponseExtraFields{ + Latency: time.Since(startTime).Milliseconds(), + }, + } + if nextCursor != "" { + result.After = &nextCursor + } + return result, nil +} + +// FileRetrieve fetches GCS object metadata, trying each key until one succeeds. +// FileID must be a gs:// URI. +func (provider *VertexProvider) FileRetrieve(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostFileRetrieveRequest) (*schemas.BifrostFileRetrieveResponse, *schemas.BifrostError) { + if len(keys) == 0 { + return nil, providerUtils.NewBifrostOperationError("no keys provided for Vertex FileRetrieve", nil) + } + + var lastErr *schemas.BifrostError + for _, key := range keys { + resp, err := provider.fileRetrieveByKey(ctx, key, request) + if err == nil { + return resp, nil + } + lastErr = err + provider.logger.Debug("Vertex FileRetrieve failed for key %s: %v", key.Name, err.Error) + } + return nil, lastErr +} + +// fileRetrieveByKey fetches GCS object metadata for a single key. +func (provider *VertexProvider) fileRetrieveByKey(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostFileRetrieveRequest) (*schemas.BifrostFileRetrieveResponse, *schemas.BifrostError) { + bucket, objectKey, parseErr := parseGCSURI(request.FileID) + if parseErr != nil { + return nil, providerUtils.NewBifrostOperationError(parseErr.Error(), nil) + } + + authHeader, authErr := gcsGetAuthHeader(key) + if authErr != nil { + return nil, providerUtils.NewBifrostOperationError(authErr.Error(), nil) + } + + req := fasthttp.AcquireRequest() + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseRequest(req) + defer fasthttp.ReleaseResponse(resp) + + req.SetRequestURI(fmt.Sprintf("%s/b/%s/o/%s", gcsStorageBase, url.PathEscape(bucket), gcsEncodeObjectName(objectKey))) + req.Header.SetMethod(http.MethodGet) + providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) + req.Header.Set("Authorization", authHeader) + + startTime := time.Now() + _, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, provider.client, req, resp) + defer wait() + if bifrostErr != nil { + return nil, bifrostErr + } + + if resp.StatusCode() != fasthttp.StatusOK { + if resp.StatusCode() == fasthttp.StatusUnauthorized || resp.StatusCode() == fasthttp.StatusForbidden { + removeVertexClient(key.VertexKeyConfig.AuthCredentials.GetValue()) + } + return nil, parseGCSAPIError(resp.Body(), resp.StatusCode(), "retrieve") + } + + var obj gcsObjectMetadata + if err := sonic.Unmarshal(resp.Body(), &obj); err != nil { + return nil, providerUtils.NewBifrostOperationError("failed to parse GCS object metadata", err) + } + + filename := obj.Metadata["bifrost_filename"] + if filename == "" { + if idx := strings.LastIndexByte(obj.Name, '/'); idx >= 0 { + filename = obj.Name[idx+1:] + } else { + filename = obj.Name + } + } + + return &schemas.BifrostFileRetrieveResponse{ + ID: request.FileID, + Object: "file", + Bytes: gcsParseSize(obj.Size), + CreatedAt: gcsParseTime(obj.TimeCreated), + UpdatedAt: gcsParseTime(obj.Updated), + Filename: filename, + Purpose: schemas.FilePurpose(obj.Metadata["bifrost_purpose"]), + Status: schemas.FileStatusProcessed, + StorageBackend: schemas.FileStorageGCS, + StorageURI: request.FileID, + ExtraFields: schemas.BifrostResponseExtraFields{ + Latency: time.Since(startTime).Milliseconds(), + }, + }, nil +} + +// FileDelete deletes a GCS object, trying each key until one succeeds. +// FileID must be a gs:// URI. Deleting a non-existent object is treated as +// success (idempotent). +func (provider *VertexProvider) FileDelete(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostFileDeleteRequest) (*schemas.BifrostFileDeleteResponse, *schemas.BifrostError) { + if len(keys) == 0 { + return nil, providerUtils.NewBifrostOperationError("no keys provided for Vertex FileDelete", nil) + } + + var lastErr *schemas.BifrostError + for _, key := range keys { + resp, err := provider.fileDeleteByKey(ctx, key, request) + if err == nil { + return resp, nil + } + lastErr = err + provider.logger.Debug("Vertex FileDelete failed for key %s: %v", key.Name, err.Error) + } + return nil, lastErr +} + +// fileDeleteByKey deletes a GCS object for a single key. +func (provider *VertexProvider) fileDeleteByKey(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostFileDeleteRequest) (*schemas.BifrostFileDeleteResponse, *schemas.BifrostError) { + bucket, objectKey, parseErr := parseGCSURI(request.FileID) + if parseErr != nil { + return nil, providerUtils.NewBifrostOperationError(parseErr.Error(), nil) + } + + authHeader, authErr := gcsGetAuthHeader(key) + if authErr != nil { + return nil, providerUtils.NewBifrostOperationError(authErr.Error(), nil) + } + + req := fasthttp.AcquireRequest() + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseRequest(req) + defer fasthttp.ReleaseResponse(resp) + + req.SetRequestURI(fmt.Sprintf("%s/b/%s/o/%s", gcsStorageBase, url.PathEscape(bucket), gcsEncodeObjectName(objectKey))) + req.Header.SetMethod(http.MethodDelete) + providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) + req.Header.Set("Authorization", authHeader) + + startTime := time.Now() + _, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, provider.client, req, resp) + defer wait() + if bifrostErr != nil { + return nil, bifrostErr + } + + // 204 = deleted; 404 = already gone — both succeed for an idempotent delete. + if resp.StatusCode() != fasthttp.StatusNoContent && resp.StatusCode() != fasthttp.StatusNotFound { + if resp.StatusCode() == fasthttp.StatusUnauthorized || resp.StatusCode() == fasthttp.StatusForbidden { + removeVertexClient(key.VertexKeyConfig.AuthCredentials.GetValue()) + } + return nil, parseGCSAPIError(resp.Body(), resp.StatusCode(), "delete") + } + + return &schemas.BifrostFileDeleteResponse{ + ID: request.FileID, + Object: "file", + Deleted: true, + ExtraFields: schemas.BifrostResponseExtraFields{ + Latency: time.Since(startTime).Milliseconds(), + }, + }, nil +} + +// FileContent downloads the raw bytes of a GCS object, trying each key until one +// succeeds. FileID must be a gs:// URI. +func (provider *VertexProvider) FileContent(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostFileContentRequest) (*schemas.BifrostFileContentResponse, *schemas.BifrostError) { + if len(keys) == 0 { + return nil, providerUtils.NewBifrostOperationError("no keys provided for Vertex FileContent", nil) + } + + var lastErr *schemas.BifrostError + for _, key := range keys { + resp, err := provider.fileContentByKey(ctx, key, request) + if err == nil { + return resp, nil + } + lastErr = err + provider.logger.Debug("Vertex FileContent failed for key %s: %v", key.Name, err.Error) + } + return nil, lastErr +} + +// fileContentByKey downloads the raw bytes of a GCS object for a single key. +func (provider *VertexProvider) fileContentByKey(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostFileContentRequest) (*schemas.BifrostFileContentResponse, *schemas.BifrostError) { + bucket, objectKey, parseErr := parseGCSURI(request.FileID) + if parseErr != nil { + return nil, providerUtils.NewBifrostOperationError(parseErr.Error(), nil) + } + + authHeader, authErr := gcsGetAuthHeader(key) + if authErr != nil { + return nil, providerUtils.NewBifrostOperationError(authErr.Error(), nil) + } + + req := fasthttp.AcquireRequest() + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseRequest(req) + defer fasthttp.ReleaseResponse(resp) + + req.SetRequestURI(fmt.Sprintf("%s/b/%s/o/%s?alt=media", gcsStorageBase, url.PathEscape(bucket), gcsEncodeObjectName(objectKey))) + req.Header.SetMethod(http.MethodGet) + providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) + req.Header.Set("Authorization", authHeader) + + startTime := time.Now() + _, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, provider.client, req, resp) + defer wait() + if bifrostErr != nil { + return nil, bifrostErr + } + + if resp.StatusCode() != fasthttp.StatusOK { + if resp.StatusCode() == fasthttp.StatusUnauthorized || resp.StatusCode() == fasthttp.StatusForbidden { + removeVertexClient(key.VertexKeyConfig.AuthCredentials.GetValue()) + } + return nil, parseGCSAPIError(resp.Body(), resp.StatusCode(), "content download") + } + + // Copy body before deferred ReleaseResponse invalidates the buffer. + content := make([]byte, len(resp.Body())) + copy(content, resp.Body()) + + contentType := string(resp.Header.Peek("Content-Type")) + if contentType == "" { + contentType = "application/octet-stream" + } + + return &schemas.BifrostFileContentResponse{ + FileID: request.FileID, + Content: content, + ContentType: contentType, + ExtraFields: schemas.BifrostResponseExtraFields{ + Latency: time.Since(startTime).Milliseconds(), + }, + }, nil } // CountTokens counts the number of tokens in the provided content using Vertex AI's countTokens endpoint. @@ -2682,7 +3974,7 @@ func (provider *VertexProvider) CountTokens(ctx *schemas.BifrostContext, key sch bifrostErr *schemas.BifrostError ) - if schemas.IsAnthropicModel(request.Model) { + if schemas.IsAnthropicModelFamily(ctx, request.Model) { jsonBody, bifrostErr = getRequestBodyForAnthropicResponses(ctx, request, request.Model, false, true, provider.networkConfig.BetaHeaderOverrides, provider.networkConfig.ExtraHeaders, provider.sendBackRawRequest, provider.sendBackRawResponse) if bifrostErr != nil { return nil, bifrostErr @@ -2713,12 +4005,12 @@ func (provider *VertexProvider) CountTokens(ctx *schemas.BifrostContext, key sch } } - projectID := key.VertexKeyConfig.ProjectID.GetValue() + projectID := resolveVertexProjectID(ctx, key) if projectID == "" { return nil, providerUtils.NewConfigurationError("project ID is not set") } - region := key.VertexKeyConfig.Region.GetValue() + region := resolveVertexRegion(ctx, key) if region == "" { return nil, providerUtils.NewConfigurationError("region is not set in key config") } @@ -2726,17 +4018,17 @@ func (provider *VertexProvider) CountTokens(ctx *schemas.BifrostContext, key sch authQuery := "" var completeURL string - if schemas.IsAnthropicModel(request.Model) { + if schemas.IsAnthropicModelFamily(ctx, request.Model) { // Use model-aware host based on request.Model, but URL path uses "count-tokens" effectiveRegion := getVertexEffectiveRegion(region, request.Model) baseURL := getVertexModelAwareAPIBaseURL(region, "v1", request.Model) completeURL = fmt.Sprintf("%s/projects/%s/locations/%s/publishers/%s/models/%s%s", baseURL, projectID, effectiveRegion, "anthropic", "count-tokens", ":rawPredict") - } else if schemas.IsGeminiModel(request.Model) || schemas.IsAllDigitsASCII(request.Model) || schemas.IsGemmaModel(request.Model) { + } else if schemas.IsGeminiModelFamily(ctx, request.Model) || schemas.IsAllDigitsASCII(request.Model) || schemas.IsGemmaModelFamily(ctx, request.Model) { if key.Value.GetValue() != "" { authQuery = fmt.Sprintf("key=%s", url.QueryEscape(key.Value.GetValue())) } - projectNumber := key.VertexKeyConfig.ProjectNumber.GetValue() + projectNumber := resolveVertexProjectNumber(ctx, key) if schemas.IsAllDigitsASCII(request.Model) && projectNumber == "" { return nil, providerUtils.NewConfigurationError("project number is not set for fine-tuned models") } @@ -2816,7 +4108,7 @@ func (provider *VertexProvider) CountTokens(ctx *schemas.BifrostContext, key sch }, nil } - if schemas.IsAnthropicModel(request.Model) { + if schemas.IsAnthropicModelFamily(ctx, request.Model) { anthropicResponse := &anthropic.AnthropicCountTokensResponse{} rawRequest, rawResponse, bifrostErr := providerUtils.HandleProviderResponse(responseBody, anthropicResponse, jsonBody, providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest), providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse)) @@ -2916,12 +4208,12 @@ func (provider *VertexProvider) Passthrough( key schemas.Key, req *schemas.BifrostPassthroughRequest, ) (*schemas.BifrostPassthroughResponse, *schemas.BifrostError) { - projectID := strings.TrimSpace(key.VertexKeyConfig.ProjectID.GetValue()) + projectID := strings.TrimSpace(resolveVertexProjectID(ctx, key)) if projectID == "" { return nil, providerUtils.NewConfigurationError("project ID is not set") } - keyRegion := key.VertexKeyConfig.Region.GetValue() + keyRegion := resolveVertexRegion(ctx, key) if keyRegion == "" { keyRegion = "global" } @@ -3065,12 +4357,12 @@ func (provider *VertexProvider) PassthroughStream( key schemas.Key, req *schemas.BifrostPassthroughRequest, ) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { - projectID := strings.TrimSpace(key.VertexKeyConfig.ProjectID.GetValue()) + projectID := strings.TrimSpace(resolveVertexProjectID(ctx, key)) if projectID == "" { return nil, providerUtils.NewConfigurationError("project ID is not set") } - keyRegion := key.VertexKeyConfig.Region.GetValue() + keyRegion := resolveVertexRegion(ctx, key) if keyRegion == "" { keyRegion = "global" } diff --git a/core/providers/vertex/vertex_test.go b/core/providers/vertex/vertex_test.go index d754f33d22..d29499f232 100644 --- a/core/providers/vertex/vertex_test.go +++ b/core/providers/vertex/vertex_test.go @@ -25,6 +25,27 @@ func TestVertex(t *testing.T) { rerankModel := strings.TrimSpace(os.Getenv("VERTEX_RERANK_MODEL")) + // Vertex file operations are GCS-backed: the bucket/prefix are passed via the typed + // StorageConfig (VERTEX_GCS_BUCKET, optional VERTEX_GCS_PREFIX), not extra_params. + var fileStorageConfig *schemas.FileStorageConfig + var batchOutputFolder *schemas.BatchOutputFolder + if gcsBucket := strings.TrimSpace(os.Getenv("VERTEX_GCS_BUCKET")); gcsBucket != "" { + gcsPrefix := strings.TrimSpace(os.Getenv("VERTEX_GCS_PREFIX")) + fileStorageConfig = &schemas.FileStorageConfig{ + GCS: &schemas.GCSStorageConfig{ + Bucket: gcsBucket, + Prefix: gcsPrefix, + }, + } + // Batch output is a gs:// prefix; Vertex writes results into its own subdirectory under it. + outputURI := "gs://" + gcsBucket + if gcsPrefix != "" { + outputURI += "/" + strings.Trim(gcsPrefix, "/") + } + outputURI += "/batch-output" + batchOutputFolder = &schemas.BatchOutputFolder{URL: outputURI} + } + testConfig := llmtests.ComprehensiveTestConfig{ Provider: schemas.Vertex, ChatModel: "gemini-2.5-pro", @@ -37,41 +58,54 @@ func TestVertex(t *testing.T) { ImageGenerationModel: "gemini-2.5-flash-image", ImageEditModel: "imagen-3.0-capability-001", VideoGenerationModel: "veo-3.1-generate-preview", + FileStorageConfig: fileStorageConfig, + BatchOutputFolder: batchOutputFolder, Scenarios: llmtests.TestScenarios{ - TextCompletion: false, // Not supported - SimpleChat: true, - CompletionStream: true, - MultiTurnConversation: true, - ToolCalls: true, - ToolCallsStreaming: true, - MultipleToolCalls: true, - MultipleToolCallsStreaming: true, - End2EndToolCalling: true, - AutomaticFunctionCall: true, - ImageURL: false, - ImageBase64: true, - ImageGeneration: true, - ImageGenerationStream: false, - ImageEdit: true, - VideoGeneration: false, // disabled for now because of long running operations - VideoRetrieve: false, - VideoRemix: false, - VideoDownload: false, - VideoList: false, - VideoDelete: false, - MultipleImages: true, - CompleteEnd2End: true, - FileBase64: true, - Embedding: true, - Rerank: rerankModel != "", - Reasoning: true, - PromptCaching: true, - ListModels: false, - CountTokens: true, - StructuredOutputs: true, // Structured outputs with nullable enum support + TextCompletion: false, // Not supported + SimpleChat: true, + CompletionStream: true, + MultiTurnConversation: true, + ToolCalls: true, + ToolCallsStreaming: true, + MultipleToolCalls: true, + MultipleToolCallsStreaming: true, + End2EndToolCalling: true, + AutomaticFunctionCall: true, + ImageURL: false, + ImageBase64: true, + ImageGeneration: true, + ImageGenerationStream: false, + ImageEdit: true, + VideoGeneration: false, // disabled for now because of long running operations + VideoRetrieve: false, + VideoRemix: false, + VideoDownload: false, + VideoList: false, + VideoDelete: false, + MultipleImages: true, + CompleteEnd2End: true, + FileBase64: true, + Embedding: true, + Rerank: rerankModel != "", + Reasoning: true, + PromptCaching: true, + ListModels: false, + CountTokens: true, + StructuredOutputs: true, // Structured outputs with nullable enum support InterleavedThinking: true, EagerInputStreaming: true, // fine-grained-tool-streaming-2025-05-14 (GA on Vertex) ServerToolsViaOpenAIEndpoint: true, // web_search only on Vertex per Table 20 (web_fetch/code_execution skip) + FileUpload: true, + FileList: true, + FileRetrieve: true, + FileDelete: true, + FileContent: true, + FileBatchInput: true, + BatchCreate: true, + BatchList: true, + BatchRetrieve: true, + BatchCancel: true, + BatchResults: true, }, } diff --git a/core/schemas/account.go b/core/schemas/account.go index 449399f554..a01d2162cc 100644 --- a/core/schemas/account.go +++ b/core/schemas/account.go @@ -2,10 +2,14 @@ package schemas import ( + "bytes" "context" + "encoding/json" "fmt" "slices" "strings" + + "github.com/bytedance/sonic" ) type KeyStatusType string @@ -141,22 +145,162 @@ type Key struct { Description string `json:"description,omitempty"` // Description of key } -type KeyAliases map[string]string +// ModelFamily is a typed enum identifying the underlying model family of an alias target. +// It enables provider routing decisions (request shape, response parsing, auth headers, +// URL construction) without substring-sniffing the wire model ID. +type ModelFamily string + +const ( + ModelFamilyAnthropic ModelFamily = "anthropic" + ModelFamilyOpenAI ModelFamily = "openai" + ModelFamilyMistral ModelFamily = "mistral" + ModelFamilyCohere ModelFamily = "cohere" + ModelFamilyGemini ModelFamily = "gemini" + ModelFamilyGemma ModelFamily = "gemma" + ModelFamilyLlama ModelFamily = "llama" + ModelFamilyImagen ModelFamily = "imagen" + ModelFamilyVeo ModelFamily = "veo" + ModelFamilyNova ModelFamily = "nova" + ModelFamilyTitan ModelFamily = "titan" +) + +// IsValid reports whether mf is a recognized model family. +func (mf *ModelFamily) IsValid() bool { + if mf == nil { + return false + } + switch *mf { + case ModelFamilyAnthropic, ModelFamilyOpenAI, ModelFamilyMistral, + ModelFamilyCohere, ModelFamilyGemini, ModelFamilyGemma, + ModelFamilyLlama, ModelFamilyImagen, ModelFamilyVeo, + ModelFamilyNova, ModelFamilyTitan: + return true + } + return false +} + +// AzureAliasCfg holds Azure-specific overrides that apply to a single alias. +// Each field, when non-nil, overrides the corresponding key-level default. +type AzureAliasCfg struct { + APIVersion *string `json:"api_version,omitempty"` // overrides the Azure OpenAI api-version query param for this alias + AnthropicVersion *string `json:"anthropic_version,omitempty"` // overrides the anthropic-version header for Claude-on-Azure deployments + Endpoint *EnvVar `json:"endpoint,omitempty"` // overrides AzureKeyConfig.Endpoint for this alias — lets one credential span deployments on multiple Azure resources +} + +// VertexAliasCfg holds Vertex-specific overrides that apply to a single alias. +type VertexAliasCfg struct { + ProjectID *EnvVar `json:"project_id,omitempty"` + ProjectNumber *EnvVar `json:"project_number,omitempty"` +} + +// BedrockAliasCfg holds Bedrock-specific overrides that apply to a single alias. +type BedrockAliasCfg struct { + InferenceProfileARN *EnvVar `json:"inference_profile_arn,omitempty"` +} + +// ReplicateAliasCfg holds Replicate-specific overrides that apply to a single alias. +type ReplicateAliasCfg struct { + UseDeploymentsEndpoint *bool `json:"use_deployments_endpoint,omitempty"` +} + +// AliasConfig is the rich value type held by KeyAliases. It carries everything +// needed to call a provider for an aliased model: the wire model identifier +// (ModelID), the canonical model name used for pricing/logging (ModelName), the +// family used for provider routing decisions (ModelFamily), and optional +// provider-specific overrides that override the key-level defaults. +type AliasConfig struct { + ModelID string `json:"model_id"` // wire model identifier sent to the provider + ModelName *string `json:"model_name,omitempty"` // canonical model name used for pricing, logging, and 2nd-tier family routing + ModelFamily *ModelFamily `json:"model_family,omitempty"` // 1st-tier family routing enum + Description string `json:"description,omitempty"` // description of the alias for users to understand its purpose (not used by bifrost) + Region *EnvVar `json:"region,omitempty"` + + *AzureAliasCfg + *VertexAliasCfg + *BedrockAliasCfg + *ReplicateAliasCfg +} + +// isLegacyShape reports whether this AliasConfig carries only ModelID and no +// other fields. Used by MarshalJSON to emit the legacy string-valued wire +// shape so older consumers that expect map[string]string keep working. +func (ac AliasConfig) isLegacyShape() bool { + return ac.ModelID != "" && + ac.ModelName == nil && + ac.ModelFamily == nil && + ac.Description == "" && + ac.Region == nil && + ac.AzureAliasCfg == nil && + ac.VertexAliasCfg == nil && + ac.BedrockAliasCfg == nil && + ac.ReplicateAliasCfg == nil +} + +// MarshalJSON emits the legacy string wire shape when only ModelID is set, so +// callers that haven't opted into the rich AliasConfig see no observable +// change on the wire. When any other field is populated, the full object is +// emitted. +func (ac AliasConfig) MarshalJSON() ([]byte, error) { + if ac.isLegacyShape() { + return Marshal(ac.ModelID) + } + type aliasConfigJSON AliasConfig + return Marshal(aliasConfigJSON(ac)) +} -func (ka KeyAliases) Validate() error { +// KeyAliases maps a user-facing model name to its AliasConfig. +// +// Both the input (UnmarshalJSON) and the output (AliasConfig.MarshalJSON) +// transparently accept and emit two JSON wire shapes: +// - Legacy: {"my-model": "provider-model-id"} — value is a string +// - New: {"my-model": {"model_id": "provider-model-id", ... }} — value is an object +// +// Legacy entries deserialize to AliasConfig{ModelID: }; an AliasConfig +// that only has ModelID set serializes back to a plain string. This keeps the +// wire format byte-for-byte compatible with the pre-refactor flow until +// ModelName / ModelFamily / provider sub-configs are populated explicitly. +type KeyAliases map[string]AliasConfig + +// Validate checks that every entry in the alias map is well-formed and that +// any provider-specific sub-configs (AzureAliasCfg, VertexAliasCfg, +// BedrockAliasCfg, ReplicateAliasCfg) are only set when the owning Key +// actually belongs to that provider. Catches misconfigurations like an +// AzureAliasCfg attached to a Bedrock key. +// +// providerKey is the provider this Key is registered under (e.g. schemas.Azure +// for keys in the azure provider config). +func (ka KeyAliases) Validate(providerKey ModelProvider) error { seen := make(map[string]struct{}, len(ka)) - for from, to := range ka { + for from, ac := range ka { if strings.TrimSpace(from) == "" { return fmt.Errorf("alias source cannot be empty") } - if strings.TrimSpace(to) == "" { - return fmt.Errorf("alias target for %q cannot be empty", from) + if strings.TrimSpace(ac.ModelID) == "" { + return fmt.Errorf("alias %q: model_id cannot be empty", from) } if strings.TrimSpace(from) != from { return fmt.Errorf("alias source %q cannot have leading or trailing whitespace", from) } - if strings.TrimSpace(to) != to { - return fmt.Errorf("alias target for %q cannot have leading or trailing whitespace", from) + if strings.TrimSpace(ac.ModelID) != ac.ModelID { + return fmt.Errorf("alias %q: model_id cannot have leading or trailing whitespace", from) + } + if ac.ModelName != nil && strings.TrimSpace(*ac.ModelName) != *ac.ModelName { + return fmt.Errorf("alias %q: model_name cannot have leading or trailing whitespace", from) + } + if ac.ModelFamily != nil && !ac.ModelFamily.IsValid() { + return fmt.Errorf("alias %q: invalid model_family %q", from, *ac.ModelFamily) + } + if ac.AzureAliasCfg != nil && providerKey != Azure { + return fmt.Errorf("alias %q: azure sub-config is only valid on Azure keys (got provider %q)", from, providerKey) + } + if ac.VertexAliasCfg != nil && providerKey != Vertex { + return fmt.Errorf("alias %q: vertex sub-config is only valid on Vertex keys (got provider %q)", from, providerKey) + } + if ac.BedrockAliasCfg != nil && providerKey != Bedrock { + return fmt.Errorf("alias %q: bedrock sub-config is only valid on Bedrock keys (got provider %q)", from, providerKey) + } + if ac.ReplicateAliasCfg != nil && providerKey != Replicate { + return fmt.Errorf("alias %q: replicate sub-config is only valid on Replicate keys (got provider %q)", from, providerKey) } normalized := strings.ToLower(from) if _, ok := seen[normalized]; ok { @@ -167,20 +311,265 @@ func (ka KeyAliases) Validate() error { return nil } +// Resolve returns the wire model identifier for the given user-facing model name. +// If no alias matches, the input is returned unchanged. Case-insensitive fallback +// matches the prior behavior. +// +// This signature is preserved for backward compatibility with existing callers +// that only need the wire model string. For access to the full AliasConfig +// (ModelName, ModelFamily, provider overrides), use ResolveConfig. func (ka KeyAliases) Resolve(model string) string { + if ac := ka.ResolveConfig(model); ac != nil { + return ac.ModelID + } + return model +} + +// ResolvedAlias is what core stashes in BifrostContext after key-level alias +// resolution. Key is the user-facing model name the client sent (LHS of the +// alias map). Config is the matched AliasConfig. +// +// Carrying the alias key alongside the config lets providers consult it as +// the lowest-precedence tier for family detection — common case: an admin +// names their alias "best-claude" but the wire ModelID is an opaque Azure +// deployment ID, so neither the config fields nor request.Model carry the +// "claude" substring; the alias key does. +type ResolvedAlias struct { + Key string + Config *AliasConfig +} + +// GetResolvedAlias returns the ResolvedAlias that core stashed in ctx after +// key-level alias resolution, or nil if no alias matched or ctx is nil. +// +// This is set by bifrost.go alongside req.SetModel(resolved). Plugins must +// not write to this key directly. +func GetResolvedAlias(ctx *BifrostContext) *ResolvedAlias { + if ctx == nil { + return nil + } + v := ctx.Value(BifrostContextKeyResolvedAlias) + if v == nil { + return nil + } + ra, _ := v.(*ResolvedAlias) + return ra +} + +// ResolveFamily returns the model family for the current attempt, walking +// the precedence: explicit alias ModelFamily → alias ModelName → alias +// ModelID → alias Key. When no alias matched, falls back to substring +// matching against fallbackModel (typically request.Model), preserving +// pre-refactor behavior. +// +// Returns an empty ModelFamily if nothing matches. +func ResolveFamily(ctx *BifrostContext, fallbackModel string) ModelFamily { + ra := GetResolvedAlias(ctx) + var candidates []string + if ra != nil && ra.Config != nil { + if ra.Config.ModelFamily != nil && *ra.Config.ModelFamily != "" { + return *ra.Config.ModelFamily + } + if ra.Config.ModelName != nil { + candidates = append(candidates, *ra.Config.ModelName) + } + candidates = append(candidates, ra.Config.ModelID, ra.Key) + } else { + candidates = append(candidates, fallbackModel) + } + for _, s := range candidates { + switch { + case IsAnthropicModel(s): + return ModelFamilyAnthropic + case IsMistralModel(s): + return ModelFamilyMistral + // Imagen and Veo are checked before Gemini as a defensive ordering: + // they are distinct Google model families whose names do not contain + // "gemini", so they could never be mis-classified here, but keeping + // them first makes the intent explicit. + case IsImagenModel(s): + return ModelFamilyImagen + case IsVeoModel(s): + return ModelFamilyVeo + case IsGeminiModel(s): + return ModelFamilyGemini + case IsGemmaModel(s): + return ModelFamilyGemma + case IsLlamaModel(s): + return ModelFamilyLlama + case IsNovaModel(s): + return ModelFamilyNova + case IsTitanModel(s): + return ModelFamilyTitan + case IsCohereModel(s): + return ModelFamilyCohere + } + } + return "" +} + +// IsAnthropicModelFamily reports whether the current attempt resolves to the +// Anthropic model family. Thin wrapper over ResolveFamily so provider code +// reads uniformly at the many call sites that branch on Anthropic vs +// non-Anthropic (request shape, response parsing, anthropic-version header, +// URL path construction). model is passed as the substring-match fallback +// used when no alias is resolved in ctx — typically request.Model. +func IsAnthropicModelFamily(ctx *BifrostContext, model string) bool { + return ResolveFamily(ctx, model) == ModelFamilyAnthropic +} + +// IsMistralModelFamily reports whether the current attempt resolves to the +// Mistral model family. See IsAnthropicModelFamily for usage notes. +func IsMistralModelFamily(ctx *BifrostContext, model string) bool { + return ResolveFamily(ctx, model) == ModelFamilyMistral +} + +// IsLlamaModelFamily reports whether the current attempt resolves to the +// Llama model family. Used by Bedrock to gate tool_choice handling — AWS +// Bedrock Converse rejects toolConfig.toolChoice.tool on Meta Llama variants. +func IsLlamaModelFamily(ctx *BifrostContext, model string) bool { + return ResolveFamily(ctx, model) == ModelFamilyLlama +} + +// IsNovaModelFamily reports whether the current attempt resolves to the +// Amazon Nova model family. Used by Bedrock to gate cache-point insertion +// and tool shaping that differs from Anthropic. +func IsNovaModelFamily(ctx *BifrostContext, model string) bool { + return ResolveFamily(ctx, model) == ModelFamilyNova +} + +// IsCohereModelFamily reports whether the current attempt resolves to the +// Cohere model family. Used by Bedrock to pick the Cohere request/response +// shape for embeddings (vs. the Titan envelope). +func IsCohereModelFamily(ctx *BifrostContext, model string) bool { + return ResolveFamily(ctx, model) == ModelFamilyCohere +} + +// IsTitanModelFamily reports whether the current attempt resolves to the +// Amazon Titan model family. Used by Bedrock to pick the Titan embedding +// request/response envelope. +func IsTitanModelFamily(ctx *BifrostContext, model string) bool { + return ResolveFamily(ctx, model) == ModelFamilyTitan +} + +// IsGeminiModelFamily reports whether the current attempt resolves to the +// Google Gemini model family. Used by Vertex to pick Gemini-shaped request +// transforms and the publishers/google URL prefix. +func IsGeminiModelFamily(ctx *BifrostContext, model string) bool { + return ResolveFamily(ctx, model) == ModelFamilyGemini +} + +// IsGemmaModelFamily reports whether the current attempt resolves to the +// Gemma model family. Vertex routes Gemma via the publishers/google path +// alongside Gemini. +func IsGemmaModelFamily(ctx *BifrostContext, model string) bool { + return ResolveFamily(ctx, model) == ModelFamilyGemma +} + +// IsImagenModelFamily reports whether the current attempt resolves to the +// Imagen model family. Used by Vertex for the :predict endpoint and Imagen- +// specific request shaping. +func IsImagenModelFamily(ctx *BifrostContext, model string) bool { + return ResolveFamily(ctx, model) == ModelFamilyImagen +} + +// IsVeoModelFamily reports whether the current attempt resolves to the Veo +// model family. Used by Vertex for video-generation request shaping. +func IsVeoModelFamily(ctx *BifrostContext, model string) bool { + return ResolveFamily(ctx, model) == ModelFamilyVeo +} + +// BuildRoutingInfo constructs a RoutingInfo for the current attempt from this +// attempt's chosen provider/model/key and the resolved alias stashed in ctx. +// +// Populates only the per-attempt fields (Provider, Model, Key, +// ResolvedKeyAlias). IsFallback and PrimaryProvider/PrimaryModel are layered +// on later by the orchestrator (handleRequest / handleStreamRequest) via +// SetFallbackRoutingInfo on the final response/error, since those signals +// belong to the orchestrator scope rather than the per-attempt one. +// +// ResolvedKeyAlias.ModelFamily reflects the family explicitly configured on +// the alias (nil when the admin didn't set one) — not the substring-resolved +// family used for routing. +func BuildRoutingInfo(ctx *BifrostContext, attemptProvider ModelProvider, attemptModel string, attemptKey Key) RoutingInfo { + info := RoutingInfo{ + Provider: attemptProvider, + Model: attemptModel, + Key: attemptKey.Name, + } + if ra := GetResolvedAlias(ctx); ra != nil && ra.Config != nil { + rka := &ResolvedKeyAlias{ + ModelID: ra.Config.ModelID, + } + if ra.Config.ModelName != nil { + mn := *ra.Config.ModelName + rka.ModelName = &mn + } + if ra.Config.ModelFamily != nil { + f := *ra.Config.ModelFamily + rka.ModelFamily = &f + } + info.ResolvedKeyAlias = rka + } + return info +} + +// ResolveConfig returns the AliasConfig for the given user-facing model name, +// or nil if no alias matches. Case-insensitive fallback matches Resolve. +func (ka KeyAliases) ResolveConfig(model string) *AliasConfig { if ka == nil { - return model + return nil } - if alias, ok := ka[model]; ok { - return alias + if ac, ok := ka[model]; ok { + return &ac } - // Fall back to case-insensitive lookup for consistency with WhiteList.Contains for k, v := range ka { if strings.EqualFold(k, model) { - return v + return &v } } - return model + return nil +} + +// UnmarshalJSON accepts both the legacy {"k":"v"} and new {"k":{...}} wire +// shapes for KeyAliases. Legacy string values are promoted to +// AliasConfig{ModelID: }. +func (ka *KeyAliases) UnmarshalJSON(data []byte) error { + trimmed := bytes.TrimSpace(data) + if len(trimmed) == 0 || string(trimmed) == "null" { + *ka = nil + return nil + } + var raw map[string]json.RawMessage + if err := sonic.Unmarshal(data, &raw); err != nil { + return err + } + result := make(KeyAliases, len(raw)) + for k, entry := range raw { + entryTrim := bytes.TrimSpace(entry) + if len(entryTrim) == 0 { + return fmt.Errorf("alias %q: empty value", k) + } + switch entryTrim[0] { + case '"': + // Legacy string value — promote to AliasConfig{ModelID: ...}. + var modelID string + if err := sonic.Unmarshal(entry, &modelID); err != nil { + return fmt.Errorf("alias %q: %w", k, err) + } + result[k] = AliasConfig{ModelID: modelID} + case '{': + var ac AliasConfig + if err := sonic.Unmarshal(entry, &ac); err != nil { + return fmt.Errorf("alias %q: %w", k, err) + } + result[k] = ac + default: + return fmt.Errorf("alias %q: value must be a string (legacy) or object", k) + } + } + *ka = result + return nil } type AzureAuthType string diff --git a/core/schemas/account_test.go b/core/schemas/account_test.go new file mode 100644 index 0000000000..ecd9653d01 --- /dev/null +++ b/core/schemas/account_test.go @@ -0,0 +1,482 @@ +package schemas + +import ( + "encoding/json" + "reflect" + "strings" + "testing" +) + +func TestKeyAliasesUnmarshalLegacyStringShape(t *testing.T) { + in := []byte(`{"best-model": "gpt-4o-deployment"}`) + var ka KeyAliases + if err := json.Unmarshal(in, &ka); err != nil { + t.Fatalf("unmarshal: %v", err) + } + want := KeyAliases{"best-model": AliasConfig{ModelID: "gpt-4o-deployment"}} + if !reflect.DeepEqual(ka, want) { + t.Fatalf("legacy shape mismatch: got %+v, want %+v", ka, want) + } +} + +func TestKeyAliasesUnmarshalRichShape(t *testing.T) { + // Provider sub-configs are embedded, so their fields appear at the top level of the JSON. + in := []byte(`{ + "best-model": { + "model_id": "azure-deployment-xyz", + "model_name": "claude-3-5-sonnet", + "model_family": "anthropic", + "description": "prod", + "api_version": "2024-08-01-preview" + } + }`) + var ka KeyAliases + if err := json.Unmarshal(in, &ka); err != nil { + t.Fatalf("unmarshal: %v", err) + } + got := ka["best-model"] + if got.ModelID != "azure-deployment-xyz" { + t.Fatalf("ModelID mismatch: %q", got.ModelID) + } + if got.ModelName == nil || *got.ModelName != "claude-3-5-sonnet" { + t.Fatalf("ModelName mismatch: %+v", got.ModelName) + } + if got.ModelFamily == nil || *got.ModelFamily != ModelFamilyAnthropic { + t.Fatalf("ModelFamily mismatch: %+v", got.ModelFamily) + } + if got.Description != "prod" { + t.Fatalf("Description mismatch: %q", got.Description) + } + if got.AzureAliasCfg == nil || got.APIVersion == nil || *got.APIVersion != "2024-08-01-preview" { + t.Fatalf("AzureAliasCfg.APIVersion mismatch: %+v", got.AzureAliasCfg) + } +} + +func TestKeyAliasesUnmarshalMixedShape(t *testing.T) { + in := []byte(`{ + "legacy": "gpt-4-deployment", + "rich": {"model_id": "azure-xyz", "model_family": "openai"} + }`) + var ka KeyAliases + if err := json.Unmarshal(in, &ka); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if got := ka["legacy"]; got.ModelID != "gpt-4-deployment" || got.ModelFamily != nil { + t.Fatalf("legacy entry wrong: %+v", got) + } + got := ka["rich"] + if got.ModelID != "azure-xyz" || got.ModelFamily == nil || *got.ModelFamily != ModelFamilyOpenAI { + t.Fatalf("rich entry wrong: %+v", got) + } +} + +func TestKeyAliasesUnmarshalEmptyAndNull(t *testing.T) { + cases := map[string]string{ + "empty-obj": `{}`, + "null": `null`, + } + for name, in := range cases { + var ka KeyAliases + if err := json.Unmarshal([]byte(in), &ka); err != nil { + t.Fatalf("%s: unmarshal: %v", name, err) + } + if len(ka) != 0 { + t.Fatalf("%s: want empty/nil, got %+v", name, ka) + } + } +} + +func TestKeyAliasesUnmarshalRoundTrip(t *testing.T) { + orig := KeyAliases{ + "best-model": AliasConfig{ + ModelID: "azure-xyz", + ModelName: Ptr("claude-3-5-sonnet"), + ModelFamily: Ptr(ModelFamilyAnthropic), + AzureAliasCfg: &AzureAliasCfg{ + APIVersion: Ptr("2024-08-01-preview"), + }, + }, + "simple": AliasConfig{ModelID: "gpt-4"}, + } + data, err := json.Marshal(orig) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var back KeyAliases + if err := json.Unmarshal(data, &back); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if !reflect.DeepEqual(orig, back) { + t.Fatalf("round-trip mismatch:\nwant: %+v\ngot: %+v", orig, back) + } +} + +func TestKeyAliasesMarshalLegacyShapeWhenOnlyModelIDSet(t *testing.T) { + // Only ModelID populated — should serialize to the legacy string-valued shape. + ka := KeyAliases{"best-model": AliasConfig{ModelID: "gpt-4o-deployment"}} + data, err := json.Marshal(ka) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if string(data) != `{"best-model":"gpt-4o-deployment"}` { + t.Fatalf("legacy shape mismatch: got %s", data) + } +} + +func TestKeyAliasesMarshalRichShapeWhenAnyExtraFieldSet(t *testing.T) { + cases := map[string]struct { + ac AliasConfig + wantKey string + wantValue any + }{ + "with_model_name": {AliasConfig{ModelID: "x", ModelName: Ptr("canonical")}, "model_name", "canonical"}, + "with_model_family": {AliasConfig{ModelID: "x", ModelFamily: Ptr(ModelFamilyAnthropic)}, "model_family", "anthropic"}, + "with_description": {AliasConfig{ModelID: "x", Description: "prod"}, "description", "prod"}, + "with_azure_subcfg": {AliasConfig{ModelID: "x", AzureAliasCfg: &AzureAliasCfg{APIVersion: Ptr("2024-08-01-preview")}}, "api_version", "2024-08-01-preview"}, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + data, err := json.Marshal(c.ac) + if err != nil { + t.Fatalf("marshal: %v", err) + } + // Rich shape should be a JSON object, not a string. + if len(data) == 0 || data[0] != '{' { + t.Fatalf("want object shape, got %s", data) + } + var out map[string]any + if err := json.Unmarshal(data, &out); err != nil { + t.Fatalf("re-unmarshal: %v", err) + } + got, ok := out[c.wantKey] + if !ok { + t.Fatalf("expected key %q in serialized output, got %s", c.wantKey, data) + } + if got != c.wantValue { + t.Fatalf("field %q: want %v, got %v (raw: %s)", c.wantKey, c.wantValue, got, data) + } + }) + } +} + +func TestKeyAliasesMarshalUnmarshalLegacyRoundTrip(t *testing.T) { + // Legacy in → legacy out: byte-for-byte stable for the unenriched case. + in := []byte(`{"best-model":"gpt-4o-deployment"}`) + var ka KeyAliases + if err := json.Unmarshal(in, &ka); err != nil { + t.Fatalf("unmarshal: %v", err) + } + out, err := json.Marshal(ka) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if string(out) != string(in) { + t.Fatalf("round-trip drift:\n in: %s\nout: %s", in, out) + } +} + +func TestKeyAliasesUnmarshalInvalidValueType(t *testing.T) { + for _, in := range []string{ + `{"k": 123}`, + `{"k": [1,2]}`, + `{"k": true}`, + } { + var ka KeyAliases + if err := json.Unmarshal([]byte(in), &ka); err == nil { + t.Fatalf("expected error for %q, got nil", in) + } + } +} + +func TestKeyAliasesResolveBackwardCompat(t *testing.T) { + ka := KeyAliases{ + "best-model": AliasConfig{ModelID: "gpt-4o-deployment"}, + } + if got := ka.Resolve("best-model"); got != "gpt-4o-deployment" { + t.Fatalf("Resolve mismatch: %q", got) + } + if got := ka.Resolve("BEST-MODEL"); got != "gpt-4o-deployment" { + t.Fatalf("Resolve case-insensitive fallback failed: %q", got) + } + if got := ka.Resolve("unmapped"); got != "unmapped" { + t.Fatalf("Resolve unmatched mismatch: %q", got) + } + var nilKA KeyAliases + if got := nilKA.Resolve("x"); got != "x" { + t.Fatalf("nil Resolve mismatch: %q", got) + } +} + +func TestKeyAliasesResolveConfig(t *testing.T) { + ka := KeyAliases{ + "best-model": AliasConfig{ModelID: "azure-xyz", ModelFamily: Ptr(ModelFamilyAnthropic)}, + } + got := ka.ResolveConfig("best-model") + if got == nil || got.ModelID != "azure-xyz" || got.ModelFamily == nil || *got.ModelFamily != ModelFamilyAnthropic { + t.Fatalf("ResolveConfig mismatch: %+v", got) + } + if ka.ResolveConfig("unmapped") != nil { + t.Fatalf("ResolveConfig should return nil for unmapped") + } +} + +func TestKeyAliasesValidate(t *testing.T) { + madeUp := ModelFamily("made-up") + azureCfg := &AzureAliasCfg{APIVersion: Ptr("2024-08-01-preview")} + bedrockCfg := &BedrockAliasCfg{InferenceProfileARN: NewEnvVar("arn:aws:bedrock:...")} + vertexCfg := &VertexAliasCfg{ProjectID: NewEnvVar("my-gcp-project")} + replicateCfg := &ReplicateAliasCfg{UseDeploymentsEndpoint: Ptr(true)} + cases := []struct { + name string + provider ModelProvider + ka KeyAliases + wantErr string + }{ + { + name: "ok", + provider: OpenAI, + ka: KeyAliases{"k": {ModelID: "v"}}, + }, + { + name: "empty source", + provider: OpenAI, + ka: KeyAliases{"": {ModelID: "v"}}, + wantErr: "alias source cannot be empty", + }, + { + name: "empty model id", + provider: OpenAI, + ka: KeyAliases{"k": {ModelID: ""}}, + wantErr: "model_id cannot be empty", + }, + { + name: "whitespace source", + provider: OpenAI, + ka: KeyAliases{" k ": {ModelID: "v"}}, + wantErr: "leading or trailing whitespace", + }, + { + name: "whitespace model_id", + ka: KeyAliases{"k": {ModelID: "v "}}, + wantErr: "model_id cannot have leading or trailing whitespace", + }, + { + name: "whitespace model_name", + ka: KeyAliases{"k": {ModelID: "v", ModelName: Ptr(" canonical ")}}, + wantErr: "model_name cannot have leading or trailing whitespace", + }, + { + name: "duplicate source case-insensitive", + ka: KeyAliases{"Key": {ModelID: "v"}, "key": {ModelID: "v"}}, + wantErr: "duplicate alias source", + }, + { + name: "invalid family", + provider: OpenAI, + ka: KeyAliases{"k": {ModelID: "v", ModelFamily: &madeUp}}, + wantErr: "invalid model_family", + }, + { + name: "azure sub-config on azure key — ok", + provider: Azure, + ka: KeyAliases{"k": {ModelID: "v", AzureAliasCfg: azureCfg}}, + }, + { + name: "azure sub-config on non-azure key — error", + provider: Bedrock, + ka: KeyAliases{"k": {ModelID: "v", AzureAliasCfg: azureCfg}}, + wantErr: "azure sub-config is only valid on Azure keys", + }, + { + name: "bedrock sub-config on bedrock key — ok", + provider: Bedrock, + ka: KeyAliases{"k": {ModelID: "v", BedrockAliasCfg: bedrockCfg}}, + }, + { + name: "bedrock sub-config on azure key — error", + provider: Azure, + ka: KeyAliases{"k": {ModelID: "v", BedrockAliasCfg: bedrockCfg}}, + wantErr: "bedrock sub-config is only valid on Bedrock keys", + }, + { + name: "vertex sub-config on vertex key — ok", + provider: Vertex, + ka: KeyAliases{"k": {ModelID: "v", VertexAliasCfg: vertexCfg}}, + }, + { + name: "vertex sub-config on openai key — error", + provider: OpenAI, + ka: KeyAliases{"k": {ModelID: "v", VertexAliasCfg: vertexCfg}}, + wantErr: "vertex sub-config is only valid on Vertex keys", + }, + { + name: "replicate sub-config on replicate key — ok", + provider: Replicate, + ka: KeyAliases{"k": {ModelID: "v", ReplicateAliasCfg: replicateCfg}}, + }, + { + name: "replicate sub-config on bedrock key — error", + provider: Bedrock, + ka: KeyAliases{"k": {ModelID: "v", ReplicateAliasCfg: replicateCfg}}, + wantErr: "replicate sub-config is only valid on Replicate keys", + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + err := c.ka.Validate(c.provider) + if c.wantErr == "" { + if err != nil { + t.Fatalf("want ok, got %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), c.wantErr) { + t.Fatalf("want error containing %q, got %v", c.wantErr, err) + } + }) + } +} + +func TestResolveFamilyPrecedence(t *testing.T) { + familyOpenAI := ModelFamilyOpenAI + + // Helper to build a BifrostContext carrying a ResolvedAlias. + withAlias := func(ra *ResolvedAlias) *BifrostContext { + bc := NewBifrostContext(nil, NoDeadline) + if ra != nil { + bc.SetValue(BifrostContextKeyResolvedAlias, ra) + } + return bc + } + + cases := []struct { + name string + ra *ResolvedAlias + fallback string + want ModelFamily + }{ + { + name: "tier 1: explicit ModelFamily wins over everything", + ra: &ResolvedAlias{ + Key: "some-claude-name", + Config: &AliasConfig{ + ModelID: "opaque-id", + ModelFamily: &familyOpenAI, // wins despite name/key smelling like Claude + }, + }, + fallback: "claude-3-5-sonnet", + want: ModelFamilyOpenAI, + }, + { + name: "tier 2: ModelName substring when no explicit family", + ra: &ResolvedAlias{ + Key: "best-model", + Config: &AliasConfig{ + ModelID: "opaque-id", + ModelName: Ptr("claude-3-5-sonnet"), + }, + }, + fallback: "opaque-id", + want: ModelFamilyAnthropic, + }, + { + name: "tier 3: ModelID substring when name absent", + ra: &ResolvedAlias{ + Key: "best-model", + Config: &AliasConfig{ + ModelID: "us.anthropic.claude-3-5-sonnet-20241022-v2:0", + }, + }, + fallback: "best-model", + want: ModelFamilyAnthropic, + }, + { + name: "tier 4: alias key substring when nothing else hits — the legacy 'best-claude→opaque-deployment-id' case the refactor is specifically meant to fix", + ra: &ResolvedAlias{ + Key: "best-claude", + Config: &AliasConfig{ + ModelID: "12345-azure-deployment", + }, + }, + fallback: "12345-azure-deployment", + want: ModelFamilyAnthropic, + }, + { + name: "no alias matched: fall back to substring on fallbackModel — preserves pre-refactor behavior", + ra: nil, + fallback: "claude-3-5-sonnet", + want: ModelFamilyAnthropic, + }, + { + name: "no alias and no substring hit anywhere", + ra: nil, + fallback: "totally-unknown-model", + want: "", + }, + { + name: "explicit empty ModelFamily pointer is treated as absent (falls through to name)", + ra: &ResolvedAlias{ + Key: "x", + Config: &AliasConfig{ + ModelID: "opaque-id", + ModelName: Ptr("claude-3-5-sonnet"), + ModelFamily: Ptr(ModelFamily("")), + }, + }, + fallback: "x", + want: ModelFamilyAnthropic, + }, + { + name: "first matching candidate wins (ModelName matches Anthropic before ModelID could match anything else)", + ra: &ResolvedAlias{ + Key: "x", + Config: &AliasConfig{ + ModelID: "mistral-large-2407", // would match Mistral but ModelName is checked first + ModelName: Ptr("claude-3-5-sonnet"), + }, + }, + fallback: "x", + want: ModelFamilyAnthropic, + }, + { + name: "uses fallback when ResolvedAlias.Config is nil (defensive)", + ra: &ResolvedAlias{Key: "x", Config: nil}, + // With Config==nil the candidates list is empty for the alias branch, + // so we drop to fallback substring matching. + fallback: "claude-3-haiku", + want: ModelFamilyAnthropic, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := ResolveFamily(withAlias(c.ra), c.fallback) + if got != c.want { + t.Fatalf("ResolveFamily: got %q, want %q", got, c.want) + } + }) + } +} + +func TestModelFamilyIsValid(t *testing.T) { + valid := []ModelFamily{ + ModelFamilyAnthropic, ModelFamilyOpenAI, ModelFamilyMistral, + ModelFamilyCohere, ModelFamilyGemini, ModelFamilyNova, ModelFamilyTitan, + } + for _, mf := range valid { + v := mf + if !v.IsValid() { + t.Fatalf("%q should be valid", mf) + } + } + for _, mf := range []ModelFamily{"", "unknown", "claude"} { + v := mf + if v.IsValid() { + t.Fatalf("%q should be invalid", mf) + } + } + // nil receiver is invalid. + var nilMF *ModelFamily + if nilMF.IsValid() { + t.Fatal("nil ModelFamily should be invalid") + } +} diff --git a/core/schemas/batch.go b/core/schemas/batch.go index 0f7bdd2f6c..92c4f32adf 100644 --- a/core/schemas/batch.go +++ b/core/schemas/batch.go @@ -79,6 +79,7 @@ type BifrostBatchCreateRequest struct { OutputFolder *BatchOutputFolder `json:"output_folder,omitempty"` // Common fields + DisplayName *string `json:"display_name,omitempty"` // Human-readable job name (e.g. Vertex displayName) Endpoint BatchEndpoint `json:"endpoint,omitempty"` // Target endpoint for batch requests CompletionWindow string `json:"completion_window,omitempty"` // Time window (e.g., "24h") Metadata map[string]string `json:"metadata,omitempty"` // User-provided metadata @@ -106,7 +107,8 @@ func (request *BifrostBatchCreateRequest) GetRawRequestBody() []byte { // BifrostBatchCreateResponse represents the response from creating a batch job. type BifrostBatchCreateResponse struct { ID string `json:"id"` - Object string `json:"object,omitempty"` // "batch" for OpenAI + Object string `json:"object,omitempty"` // "batch" for OpenAI + DisplayName *string `json:"display_name,omitempty"` // Human-readable job name (e.g. Vertex displayName) Endpoint string `json:"endpoint,omitempty"` InputFileID string `json:"input_file_id,omitempty"` CompletionWindow string `json:"completion_window,omitempty"` @@ -188,6 +190,7 @@ func (request *BifrostBatchRetrieveRequest) GetRawRequestBody() []byte { type BifrostBatchRetrieveResponse struct { ID string `json:"id"` Object string `json:"object,omitempty"` + DisplayName *string `json:"display_name,omitempty"` // Human-readable job name (e.g. Vertex displayName) Endpoint string `json:"endpoint,omitempty"` InputFileID string `json:"input_file_id,omitempty"` CompletionWindow string `json:"completion_window,omitempty"` diff --git a/core/schemas/bifrost.go b/core/schemas/bifrost.go index 8cdcb8470c..57f64891d2 100644 --- a/core/schemas/bifrost.go +++ b/core/schemas/bifrost.go @@ -228,6 +228,7 @@ const ( BifrostContextKeyGovernanceIncludeOnlyKeys BifrostContextKey = "bf-governance-include-only-keys" // []string (to store the include-only key IDs for provider config routing (set by bifrost governance plugin - DO NOT SET THIS MANUALLY)) BifrostContextKeyNumberOfRetries BifrostContextKey = "bifrost-number-of-retries" // int (to store the number of retries (set by bifrost - DO NOT SET THIS MANUALLY)) BifrostContextKeyFallbackIndex BifrostContextKey = "bifrost-fallback-index" // int (to store the fallback index (set by bifrost - DO NOT SET THIS MANUALLY)) 0 for primary, 1 for first fallback, etc. + BifrostContextKeyResolvedAlias BifrostContextKey = "bifrost-resolved-alias" // *ResolvedAlias (set by bifrost after key-level alias resolution — providers read this for model_family routing and provider-specific overrides; nil/absent when no alias matched) BifrostContextKeyStreamEndIndicator BifrostContextKey = "bifrost-stream-end-indicator" // bool (set by bifrost - DO NOT SET THIS MANUALLY)) BifrostContextKeyStreamIdleTimeout BifrostContextKey = "bifrost-stream-idle-timeout" // time.Duration (per-chunk idle timeout for streaming) BifrostContextKeySkipKeySelection BifrostContextKey = "bifrost-skip-key-selection" // bool (will pass an empty key to the provider) @@ -264,8 +265,6 @@ const ( BifrostContextKeyGovernanceRateLimitIDs BifrostContextKey = "bifrost-governance-rate-limit-ids" // []string (rate limit IDs applicable to this request - set by governance plugin) BifrostContextKeyPromptsPluginName BifrostContextKey = "prompts-plugin-name" // string (name of the prompts plugin to use - set by bifrost - DO NOT SET THIS MANUALLY)) BifrostContextKeyIsEnterprise BifrostContextKey = "is-enterprise" // bool (set by bifrost - DO NOT SET THIS MANUALLY)) - BifrostContextKeyAvailableProviders BifrostContextKey = "available-providers" // []ModelProvider (set by bifrost - DO NOT SET THIS MANUALLY)) - BifrostContextKeyResolvedProvider BifrostContextKey = "bifrost-resolved-provider" // ModelProvider (set by routing - DO NOT SET THIS MANUALLY)) BifrostContextKeyStoreRawRequestResponse BifrostContextKey = "bifrost-store-raw-request-response" // bool (per-request override — read by bifrost.go, never overwritten) BifrostContextKeyCaptureRawRequest BifrostContextKey = "bifrost-capture-raw-request" // bool (set by bifrost - DO NOT SET THIS MANUALLY) — true when providers should capture raw request bytes BifrostContextKeyCaptureRawResponse BifrostContextKey = "bifrost-capture-raw-response" // bool (set by bifrost - DO NOT SET THIS MANUALLY) — true when providers should capture raw response bytes @@ -290,6 +289,8 @@ const ( BifrostContextKeyRealtimeVoice BifrostContextKey = "bifrost-realtime-voice" // string BifrostIsAsyncRequest BifrostContextKey = "bifrost-is-async-request" // bool (set by bifrost - DO NOT SET THIS MANUALLY)) - whether the request is an async request (only used in gateway) BifrostContextKeyRequestHeaders BifrostContextKey = "bifrost-request-headers" // map[string]string (all request headers with lowercased keys) + BifrostContextKeyRequestQuery BifrostContextKey = "bifrost-request-query" // map[string]string (request query params with lowercased keys; consumed by governance routing CEL rules) + BifrostContextKeyRoutingAllowedProviders BifrostContextKey = "bifrost-routing-allowed-providers" // []ModelProvider; when set, downstream routing layers (enterprise LB, model-catalog-resolver) must intersect their candidate providers with this set. Plugins set this when they have an opinion about which providers are valid for the request — even if they couldn't pick one themselves. Empty slice means "no provider is permitted" (fail-closed). BifrostContextKeyAllowPerRequestStorageOverride BifrostContextKey = "bifrost-allow-per-request-storage-override" // bool (set by transport from config — gates whether x-bf-disable-content-logging and x-bf-store-raw-request-response per-request overrides are honored) BifrostContextKeyAllowPerRequestRawOverride BifrostContextKey = "bifrost-allow-per-request-raw-override" // bool (set by transport from config — gates whether x-bf-send-back-raw-request and x-bf-send-back-raw-response per-request overrides are honored) BifrostContextKeyDisableContentLogging BifrostContextKey = "x-bf-disable-content-logging" // bool (per-request override for content logging; only honored when BifrostContextKeyAllowPerRequestStorageOverride is true) @@ -303,39 +304,39 @@ const ( BifrostContextKeyIsAzureUserAgent BifrostContextKey = "bifrost-is-azure-user-agent" // bool (set by bifrost - DO NOT SET THIS MANUALLY)) - whether the request is an Azure user agent (only used in gateway) BifrostContextKeyUserRoleID BifrostContextKey = "bifrost-user-role-id" BifrostContextKeyVideoOutputRequested BifrostContextKey = "bifrost-video-output-requested" - BifrostContextKeyValidateKeys BifrostContextKey = "bifrost-validate-keys" // bool (triggers additional key validation during provider add/update) - BifrostContextKeyProviderResponseHeaders BifrostContextKey = "bifrost-provider-response-headers" // map[string]string (set by provider handlers for response header forwarding) - BifrostContextKeyMCPAddedTools BifrostContextKey = "bifrost-mcp-added-tools" // []string (set by bifrost - DO NOT SET THIS MANUALLY)) - list of tools added to the request by MCP, all the tool are in the format "clientName-toolName" - BifrostContextKeyLargePayloadMode BifrostContextKey = "bifrost-large-payload-mode" // bool (set by bifrost - DO NOT SET THIS MANUALLY)) indicates large payload streaming mode is active - BifrostContextKeyLargePayloadReader BifrostContextKey = "bifrost-large-payload-reader" // io.Reader (set by bifrost - DO NOT SET THIS MANUALLY)) upstream reader for large payloads - BifrostContextKeyLargePayloadContentLength BifrostContextKey = "bifrost-large-payload-content-length" // int (set by bifrost - DO NOT SET THIS MANUALLY)) content length for large payloads - BifrostContextKeyLargePayloadContentType BifrostContextKey = "bifrost-large-payload-content-type" // string (set by enterprise - DO NOT SET THIS MANUALLY)) original content type for large payload passthrough - BifrostContextKeyLargePayloadMetadata BifrostContextKey = "bifrost-large-payload-metadata" // *LargePayloadMetadata (set by bifrost - DO NOT SET THIS MANUALLY)) routing metadata for large payloads - BifrostContextKeyLargePayloadRequestThreshold BifrostContextKey = "bifrost-large-payload-request-threshold" // int64 (set by enterprise - DO NOT SET THIS MANUALLY)) request threshold used by transport heuristics - BifrostContextKeyLargeResponseMode BifrostContextKey = "bifrost-large-response-mode" // bool (set by bifrost - DO NOT SET THIS MANUALLY)) indicates large response streaming mode is active - BifrostContextKeyLargePayloadRequestPreview BifrostContextKey = "bifrost-large-payload-request-preview" // string (set by bifrost - DO NOT SET THIS MANUALLY)) truncated request body preview for logging - BifrostContextKeyLargePayloadResponsePreview BifrostContextKey = "bifrost-large-payload-response-preview" // string (set by bifrost - DO NOT SET THIS MANUALLY)) truncated response body preview for logging - BifrostContextKeyLargeResponseReader BifrostContextKey = "bifrost-large-response-reader" // io.ReadCloser (set by bifrost - DO NOT SET THIS MANUALLY)) upstream reader for large responses - BifrostContextKeyLargeResponseContentLength BifrostContextKey = "bifrost-large-response-content-length" // int (set by bifrost - DO NOT SET THIS MANUALLY)) content length for large responses - BifrostContextKeyLargeResponseContentType BifrostContextKey = "bifrost-large-response-content-type" // string (set by bifrost - DO NOT SET THIS MANUALLY)) upstream content type for large responses - BifrostContextKeyLargeResponseContentDisposition BifrostContextKey = "bifrost-large-response-content-disposition" // string (set by bifrost - DO NOT SET THIS MANUALLY)) downstream content disposition for large responses - BifrostContextKeyLargeResponseThreshold BifrostContextKey = "bifrost-large-response-threshold" // int64 (set by enterprise - DO NOT SET THIS MANUALLY)) threshold for response streaming - BifrostContextKeyLargePayloadPrefetchSize BifrostContextKey = "bifrost-large-payload-prefetch-size" // int (set by enterprise - DO NOT SET THIS MANUALLY)) prefetch buffer size for metadata extraction from large responses - BifrostContextKeyDeferredUsage BifrostContextKey = "bifrost-deferred-usage" // chan *BifrostLLMUsage (set by provider Phase B — delivers usage after response streaming completes) - BifrostContextKeyDeferredLargePayloadMetadata BifrostContextKey = "bifrost-deferred-large-payload-metadata" // <-chan *LargePayloadMetadata (set by enterprise Phase B request — delivers metadata after body streaming) - BifrostContextKeySSEReaderFactory BifrostContextKey = "bifrost-sse-reader-factory" // *providerUtils.SSEReaderFactory (set by enterprise — replaces default bufio.Scanner SSE readers with streaming readers) - BifrostContextKeySessionID BifrostContextKey = "bifrost-session-id" // string session ID for the request (session stickiness) - BifrostContextKeySessionTTL BifrostContextKey = "bifrost-session-ttl" // time.Duration session TTL for the request (session stickiness) - BifrostContextKeyMCPExtraHeaders BifrostContextKey = "bifrost-mcp-extra-headers" // map[string][]string (these headers are forwarded only to the MCP while tool execution if they are in the allowlist of the MCP client) - BifrostContextKeyMCPLogID BifrostContextKey = "bifrost-mcp-log-id" // string (unique UUID for each MCP tool log entry - set per goroutine by agent executor - DO NOT SET THIS MANUALLY) - BifrostContextKeyCompatConvertTextToChat BifrostContextKey = "bifrost-compat-convert-text-to-chat" // bool (per-request override from x-bf-compat header) - BifrostContextKeyCompatConvertChatToResponses BifrostContextKey = "bifrost-compat-convert-chat-to-responses" // bool (per-request override from x-bf-compat header) - BifrostContextKeyCompatShouldDropParams BifrostContextKey = "bifrost-compat-should-drop-params" // bool (per-request override from x-bf-compat header) - BifrostContextKeyCompatShouldConvertParams BifrostContextKey = "bifrost-compat-should-convert-params" // bool (per-request override from x-bf-compat header) - BifrostContextKeySupportsAssistantPrefill BifrostContextKey = "bifrost-supports-assistant-prefill" // bool (set by compat plugin) - if model supports assistant prefill - BifrostContextKeyAttemptTrail BifrostContextKey = "bifrost-attempt-trail" // []KeyAttemptRecord (set by bifrost - DO NOT SET THIS MANUALLY) - per-attempt key selection history - BifrostContextKeyDimensions BifrostContextKey = "bifrost-dimensions" // map[string]string (set by HTTP transport from x-bf-dim-* headers) BifrostContextKeyDimensions holds per-request key/value dimensions supplied via x-bf-dim- request headers. These dimensions are forwarded to internal logs (as metadata) - BifrostContextKeySkipModelCatalogProviderSelection BifrostContextKey = "bifrost-skip-model-catalog-provider-selection" // bool (set by bifrost - DO NOT SET THIS MANUALLY)) - skip model catalog provider selection + BifrostContextKeyValidateKeys BifrostContextKey = "bifrost-validate-keys" // bool (triggers additional key validation during provider add/update) + BifrostContextKeyProviderResponseHeaders BifrostContextKey = "bifrost-provider-response-headers" // map[string]string (set by provider handlers for response header forwarding) + BifrostContextKeyMCPAddedTools BifrostContextKey = "bifrost-mcp-added-tools" // []string (set by bifrost - DO NOT SET THIS MANUALLY)) - list of tools added to the request by MCP, all the tool are in the format "clientName-toolName" + BifrostContextKeyLargePayloadMode BifrostContextKey = "bifrost-large-payload-mode" // bool (set by bifrost - DO NOT SET THIS MANUALLY)) indicates large payload streaming mode is active + BifrostContextKeyLargePayloadReader BifrostContextKey = "bifrost-large-payload-reader" // io.Reader (set by bifrost - DO NOT SET THIS MANUALLY)) upstream reader for large payloads + BifrostContextKeyLargePayloadContentLength BifrostContextKey = "bifrost-large-payload-content-length" // int (set by bifrost - DO NOT SET THIS MANUALLY)) content length for large payloads + BifrostContextKeyLargePayloadContentType BifrostContextKey = "bifrost-large-payload-content-type" // string (set by enterprise - DO NOT SET THIS MANUALLY)) original content type for large payload passthrough + BifrostContextKeyLargePayloadMetadata BifrostContextKey = "bifrost-large-payload-metadata" // *LargePayloadMetadata (set by bifrost - DO NOT SET THIS MANUALLY)) routing metadata for large payloads + BifrostContextKeyLargePayloadRequestThreshold BifrostContextKey = "bifrost-large-payload-request-threshold" // int64 (set by enterprise - DO NOT SET THIS MANUALLY)) request threshold used by transport heuristics + BifrostContextKeyLargeResponseMode BifrostContextKey = "bifrost-large-response-mode" // bool (set by bifrost - DO NOT SET THIS MANUALLY)) indicates large response streaming mode is active + BifrostContextKeyLargePayloadRequestPreview BifrostContextKey = "bifrost-large-payload-request-preview" // string (set by bifrost - DO NOT SET THIS MANUALLY)) truncated request body preview for logging + BifrostContextKeyLargePayloadResponsePreview BifrostContextKey = "bifrost-large-payload-response-preview" // string (set by bifrost - DO NOT SET THIS MANUALLY)) truncated response body preview for logging + BifrostContextKeyLargeResponseReader BifrostContextKey = "bifrost-large-response-reader" // io.ReadCloser (set by bifrost - DO NOT SET THIS MANUALLY)) upstream reader for large responses + BifrostContextKeyLargeResponseContentLength BifrostContextKey = "bifrost-large-response-content-length" // int (set by bifrost - DO NOT SET THIS MANUALLY)) content length for large responses + BifrostContextKeyLargeResponseContentType BifrostContextKey = "bifrost-large-response-content-type" // string (set by bifrost - DO NOT SET THIS MANUALLY)) upstream content type for large responses + BifrostContextKeyLargeResponseContentDisposition BifrostContextKey = "bifrost-large-response-content-disposition" // string (set by bifrost - DO NOT SET THIS MANUALLY)) downstream content disposition for large responses + BifrostContextKeyLargeResponseThreshold BifrostContextKey = "bifrost-large-response-threshold" // int64 (set by enterprise - DO NOT SET THIS MANUALLY)) threshold for response streaming + BifrostContextKeyLargePayloadPrefetchSize BifrostContextKey = "bifrost-large-payload-prefetch-size" // int (set by enterprise - DO NOT SET THIS MANUALLY)) prefetch buffer size for metadata extraction from large responses + BifrostContextKeyDeferredUsage BifrostContextKey = "bifrost-deferred-usage" // chan *BifrostLLMUsage (set by provider Phase B — delivers usage after response streaming completes) + BifrostContextKeyDeferredLargePayloadMetadata BifrostContextKey = "bifrost-deferred-large-payload-metadata" // <-chan *LargePayloadMetadata (set by enterprise Phase B request — delivers metadata after body streaming) + BifrostContextKeySSEReaderFactory BifrostContextKey = "bifrost-sse-reader-factory" // *providerUtils.SSEReaderFactory (set by enterprise — replaces default bufio.Scanner SSE readers with streaming readers) + BifrostContextKeySessionID BifrostContextKey = "bifrost-session-id" // string session ID for the request (session stickiness) + BifrostContextKeySessionTTL BifrostContextKey = "bifrost-session-ttl" // time.Duration session TTL for the request (session stickiness) + BifrostContextKeyMCPExtraHeaders BifrostContextKey = "bifrost-mcp-extra-headers" // map[string][]string (these headers are forwarded only to the MCP while tool execution if they are in the allowlist of the MCP client) + BifrostContextKeyMCPLogID BifrostContextKey = "bifrost-mcp-log-id" // string (unique UUID for each MCP tool log entry - set per goroutine by agent executor - DO NOT SET THIS MANUALLY) + BifrostContextKeyMCPHealthCheckRequest BifrostContextKey = "bifrost-mcp-health-check-request" // bool (set by bifrost - DO NOT SET THIS MANUALLY) - true when the MCP ping/list-tools request was generated by bifrost itself for health checks rather than originating from a caller + BifrostContextKeyCompatConvertTextToChat BifrostContextKey = "bifrost-compat-convert-text-to-chat" // bool (per-request override from x-bf-compat header) + BifrostContextKeyCompatConvertChatToResponses BifrostContextKey = "bifrost-compat-convert-chat-to-responses" // bool (per-request override from x-bf-compat header) + BifrostContextKeyCompatShouldDropParams BifrostContextKey = "bifrost-compat-should-drop-params" // bool (per-request override from x-bf-compat header) + BifrostContextKeyCompatShouldConvertParams BifrostContextKey = "bifrost-compat-should-convert-params" // bool (per-request override from x-bf-compat header) + BifrostContextKeySupportsAssistantPrefill BifrostContextKey = "bifrost-supports-assistant-prefill" // bool (set by compat plugin) - if model supports assistant prefill + BifrostContextKeyAttemptTrail BifrostContextKey = "bifrost-attempt-trail" // []KeyAttemptRecord (set by bifrost - DO NOT SET THIS MANUALLY) - per-attempt key selection history + BifrostContextKeyDimensions BifrostContextKey = "bifrost-dimensions" // map[string]string (set by HTTP transport from x-bf-dim-* headers) BifrostContextKeyDimensions holds per-request key/value dimensions supplied via x-bf-dim- request headers. These dimensions are forwarded to internal logs (as metadata) IsAPIKeyAuthContextKey BifrostContextKey = "is_api_key_auth" IsLocalAdminContextKey BifrostContextKey = "is_local_admin" // bool (set by auth middleware when password-based auth succeeds - local admin user bypasses RBAC) BifrostContextKeyPassthroughOverridesPresent BifrostContextKey = "passthrough_overrides_present" // bool (set by HTTP transport) - passthrough raw request requested @@ -356,6 +357,12 @@ const ( RoutingEngineRoutingRule = "routing-rule" RoutingEngineLoadbalancing = "loadbalancing" RoutingEngineModelCatalog = "model-catalog" + // RoutingEngineCore represents the Bifrost core orchestrator's own + // routing decisions — primarily fallback transitions. Emitted when the + // primary attempt fails and core advances through the fallback chain so + // the per-request audit trail closes the loop on what plugin-level + // engines (governance, loadbalancing, etc.) selected upstream. + RoutingEngineCore = "core" ) // KeyAttemptRecord captures the outcome of a single request attempt within executeRequestWithRetries. @@ -1095,6 +1102,107 @@ func (r *BifrostResponse) GetExtraFields() *BifrostResponseExtraFields { return &BifrostResponseExtraFields{} } +// syncDeprecatedFromRoutingInfo backfills the deprecated Provider / +// OriginalModelRequested / ResolvedModelUsed triplet on an ExtraFields-like +// target from a finalized RoutingInfo, applying the rules documented on each +// deprecated field. Centralized so PopulateRoutingInfo and +// SetFallbackRoutingInfo cannot drift apart. +func syncDeprecatedFromRoutingInfo(info RoutingInfo, provider *ModelProvider, originalModelRequested, resolvedModelUsed *string) { + if provider != nil && info.Provider != "" { + *provider = info.Provider + } + // OriginalModelRequested: collapses to the caller-sent model. On a fallback + // attempt that's the primary's model (the user never asked for the fallback's); + // otherwise it's this attempt's model. + if originalModelRequested != nil { + if info.IsFallback && info.PrimaryModel != nil && *info.PrimaryModel != "" { + *originalModelRequested = *info.PrimaryModel + } else if info.Model != "" { + *originalModelRequested = info.Model + } + } + // ResolvedModelUsed: the wire model. Alias's ModelID when an alias matched, + // otherwise the attempt's Model. + if resolvedModelUsed != nil { + if info.ResolvedKeyAlias != nil && info.ResolvedKeyAlias.ModelID != "" { + *resolvedModelUsed = info.ResolvedKeyAlias.ModelID + } else if info.Model != "" { + *resolvedModelUsed = info.Model + } + } +} + +// PopulateRoutingInfo sets ExtraFields.RoutingInfo on the active sub-response +// and keeps the deprecated Provider/OriginalModelRequested/ResolvedModelUsed +// triplet in sync per their documented derivation rules. +// Core always calls this both before and after RunPostLLMHooks so any plugin +// modifications are no-ops — tampering with RoutingInfo inside plugins is +// discouraged. +func (r *BifrostResponse) PopulateRoutingInfo(info RoutingInfo) { + if r == nil { + return + } + if ef := r.GetExtraFields(); ef != nil { + ef.RoutingInfo = info + syncDeprecatedFromRoutingInfo(info, &ef.Provider, &ef.OriginalModelRequested, &ef.ResolvedModelUsed) + } +} + +// PopulateRoutingInfo sets ExtraFields.RoutingInfo on the error and syncs the +// deprecated triplet. Core calls this both before and after RunPostLLMHooks +// alongside PopulateExtraFields. +func (e *BifrostError) PopulateRoutingInfo(info RoutingInfo) { + if e == nil { + return + } + e.ExtraFields.RoutingInfo = info + syncDeprecatedFromRoutingInfo(info, &e.ExtraFields.Provider, &e.ExtraFields.OriginalModelRequested, &e.ExtraFields.ResolvedModelUsed) +} + +// SetFallbackRoutingInfo marks the active sub-response's RoutingInfo as a +// fallback attempt and records the primary attempt's provider/model. Also +// re-syncs the deprecated OriginalModelRequested to the primary model per +// its documented derivation rule. +// Called by the orchestrator (handleRequest) on each fallback attempt's +// result/error — the per-attempt code never sets these fields itself. +func (r *BifrostResponse) SetFallbackRoutingInfo(primaryProvider ModelProvider, primaryModel string) { + if r == nil { + return + } + ef := r.GetExtraFields() + if ef == nil { + return + } + ef.RoutingInfo.IsFallback = true + if primaryProvider != "" { + p := primaryProvider + ef.RoutingInfo.PrimaryProvider = &p + } + if primaryModel != "" { + m := primaryModel + ef.RoutingInfo.PrimaryModel = &m + } + syncDeprecatedFromRoutingInfo(ef.RoutingInfo, &ef.Provider, &ef.OriginalModelRequested, &ef.ResolvedModelUsed) +} + +// SetFallbackRoutingInfo is the BifrostError counterpart — see the +// BifrostResponse method for semantics. +func (e *BifrostError) SetFallbackRoutingInfo(primaryProvider ModelProvider, primaryModel string) { + if e == nil { + return + } + e.ExtraFields.RoutingInfo.IsFallback = true + if primaryProvider != "" { + p := primaryProvider + e.ExtraFields.RoutingInfo.PrimaryProvider = &p + } + if primaryModel != "" { + m := primaryModel + e.ExtraFields.RoutingInfo.PrimaryModel = &m + } + syncDeprecatedFromRoutingInfo(e.ExtraFields.RoutingInfo, &e.ExtraFields.Provider, &e.ExtraFields.OriginalModelRequested, &e.ExtraFields.ResolvedModelUsed) +} + // PopulateExtraFields sets RequestType, Provider, OriginalModelRequested, and ResolvedModelUsed on the // active sub-response. Core always calls this both before and after RunPostLLMHooks, so any plugin // modifications to these 4 fields are no-ops — tampering with them inside plugins is discouraged. @@ -1448,9 +1556,20 @@ func (r *BifrostMCPResponse) PopulateExtraFields(mcpRequestType MCPRequestType, // BifrostResponseExtraFields contains additional fields in a response. type BifrostResponseExtraFields struct { RequestType RequestType `json:"request_type"` - Provider ModelProvider `json:"provider,omitempty"` - OriginalModelRequested string `json:"original_model_requested,omitempty"` // the model alias the caller sent in the request - ResolvedModelUsed string `json:"resolved_model_used,omitempty"` // the actual provider API identifier used (equals OriginalModelRequested when no alias mapping exists) + RoutingInfo RoutingInfo `json:"routing_info"` + // Deprecated: use RoutingInfo.Provider. Still populated for backward + // compatibility; new consumers should read from RoutingInfo. + Provider ModelProvider `json:"provider,omitempty"` + // Deprecated: use RoutingInfo.PrimaryModel when RoutingInfo.IsFallback + // is true, otherwise RoutingInfo.Model — both branches collapse to the + // model string the caller sent in the request. Still populated for + // backward compatibility; new consumers should read from RoutingInfo. + OriginalModelRequested string `json:"original_model_requested,omitempty"` + // Deprecated: use RoutingInfo.ResolvedKeyAlias.ModelID when an alias + // matched (i.e. RoutingInfo.ResolvedKeyAlias != nil), otherwise + // RoutingInfo.Model. Still populated for backward compatibility; new + // consumers should read from RoutingInfo. + ResolvedModelUsed string `json:"resolved_model_used,omitempty"` Latency int64 `json:"latency"` // in milliseconds (for streaming responses this will be each chunk latency, and the last chunk latency will be the total latency) ChunkIndex int `json:"chunk_index"` // used for streaming responses to identify the chunk index, will be 0 for non-streaming responses RawRequest interface{} `json:"raw_request,omitempty"` @@ -1463,6 +1582,28 @@ type BifrostResponseExtraFields struct { PassthroughPath string `json:"passthrough_path,omitempty"` // Stripped provider path for passthrough requests, e.g. "/v1/chat/completions" } +type RoutingInfo struct { + // What actually handled this attempt + Provider ModelProvider `json:"provider,omitempty"` + Model string `json:"model,omitempty"` // model name passed to this attempt's key + Key string `json:"key,omitempty"` // KeyName of the key used + + // Populated only when Model matched an entry in this key's Aliases map + ResolvedKeyAlias *ResolvedKeyAlias `json:"resolved_key_alias,omitempty"` + + IsFallback bool `json:"is_fallback,omitempty"` + + // What the caller asked for, before any fallback resolution (populated only when fallback resolution occurred) + PrimaryProvider *ModelProvider `json:"primary_provider,omitempty"` + PrimaryModel *string `json:"primary_model,omitempty"` +} + +type ResolvedKeyAlias struct { + ModelID string `json:"model_id"` // wire model identifier actually sent to the provider + ModelName *string `json:"model_name,omitempty"` // canonical name (used for pricing/logs) + ModelFamily *ModelFamily `json:"model_family,omitempty"` // resolved family for routing +} + type BifrostMCPResponseExtraFields struct { MCPRequestType MCPRequestType `json:"mcp_request_type"` // request type this response corresponds to — lets PostMCPHook discriminate ping/list_tools from tool execute on success too ClientName string `json:"client_name"` @@ -1698,8 +1839,19 @@ func (e *ErrorField) UnmarshalJSON(data []byte) error { // BifrostErrorExtraFields contains additional fields in an error response. type BifrostErrorExtraFields struct { - Provider ModelProvider `json:"provider,omitempty"` - OriginalModelRequested string `json:"original_model_requested,omitempty"` + RoutingInfo RoutingInfo `json:"routing_info"` + // Deprecated: use RoutingInfo.Provider. Still populated for backward + // compatibility; new consumers should read from RoutingInfo. + Provider ModelProvider `json:"provider,omitempty"` + // Deprecated: use RoutingInfo.PrimaryModel when RoutingInfo.IsFallback + // is true, otherwise RoutingInfo.Model — both branches collapse to the + // model string the caller sent in the request. Still populated for + // backward compatibility; new consumers should read from RoutingInfo. + OriginalModelRequested string `json:"original_model_requested,omitempty"` + // Deprecated: use RoutingInfo.ResolvedKeyAlias.ModelID when an alias + // matched (i.e. RoutingInfo.ResolvedKeyAlias != nil), otherwise + // RoutingInfo.Model. Still populated for backward compatibility; new + // consumers should read from RoutingInfo. ResolvedModelUsed string `json:"resolved_model_used,omitempty"` RequestType RequestType `json:"request_type,omitempty"` MCPRequestType MCPRequestType `json:"mcp_request_type,omitempty"` diff --git a/core/schemas/chatcompletions.go b/core/schemas/chatcompletions.go index c6b483c16b..555b00d55b 100644 --- a/core/schemas/chatcompletions.go +++ b/core/schemas/chatcompletions.go @@ -40,6 +40,7 @@ type BifrostChatResponse struct { Model string `json:"model"` Object string `json:"object"` // "chat.completion" or "chat.completion.chunk" ServiceTier *BifrostServiceTier `json:"service_tier,omitempty"` + Speed *string `json:"speed,omitempty"` // "fast" | "standard" — speed actually served (Anthropic fast mode); drives fast-mode billing SystemFingerprint string `json:"system_fingerprint"` Usage *BifrostLLMUsage `json:"usage"` ExtraFields BifrostResponseExtraFields `json:"extra_fields"` diff --git a/core/schemas/context.go b/core/schemas/context.go index 5cf842e4b8..9f8e34695d 100644 --- a/core/schemas/context.go +++ b/core/schemas/context.go @@ -26,6 +26,7 @@ var reservedKeys = []any{ BifrostContextKeyURLPath, BifrostContextKeyDeferTraceCompletion, BifrostContextKeyAttemptTrail, + BifrostContextKeyMCPHealthCheckRequest, } // pluginLogStore holds plugin log entries accumulated during request processing. @@ -450,12 +451,11 @@ func (bc *BifrostContext) GetRoutingEngineLogs() []RoutingEngineLogEntry { return nil } -// AppendToContextList appends a value to the context list value. -// Parameters: -// - ctx: The Bifrost context -// - key: The key to append the value to -// - value: The value to append -func AppendToContextList[T any](ctx *BifrostContext, key BifrostContextKey, value T) { +// AppendToContextList appends value to the context list at key, skipping the +// append when value already exists in the list. Downstream consumers of these +// lists (notably `routing_engines_used` → Prometheus labels) treat duplicate +// entries as bugs, so set semantics are enforced at the write site. +func AppendToContextList[T comparable](ctx *BifrostContext, key BifrostContextKey, value T) { if ctx == nil { return } @@ -463,6 +463,9 @@ func AppendToContextList[T any](ctx *BifrostContext, key BifrostContextKey, valu if !ok { existingValues = []T{} } + if slices.Contains(existingValues, value) { + return + } ctx.SetValue(key, append(existingValues, value)) } diff --git a/core/schemas/files.go b/core/schemas/files.go index 073b236c0b..7bbecd71a7 100644 --- a/core/schemas/files.go +++ b/core/schemas/files.go @@ -19,11 +19,12 @@ const ( type FileStatus string const ( - FileStatusUploaded FileStatus = "uploaded" - FileStatusProcessed FileStatus = "processed" - FileStatusProcessing FileStatus = "processing" - FileStatusError FileStatus = "error" - FileStatusDeleted FileStatus = "deleted" + FileStatusUploaded FileStatus = "uploaded" + FileStatusProcessed FileStatus = "processed" + FileStatusProcessing FileStatus = "processing" + FileStatusPendingUpload FileStatus = "pending_upload" // resumable session minted, bytes not yet received (Vertex) + FileStatusError FileStatus = "error" + FileStatusDeleted FileStatus = "deleted" ) // FileStorageBackend represents the storage backend type. @@ -113,6 +114,10 @@ type BifrostFileUploadResponse struct { StorageBackend FileStorageBackend `json:"storage_backend,omitempty"` StorageURI string `json:"storage_uri,omitempty"` // S3/GCS URI if applicable + // GCS resumable upload session URL (Vertex only, set when File bytes are not provided). + // Client PUTs file bytes directly to this URL; Bifrost stays out of the data path. + UploadURL *string `json:"upload_url,omitempty"` + ExtraFields BifrostResponseExtraFields `json:"extra_fields"` } diff --git a/core/schemas/images.go b/core/schemas/images.go index fba3c2c08a..07347df4b2 100644 --- a/core/schemas/images.go +++ b/core/schemas/images.go @@ -213,6 +213,25 @@ type ImageTokenDetails struct { TextTokens int `json:"text_tokens,omitempty"` } +// DeepCopy returns an independent copy of u with no shared pointer fields, +// safe for callers (e.g. cost calculation) that need to derive values +// without mutating the original response. Returns nil for a nil receiver. +func (u *ImageUsage) DeepCopy() *ImageUsage { + if u == nil { + return nil + } + out := *u + if u.InputTokensDetails != nil { + details := *u.InputTokensDetails + out.InputTokensDetails = &details + } + if u.OutputTokensDetails != nil { + details := *u.OutputTokensDetails + out.OutputTokensDetails = &details + } + return &out +} + // Streaming Response type BifrostImageGenerationStreamResponse struct { ID string `json:"id,omitempty"` diff --git a/core/schemas/listmodels_test.go b/core/schemas/listmodels_test.go new file mode 100644 index 0000000000..b19529101b --- /dev/null +++ b/core/schemas/listmodels_test.go @@ -0,0 +1,128 @@ +package schemas + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestListModelsResponseMarshal_PreservesEnvelopeAndEnrichedPricing(t *testing.T) { + t.Parallel() + + model := Model{ + ID: "openai/gpt-5.5", + OwnedBy: Ptr("openai"), + RawModelJSON: json.RawMessage(`{"id":"gpt-5.5","object":"model","description":"Rich metadata model","supported_parameters":["tools"],"knowledge_cutoff":"2025-01"}`), + Pricing: &Pricing{ + Prompt: Ptr("0.000001"), + Completion: Ptr("0.000004"), + }, + } + + resp := BifrostListModelsResponse{ + Data: []Model{model}, + ExtraFields: BifrostResponseExtraFields{ + Provider: OpenAI, + Latency: 12, + }, + KeyStatuses: []KeyStatus{{ + KeyID: "key-1", + Status: KeyStatusSuccess, + Provider: OpenAI, + }}, + } + + payload, err := json.Marshal(resp) + require.NoError(t, err) + + var decoded map[string]any + require.NoError(t, json.Unmarshal(payload, &decoded)) + assert.Contains(t, decoded, "data") + assert.Contains(t, decoded, "extra_fields") + assert.Contains(t, decoded, "key_statuses") + + modelMap := decoded["data"].([]any)[0].(map[string]any) + assert.Equal(t, "openai/gpt-5.5", modelMap["id"]) + assert.Equal(t, "model", modelMap["object"]) + assert.Equal(t, "Rich metadata model", modelMap["description"]) + assert.Equal(t, "2025-01", modelMap["knowledge_cutoff"]) + + pricing := modelMap["pricing"].(map[string]any) + assert.Equal(t, "0.000001", pricing["prompt"]) + assert.Equal(t, "0.000004", pricing["completion"]) + + extraFields := decoded["extra_fields"].(map[string]any) + assert.Equal(t, "openai", extraFields["provider"]) + assert.Equal(t, float64(12), extraFields["latency"]) + + keyStatus := decoded["key_statuses"].([]any)[0].(map[string]any) + assert.Equal(t, "key-1", keyStatus["key_id"]) + assert.Equal(t, "success", keyStatus["status"]) + assert.Equal(t, "openai", keyStatus["provider"]) +} + +func TestParseListModelString_NormalizesKnownProviderCasing(t *testing.T) { + t.Parallel() + + provider, model := ParseListModelString("OpenAI/gpt-4o", "") + assert.Equal(t, OpenAI, provider) + assert.Equal(t, "gpt-4o", model) +} + +func TestModelUnmarshalJSON_NormalizesEmptyNestedStructsToNil(t *testing.T) { + t.Parallel() + + var model Model + err := json.Unmarshal([]byte(`{ + "id":"openai/gpt-5.5", + "pricing":{}, + "architecture":{}, + "top_provider":{}, + "per_request_limits":{}, + "default_parameters":{} + }`), &model) + require.NoError(t, err) + assert.Nil(t, model.Pricing) + assert.Nil(t, model.Architecture) + assert.Nil(t, model.TopProvider) + assert.Nil(t, model.PerRequestLimits) + assert.Nil(t, model.DefaultParameters) +} + +func TestModelMarshalJSON_DeepMergesNestedMetadata(t *testing.T) { + t.Parallel() + + model := Model{ + ID: "openai/gpt-5.5", + RawModelJSON: json.RawMessage(`{ + "id":"gpt-5.5", + "pricing":{"prompt":"0.1","input_cache_read":"0.02"}, + "top_provider":{"is_moderated":true,"provider_name":"openrouter"} + }`), + Pricing: &Pricing{ + Prompt: Ptr("0.3"), + Completion: Ptr("0.4"), + }, + TopProvider: &TopProvider{ + ContextLength: Ptr(4096), + }, + } + + payload, err := json.Marshal(model) + require.NoError(t, err) + + var decoded map[string]any + require.NoError(t, json.Unmarshal(payload, &decoded)) + + pricing := decoded["pricing"].(map[string]any) + assert.Equal(t, "0.3", pricing["prompt"]) + assert.Equal(t, "0.4", pricing["completion"]) + assert.Equal(t, "0.02", pricing["input_cache_read"]) + + topProvider := decoded["top_provider"].(map[string]any) + assert.Equal(t, true, topProvider["is_moderated"]) + assert.Equal(t, float64(4096), topProvider["context_length"]) + assert.Equal(t, "openrouter", topProvider["provider_name"]) +} diff --git a/core/schemas/models.go b/core/schemas/models.go index 36dbd971ee..05aeabcdfa 100644 --- a/core/schemas/models.go +++ b/core/schemas/models.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "reflect" ) // DefaultPageSize is the default page size for listing models @@ -51,6 +52,18 @@ type BifrostListModelsRequest struct { // Unfiltered: If true, the response will include all models for the provider, regardless of the allowed models (internal bifrost use only, not sent to the provider) Unfiltered bool `json:"-"` + // KeyID: If non-nil, scope the call to a single key (matched by Key.ID). + // Lets callers cache list-models output per-key for fine-grained + // invalidation. Internal bifrost use only; not sent to the provider. + // + // Matching runs against the already-filtered set of supported keys for the + // provider — keys that are disabled (Enabled == false) or fail validation + // are excluded before the lookup, so a KeyID referring to such a key + // produces the same "no key found" error as a KeyID that does not exist + // at all. Callers needing to distinguish those cases must check the raw + // account configuration themselves. + KeyID *string `json:"-"` + // ExtraParams: Additional provider-specific query parameters // This allows for flexibility to pass any custom parameters that specific providers might support ExtraParams map[string]interface{} `json:"-"` @@ -139,7 +152,7 @@ type Model struct { CanonicalSlug *string `json:"canonical_slug,omitempty"` Name *string `json:"name,omitempty"` NormalizedName *string `json:"normalized_name,omitempty"` // Human-readable name derived from the datasheet base_model (e.g. "Claude Sonnet 4.5") - Alias *string `json:"alias,omitempty"` // Provider API identifier this model alias maps to (e.g. Azure deployment name, Bedrock ARN) + Alias *string `json:"alias,omitempty"` // Provider API identifier this model alias maps to (e.g. Azure deployment name, Bedrock ARN) Created *int64 `json:"created,omitempty"` ContextLength *int `json:"context_length,omitempty"` MaxInputTokens *int `json:"max_input_tokens,omitempty"` @@ -161,11 +174,141 @@ type Model struct { OwnedBy *string `json:"owned_by,omitempty"` SupportedMethods []string `json:"supported_methods,omitempty"` + RawModelJSON json.RawMessage `json:"-"` + // ProviderExtra carries opaque provider-specific data (e.g. Anthropic capabilities) // through the Bifrost pipeline for integration reverse-conversion. Never serialized. ProviderExtra json.RawMessage `json:"-"` } +type modelAlias Model + +type modelUnmarshalAlias struct { + modelAlias + ContextWindow *int `json:"context_window,omitempty"` +} + +var nestedModelJSONKeys = map[string]struct{}{ + "architecture": {}, + "pricing": {}, + "top_provider": {}, + "per_request_limits": {}, + "default_parameters": {}, +} + +func nilIfZeroStruct[T any](value *T) *T { + if value == nil { + return nil + } + if reflect.ValueOf(*value).IsZero() { + return nil + } + return value +} + +func isJSONObject(raw json.RawMessage) bool { + var value map[string]json.RawMessage + return json.Unmarshal(raw, &value) == nil +} + +func isEmptyJSONObject(raw json.RawMessage) bool { + var value map[string]json.RawMessage + if err := json.Unmarshal(raw, &value); err != nil { + return false + } + return len(value) == 0 +} + +func mergeJSONObject(base, overlay json.RawMessage) (json.RawMessage, bool) { + var baseMap map[string]json.RawMessage + if err := json.Unmarshal(base, &baseMap); err != nil { + return nil, false + } + + var overlayMap map[string]json.RawMessage + if err := json.Unmarshal(overlay, &overlayMap); err != nil { + return nil, false + } + + for key, value := range overlayMap { + if existing, ok := baseMap[key]; ok && isJSONObject(existing) && isJSONObject(value) { + if merged, ok := mergeJSONObject(existing, value); ok { + baseMap[key] = merged + continue + } + } + baseMap[key] = value + } + + merged, err := json.Marshal(baseMap) + if err != nil { + return nil, false + } + return merged, true +} + +func (m *Model) UnmarshalJSON(data []byte) error { + var decoded modelUnmarshalAlias + if err := json.Unmarshal(data, &decoded); err != nil { + return err + } + + result := Model(decoded.modelAlias) + if result.ContextLength == nil && decoded.ContextWindow != nil { + result.ContextLength = decoded.ContextWindow + } + result.Architecture = nilIfZeroStruct(result.Architecture) + result.Pricing = nilIfZeroStruct(result.Pricing) + result.TopProvider = nilIfZeroStruct(result.TopProvider) + result.PerRequestLimits = nilIfZeroStruct(result.PerRequestLimits) + result.DefaultParameters = nilIfZeroStruct(result.DefaultParameters) + if len(data) > 0 { + result.RawModelJSON = append(json.RawMessage(nil), data...) + } + + *m = result + return nil +} + +func (m Model) MarshalJSON() ([]byte, error) { + alias := modelAlias(m) + if len(m.RawModelJSON) == 0 { + return json.Marshal(alias) + } + + merged := map[string]json.RawMessage{} + if err := json.Unmarshal(m.RawModelJSON, &merged); err != nil { + return json.Marshal(alias) + } + + overlayBytes, err := json.Marshal(alias) + if err != nil { + return nil, err + } + + var overlay map[string]json.RawMessage + if err := json.Unmarshal(overlayBytes, &overlay); err != nil { + return nil, err + } + + for key, value := range overlay { + if _, ok := nestedModelJSONKeys[key]; ok { + if existing, hasExisting := merged[key]; hasExisting { + if isEmptyJSONObject(value) { + continue + } + if mergedValue, ok := mergeJSONObject(existing, value); ok { + merged[key] = mergedValue + continue + } + } + } + merged[key] = value + } + + return json.Marshal(merged) +} + type Architecture struct { Modality *string `json:"modality,omitempty"` Tokenizer *string `json:"tokenizer,omitempty"` diff --git a/core/schemas/passthrough.go b/core/schemas/passthrough.go index ed743da386..12ef96508b 100644 --- a/core/schemas/passthrough.go +++ b/core/schemas/passthrough.go @@ -17,6 +17,7 @@ type BifrostPassthroughUsage struct { // Text / chat / responses / embeddings LLMUsage *BifrostLLMUsage ServiceTier *BifrostServiceTier // "priority" | "flex" | nil (default) + Speed *string // "fast" | "standard" — speed actually served (Anthropic fast mode); drives fast-mode billing // Image generation / edit / variation ImageUsage *ImageUsage diff --git a/core/schemas/plugin.go b/core/schemas/plugin.go index 7ca4a0ec68..186af928b7 100644 --- a/core/schemas/plugin.go +++ b/core/schemas/plugin.go @@ -170,12 +170,22 @@ func ReleaseHTTPResponse(resp *HTTPResponse) { // PostHooks are executed in the reverse order of PreHooks. // // Execution order: -// 1. HTTPTransportPreHook (HTTP transport only, executed in registration order) -// 2. PreLLMHook (executed in registration order) -// 3. Provider call -// 4. PostLLMHook (executed in reverse order of PreHooks) -// 5. HTTPTransportPostHook (HTTP transport only, executed in reverse order) -// 5a. HTTPTransportStreamChunkHook (for streaming responses, called per-chunk in reverse order) +// 1. HTTPTransportPreHook (HTTP transport only, once per request, executed in registration order) +// 2. PreRequestHook (once per request, executed in registration order) +// 3. PreLLMHook (executed in registration order, runs again on each fallback attempt) +// 4. Provider call +// 5. PostLLMHook (executed in reverse order of PreHooks, runs on each fallback attempt) +// 6. HTTPTransportPostHook (HTTP transport only, once per request, executed in reverse order) +// 6a. HTTPTransportStreamChunkHook (for streaming responses, called per-chunk in reverse order) +// +// Per-request vs per-attempt phases: +// - HTTPTransportPreHook, PreRequestHook, HTTPTransportPostHook run ONCE per top-level request. +// - PreLLMHook, PostLLMHook run ONCE PER ATTEMPT: the primary provider call, plus once per +// fallback attempt. Mutations a PreLLMHook makes to the request only carry to later +// fallbacks where prepareFallbackRequest happens to share pointers (shallow copy) — +// visibility across fallbacks is incidental. PreRequestHook is the explicit phase whose +// mutations are committed to the request before any fan-out and are observed by every +// subsequent plugin, every PreLLMHook invocation, the provider call, and every fallback. // // Common use cases: rate limiting, caching, logging, monitoring, request transformation, governance. // @@ -257,6 +267,22 @@ type HTTPTransportPlugin interface { type LLMPlugin interface { BasePlugin + // PreRequestHook is called once per top-level request, after HTTPTransportPreHook and before + // PreLLMHook. It is the canonical phase for deciding which provider/model/fallbacks the + // request should be sent to. Plugins are free to mutate any field on req (Provider, Model, + // Fallbacks, Input, Params, Tools, ...) — unlike PreLLMHook, mutations made here are + // committed to the request and are observed by all subsequent plugins, the provider call, + // and every fallback attempt. + // + // Error semantics match PreLLMHook: a non-nil error is non-blocking — it is logged as a + // warning, the request continues, and the pipeline moves on to the next plugin. PreRequestHook + // CANNOT abort the request via error return. Plugins that need to gate or reject a request + // (e.g., authorization, content policy) must do so in HTTPTransportPreHook or via a + // short-circuit response in PreLLMHook — not by returning an error here. + // + // Plugins that don't participate in routing should return nil. + PreRequestHook(ctx *BifrostContext, req *BifrostRequest) error + PreLLMHook(ctx *BifrostContext, req *BifrostRequest) (*BifrostRequest, *LLMPluginShortCircuit, error) PostLLMHook(ctx *BifrostContext, resp *BifrostResponse, bifrostErr *BifrostError) (*BifrostResponse, *BifrostError, error) } diff --git a/core/schemas/responses.go b/core/schemas/responses.go index 6b987608ce..f9545f5c8e 100644 --- a/core/schemas/responses.go +++ b/core/schemas/responses.go @@ -129,6 +129,7 @@ type BifrostResponsesResponse struct { Reasoning *ResponsesParametersReasoning `json:"reasoning"` // Configuration options for reasoning models SafetyIdentifier *string `json:"safety_identifier"` // Safety identifier ServiceTier *BifrostServiceTier `json:"service_tier"` + Speed *string `json:"speed,omitempty"` // "fast" | "standard" — speed actually served (Anthropic fast mode); drives fast-mode billing Status *string `json:"status,omitempty"` // completed, failed, in_progress, cancelled, queued, or incomplete StreamOptions *ResponsesStreamOptions `json:"stream_options,omitempty"` StopReason *string `json:"stop_reason,omitempty"` // Not in OpenAI's spec, but sent by other providers diff --git a/core/schemas/span_filter.go b/core/schemas/span_filter.go new file mode 100644 index 0000000000..d3f92f9e3c --- /dev/null +++ b/core/schemas/span_filter.go @@ -0,0 +1,124 @@ +package schemas + +import ( + "fmt" + "slices" + "strings" +) + +// PluginSpanFilterMode controls whether the plugins list is an allowlist or denylist. +type PluginSpanFilterMode string + +const ( + // PluginSpanFilterModeInclude exports only the listed plugins' spans. + PluginSpanFilterModeInclude PluginSpanFilterMode = "include" + // PluginSpanFilterModeExclude exports everything except the listed plugins' spans. + PluginSpanFilterModeExclude PluginSpanFilterMode = "exclude" +) + +// PluginSpanFilter configures which plugin spans an observability connector exports. +// Mode "include" exports only the listed plugins; mode "exclude" exports everything +// except them. It is shared by every observability connector (OTEL, Datadog, BigQuery) +// so the span-name contract and reparenting behavior stay consistent across exporters. +type PluginSpanFilter struct { + Mode PluginSpanFilterMode `json:"mode"` + Plugins []string `json:"plugins"` +} + +// Validate reports whether the filter's mode is one of the two valid modes. +// A nil filter is valid (it filters nothing). +func (f *PluginSpanFilter) Validate() error { + if f == nil { + return nil + } + switch f.Mode { + case PluginSpanFilterModeInclude, PluginSpanFilterModeExclude: + return nil + default: + return fmt.Errorf("plugin_span_filter.mode %q is invalid: must be %q or %q", + f.Mode, PluginSpanFilterModeInclude, PluginSpanFilterModeExclude) + } +} + +// SanitizePluginSpanName normalizes a plugin's name into the form embedded in its +// span names. +func SanitizePluginSpanName(name string) string { + return strings.ToLower(strings.ReplaceAll(name, " ", "-")) +} + +// PluginNameFromSpan extracts "" from a plugin span whose name follows the +// core tracer contract "plugin..", where is one of prehook, +// posthook, prerequesthook, mcp_prehook, mcp_posthook, mcp_connect_prehook, or mcp_connect_posthook +// (see core/bifrost.go). It returns "" for non-plugin spans or names that don't match +// the contract (wrong prefix, or fewer than three segments), so malformed names pass +// through ShouldExportSpan as exported rather than being silently filtered. +// +// The segment is intentionally not constrained to a fixed list: the tracer +// emits several hook stages (including the mcp_* variants above), so pinning it to +// just prehook/posthook would make every MCP-hook span unfilterable. +func PluginNameFromSpan(span *Span) string { + if span == nil || span.Kind != SpanKindPlugin { + return "" + } + parts := strings.SplitN(span.Name, ".", 3) + if len(parts) != 3 || parts[0] != "plugin" || parts[1] == "" { + return "" + } + return parts[1] +} + +// ShouldExportSpan reports whether a span survives the filter. Non-plugin spans and +// spans evaluated against a nil filter are always exported. Plugin spans are checked +// against the filter's plugin list and mode. +func (f *PluginSpanFilter) ShouldExportSpan(span *Span) bool { + if f == nil || span == nil || span.Kind != SpanKindPlugin { + return true + } + pluginName := PluginNameFromSpan(span) + if pluginName == "" { + // Malformed plugin span name: export rather than silently drop. + return true + } + inList := slices.Contains(f.Plugins, pluginName) + if f.Mode == PluginSpanFilterModeInclude { + return inList + } + return !inList // exclude mode +} + +// BuildReparentMap returns a map of filteredSpanID → effective ancestor spanID for all +// spans that the filter removes. When plugin spans are chained (each span's parent is the +// previous plugin's span), removing a span from the middle would leave its children with a +// dangling parent ID. The map lets callers rewrite those parent IDs to the nearest exported +// ancestor, handling consecutive filtered spans in a chain. Returns nil when the filter is +// nil or nothing is filtered. +func (f *PluginSpanFilter) BuildReparentMap(spans []*Span) map[string]string { + if f == nil { + return nil + } + // First pass: record direct parent ID for every filtered span. + filtered := make(map[string]string) // spanID -> parentID + for _, span := range spans { + if !f.ShouldExportSpan(span) { + filtered[span.SpanID] = span.ParentID + } + } + if len(filtered) == 0 { + return nil + } + // Second pass: resolve chains so each filtered span maps to its first exported ancestor. + // Cap the walk at len(filtered) to break out of any cycle caused by malformed span data. + maxHops := len(filtered) + for spanID := range filtered { + parentID := filtered[spanID] + for range maxHops { + grandParentID, isFiltered := filtered[parentID] + if !isFiltered { + break + } + parentID = grandParentID + } + filtered[spanID] = parentID + } + return filtered +} diff --git a/core/schemas/span_filter_test.go b/core/schemas/span_filter_test.go new file mode 100644 index 0000000000..19e262b727 --- /dev/null +++ b/core/schemas/span_filter_test.go @@ -0,0 +1,179 @@ +package schemas + +import "testing" + +func pluginSpan(id, parent, name string) *Span { + return &Span{SpanID: id, ParentID: parent, Name: name, Kind: SpanKindPlugin} +} + +func TestPluginSpanFilter_Validate(t *testing.T) { + tests := []struct { + name string + filter *PluginSpanFilter + wantErr bool + }{ + {"nil filter", nil, false}, + {"include", &PluginSpanFilter{Mode: PluginSpanFilterModeInclude}, false}, + {"exclude", &PluginSpanFilter{Mode: PluginSpanFilterModeExclude}, false}, + {"invalid mode", &PluginSpanFilter{Mode: "nonsense"}, true}, + {"empty mode", &PluginSpanFilter{Mode: ""}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := tt.filter.Validate(); (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestPluginNameFromSpan(t *testing.T) { + tests := []struct { + name string + span *Span + want string + }{ + {"prehook", pluginSpan("1", "", "plugin.logging.prehook"), "logging"}, + {"posthook", pluginSpan("1", "", "plugin.compat.posthook"), "compat"}, + {"mcp hook stage still resolves", pluginSpan("1", "", "plugin.governance.mcp_connect_prehook"), "governance"}, + {"non-plugin kind", &Span{Name: "plugin.logging.prehook", Kind: SpanKindLLMCall}, ""}, + {"malformed name", pluginSpan("1", "", "plugin"), ""}, + {"missing stage", pluginSpan("1", "", "plugin.logging"), ""}, + {"wrong prefix", pluginSpan("1", "", "otel.logging.prehook"), ""}, + {"empty name segment", pluginSpan("1", "", "plugin..prehook"), ""}, + {"nil span", nil, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := PluginNameFromSpan(tt.span); got != tt.want { + t.Errorf("PluginNameFromSpan() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestSanitizePluginSpanName(t *testing.T) { + tests := []struct { + in string + want string + }{ + {"logging", "logging"}, + {"enterprise-prompts", "enterprise-prompts"}, + {"Model Catalog Resolver", "model-catalog-resolver"}, + {"UPPER", "upper"}, + {"", ""}, + } + for _, tt := range tests { + if got := SanitizePluginSpanName(tt.in); got != tt.want { + t.Errorf("SanitizePluginSpanName(%q) = %q, want %q", tt.in, got, tt.want) + } + } +} + +// TestSanitizedNameMatchesSpanExtraction locks the invariant that the name used to build a +// plugin span (SanitizePluginSpanName(GetName())) is exactly what PluginNameFromSpan extracts +// back out. If these ever diverge, the UI's filterable-plugin list stops matching real spans +// and span filtering silently no-ops — the bug this contract exists to prevent. +func TestSanitizedNameMatchesSpanExtraction(t *testing.T) { + pluginNames := []string{"logging", "enterprise-prompts", "adaptive-loadbalancer", "Has Spaces", "MixedCase"} + stages := []string{"prehook", "posthook", "prerequesthook", "mcp_prehook", "mcp_connect_posthook"} + for _, raw := range pluginNames { + sanitized := SanitizePluginSpanName(raw) + for _, stage := range stages { + spanName := "plugin." + sanitized + "." + stage + if got := PluginNameFromSpan(pluginSpan("1", "", spanName)); got != sanitized { + t.Errorf("PluginNameFromSpan(%q) = %q, want %q", spanName, got, sanitized) + } + } + } +} + +func TestPluginSpanFilter_ShouldExportSpan(t *testing.T) { + llm := &Span{SpanID: "llm", Name: "llm.call", Kind: SpanKindLLMCall} + logging := pluginSpan("p1", "", "plugin.logging.prehook") + compat := pluginSpan("p2", "", "plugin.compat.prehook") + + tests := []struct { + name string + filter *PluginSpanFilter + span *Span + want bool + }{ + {"nil filter exports plugin", nil, logging, true}, + {"non-plugin always exported", &PluginSpanFilter{Mode: PluginSpanFilterModeInclude, Plugins: []string{"logging"}}, llm, true}, + {"include lists plugin", &PluginSpanFilter{Mode: PluginSpanFilterModeInclude, Plugins: []string{"logging"}}, logging, true}, + {"include omits plugin", &PluginSpanFilter{Mode: PluginSpanFilterModeInclude, Plugins: []string{"logging"}}, compat, false}, + {"exclude lists plugin", &PluginSpanFilter{Mode: PluginSpanFilterModeExclude, Plugins: []string{"logging"}}, logging, false}, + {"exclude omits plugin", &PluginSpanFilter{Mode: PluginSpanFilterModeExclude, Plugins: []string{"logging"}}, compat, true}, + {"malformed plugin span exported", &PluginSpanFilter{Mode: PluginSpanFilterModeInclude, Plugins: []string{"logging"}}, pluginSpan("p3", "", "plugin"), true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.filter.ShouldExportSpan(tt.span); got != tt.want { + t.Errorf("ShouldExportSpan() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPluginSpanFilter_BuildReparentMap(t *testing.T) { + t.Run("nil filter returns nil", func(t *testing.T) { + f := (*PluginSpanFilter)(nil) + if got := f.BuildReparentMap([]*Span{pluginSpan("1", "", "plugin.logging.prehook")}); got != nil { + t.Errorf("expected nil, got %v", got) + } + }) + + t.Run("nothing filtered returns nil", func(t *testing.T) { + f := &PluginSpanFilter{Mode: PluginSpanFilterModeExclude, Plugins: []string{"absent"}} + spans := []*Span{pluginSpan("1", "", "plugin.logging.prehook")} + if got := f.BuildReparentMap(spans); got != nil { + t.Errorf("expected nil, got %v", got) + } + }) + + t.Run("single filtered span maps to its parent", func(t *testing.T) { + // root(llm) <- logging <- compat. Exclude logging only. + f := &PluginSpanFilter{Mode: PluginSpanFilterModeExclude, Plugins: []string{"logging"}} + spans := []*Span{ + {SpanID: "root", Name: "llm.call", Kind: SpanKindLLMCall}, + pluginSpan("logging", "root", "plugin.logging.prehook"), + pluginSpan("compat", "logging", "plugin.compat.prehook"), + } + got := f.BuildReparentMap(spans) + if got["logging"] != "root" { + t.Errorf("logging should reparent to root, got %q", got["logging"]) + } + }) + + t.Run("chain of filtered spans resolves to first exported ancestor", func(t *testing.T) { + // root(llm) <- a <- b <- c. Exclude a and b. c should reparent to root. + f := &PluginSpanFilter{Mode: PluginSpanFilterModeExclude, Plugins: []string{"a", "b"}} + spans := []*Span{ + {SpanID: "root", Name: "llm.call", Kind: SpanKindLLMCall}, + pluginSpan("a", "root", "plugin.a.prehook"), + pluginSpan("b", "a", "plugin.b.prehook"), + pluginSpan("c", "b", "plugin.c.prehook"), + } + got := f.BuildReparentMap(spans) + if got["a"] != "root" { + t.Errorf("a should resolve to root, got %q", got["a"]) + } + if got["b"] != "root" { + t.Errorf("b should resolve to root, got %q", got["b"]) + } + if _, ok := got["c"]; ok { + t.Errorf("c is exported and should not be in the map") + } + }) + + t.Run("cycle is bounded and does not hang", func(t *testing.T) { + // Malformed: a's parent is b, b's parent is a. Both filtered. + f := &PluginSpanFilter{Mode: PluginSpanFilterModeExclude, Plugins: []string{"a", "b"}} + spans := []*Span{ + pluginSpan("a", "b", "plugin.a.prehook"), + pluginSpan("b", "a", "plugin.b.prehook"), + } + _ = f.BuildReparentMap(spans) // must terminate + }) +} diff --git a/core/schemas/utils.go b/core/schemas/utils.go index 6ab1b9a3a7..d1fe95e963 100644 --- a/core/schemas/utils.go +++ b/core/schemas/utils.go @@ -105,6 +105,26 @@ func ParseModelString(model string, defaultProvider ModelProvider) (ModelProvide return defaultProvider, model } +func ParseListModelString(model string, defaultProvider ModelProvider) (ModelProvider, string) { + provider, parsedModel := ParseModelString(model, defaultProvider) + if !strings.Contains(model, "/") { + return provider, parsedModel + } + if provider != defaultProvider || parsedModel != model { + return provider, parsedModel + } + + parts := strings.SplitN(model, "/", 2) + if len(parts) == 2 { + normalizedProvider := strings.ToLower(parts[0]) + if IsKnownProvider(normalizedProvider) { + return ModelProvider(normalizedProvider), parts[1] + } + } + + return provider, parsedModel +} + // IsAllDigitsASCII checks if a string contains only ASCII digits (0-9). func IsAllDigitsASCII(s string) bool { if s == "" { @@ -1433,6 +1453,19 @@ func IsImagenModel(model string) bool { return strings.Contains(strings.ToLower(model), "imagen") } +// IsCohereModel checks if the model is a Cohere model. Matches the Bedrock +// identifier prefix ("cohere.embed-*", "cohere.command-*") which is the wire +// shape that flows through alias resolution. +func IsCohereModel(model string) bool { + return strings.Contains(model, "cohere") +} + +// IsTitanModel checks if the model is an Amazon Titan model. Matches the +// Bedrock identifier prefix ("amazon.titan-*"). +func IsTitanModel(model string) bool { + return strings.Contains(model, "titan") +} + // List of grok reasoning models var grokReasoningModels = []string{ "grok-3", diff --git a/core/utils.go b/core/utils.go index 3993b2dd11..b978b223c4 100644 --- a/core/utils.go +++ b/core/utils.go @@ -135,17 +135,17 @@ func calculateBackoff(attempt int, config *schemas.ProviderConfig) time.Duration return min(result, config.NetworkConfig.RetryBackoffMax) } -// validateRequest validates the given request. -func validateRequest(req *schemas.BifrostRequest) *schemas.BifrostError { +// validateRequestAfterPreRequestHooks validates the provider and model fields of the given request. +func validateRequestAfterPreRequestHooks(req *schemas.BifrostRequest) *schemas.BifrostError { if req == nil { return newBifrostErrorFromMsg("bifrost request cannot be nil") } provider, model, _ := req.GetRequestFields() if provider == "" { - return newBifrostErrorFromMsg("provider is required") + return newBifrostErrorFromMsg("could not auto resolve a provider for the request, please specify a provider explicitly") } if isModelRequired(req.RequestType) && model == "" { - return newBifrostErrorFromMsg("model is required") + return newBifrostErrorFromMsg("could not auto resolve a model for the request, please specify a model explicitly") } return nil } @@ -214,6 +214,32 @@ func IsRateLimitErrorMessage(errorMessage string) bool { return false } +// routingErrorSummary produces a sanitized, audit-safe one-line summary of a +// BifrostError for emission to the per-request routing engine log trail. +// It deliberately omits the upstream provider message — which can echo back +// API keys, tokens, or user input — and surfaces only the error type and HTTP +// status code. Used by the core fallback orchestrator so the routing log +// records *why* a fallback was triggered without leaking secrets into log +// storage or the UI. +func routingErrorSummary(e *schemas.BifrostError) string { + if e == nil { + return "unknown error" + } + parts := make([]string, 0, 2) + if e.Error != nil && e.Error.Type != nil && *e.Error.Type != "" { + parts = append(parts, *e.Error.Type) + } else if e.Type != nil && *e.Type != "" { + parts = append(parts, *e.Type) + } + if e.StatusCode != nil { + parts = append(parts, fmt.Sprintf("HTTP %d", *e.StatusCode)) + } + if len(parts) == 0 { + return "request failed" + } + return strings.Join(parts, " ") +} + // newBifrostError wraps a standard error into a BifrostError with IsBifrostError set to false. // This helper function reduces code duplication when handling non-Bifrost errors. func newBifrostError(err error) *schemas.BifrostError { @@ -414,6 +440,18 @@ func GetResponseFields(result *schemas.BifrostResponse, err *schemas.BifrostErro return } +// GetResponseRoutingInfo extracts the RoutingInfo recorded on a completed +// attempt — from the accumulated response, or the error when the attempt failed. +func GetResponseRoutingInfo(result *schemas.BifrostResponse, err *schemas.BifrostError) schemas.RoutingInfo { + if result != nil { + return result.GetExtraFields().RoutingInfo + } + if err != nil { + return err.ExtraFields.RoutingInfo + } + return schemas.RoutingInfo{} +} + // MarshalUnsafe marshals the given value to a JSON string without escaping HTML characters. // Returns empty string if marshaling fails. func MarshalUnsafe(v any) string { @@ -519,9 +557,9 @@ func ValidateExternalURL(urlStr string, allowPrivateNetwork bool) error { return nil } -// sanitizeSpanName sanitizes a span name to remove capital letters and spaces to make it a valid span name +// sanitizeSpanName sanitizes a span name to remove capital letters and spaces to make it a valid span name. func sanitizeSpanName(name string) string { - return strings.ToLower(strings.ReplaceAll(name, " ", "-")) + return schemas.SanitizePluginSpanName(name) } // IsCodemodeTool returns true if the given tool name is a codemode tool. diff --git a/docs/architecture/core/plugins.mdx b/docs/architecture/core/plugins.mdx index 4a901272ee..479a4ad5ed 100644 --- a/docs/architecture/core/plugins.mdx +++ b/docs/architecture/core/plugins.mdx @@ -68,8 +68,12 @@ Every plugin goes through a well-defined lifecycle that ensures proper resource stateDiagram-v2 [*] --> PluginInit: Plugin Creation PluginInit --> Registered: Add to BifrostConfig - Registered --> PreHookCall: Request Received + Registered --> PreRequestHookCall: Request Received (once per request) + PreRequestHookCall --> RouteDecided: Provider/Model resolved + PreRequestHookCall --> RouteDecided: Return Error (logged, non-blocking) + + RouteDecided --> PreHookCall: Per-attempt phase PreHookCall --> ModifyRequest: Normal Flow PreHookCall --> ShortCircuitResponse: Return Response PreHookCall --> ShortCircuitError: Return Error @@ -87,7 +91,7 @@ stateDiagram-v2 FallbackCheck --> TryFallback: AllowFallbacks=true/nil FallbackCheck --> ResponseReady: AllowFallbacks=false - TryFallback --> PreHookCall: Next Provider + TryFallback --> PreHookCall: Next Provider (PreRequestHook NOT re-run) ModifyResponse --> ResponseReady: Modified RecoverError --> ResponseReady: Recovered @@ -148,6 +152,12 @@ sequenceDiagram participant Provider Client->>Bifrost: Request + Note over Bifrost,Plugin2: PreRequestHook phase (once per request, before any fan-out) + Bifrost->>Plugin1: PreRequestHook(request) + Plugin1-->>Bifrost: routed request + Bifrost->>Plugin2: PreRequestHook(request) + Plugin2-->>Bifrost: routed request + Note over Bifrost,Plugin2: PreLLMHook phase (per provider attempt) Bifrost->>Plugin1: PreLLMHook(request) Plugin1-->>Bifrost: modified request Bifrost->>Plugin2: PreLLMHook(request) @@ -163,9 +173,19 @@ sequenceDiagram **Execution Order:** -1. **PreHooks:** Execute in registration order (1 → 2 → N) -2. **Provider Call:** If no short-circuit occurred -3. **PostHooks:** Execute in reverse order (N → 2 → 1) +1. **PreRequestHooks** (per-request, registration order 1 → 2 → N): the **routing phase**. Plugins decide which provider/model the request goes to. Mutations to `req.Provider`/`req.Model`/`req.Fallbacks` commit to the shared request and are observed by every subsequent phase and every fallback attempt. There is no short-circuit. Plugin errors are non-blocking — logged as warnings and the pipeline continues to the next plugin. After all PreRequestHooks have run, the core validates `req.Provider`: an unresolved provider returns a 400 to the caller. +2. **PreLLMHooks** (per attempt, registration order 1 → 2 → N): pre-call transforms — caching, validation, content modification. May short-circuit with a synthetic response. +3. **Provider Call:** if no short-circuit occurred. +4. **PostLLMHooks** (per attempt, reverse order N → 2 → 1): response transforms — error recovery, logging, observability. + +**Per-request vs per-attempt:** `PreRequestHook` runs **exactly once** at the top of `handleRequest` / `handleStreamRequest`, before any provider call. `PreLLMHook` and `PostLLMHook` run **once per provider attempt** — so if the primary call fails and a fallback fires, `PreLLMHook` and `PostLLMHook` run again on the fallback, but `PreRequestHook` does **not**. This is what makes `PreRequestHook` the right place for routing decisions: the decision is committed once and applies uniformly to the primary attempt and every fallback. + + +**When to use which hook:** +- **PreRequestHook** → routing decisions (governance rules, load balancing, model-catalog provider resolution). Mutations to `req.Provider`/`req.Model`/`req.Fallbacks` stick. +- **PreLLMHook** → per-attempt transforms (semantic-cache lookups, request validation, content rewrites). Mutations to provider/model are intentionally no-ops here. +- **PostLLMHook** → per-attempt response handling (caching writes, logging, error recovery). + #### **Short-Circuit Response Flow (Cache Hit)** @@ -178,6 +198,7 @@ sequenceDiagram participant Provider Client->>Bifrost: Request + Note over Bifrost,Cache: PreRequestHook phase (routing decided) Bifrost->>Auth: PreLLMHook(request) Auth-->>Bifrost: modified request Bifrost->>Cache: PreLLMHook(request) @@ -203,6 +224,7 @@ sequenceDiagram participant Provider Client->>Bifrost: Stream Request + Note over Bifrost,Plugin2: PreRequestHook phase (routing decided) Bifrost->>Plugin1: PreLLMHook(request) Plugin1-->>Bifrost: modified request Bifrost->>Plugin2: PreLLMHook(request) diff --git a/docs/deployment-guides/config-json/client.mdx b/docs/deployment-guides/config-json/client.mdx index 532d9e7ac4..f5193222fd 100644 --- a/docs/deployment-guides/config-json/client.mdx +++ b/docs/deployment-guides/config-json/client.mdx @@ -101,6 +101,7 @@ This setting is also configurable via the UI (**MCP Gateway → MCP Settings**) | `max_request_body_size_mb` | integer | `100` | Maximum allowed request body size in MB | | `whitelisted_routes` | array of strings | `[]` | Routes that bypass auth middleware | | `allowed_headers` | array of strings | `[]` | Additional headers permitted for CORS and WebSocket | +| `allow_direct_keys` | boolean | `false` | Allow callers to bypass the registered key pool by sending `x-bf-direct-key: true` and a raw provider key in `Authorization` / `x-api-key` / `x-goog-api-key`. See [Direct API Key](../../providers/request-options#direct-api-key) | ```json { diff --git a/docs/deployment-guides/config-json/providers.mdx b/docs/deployment-guides/config-json/providers.mdx index f988637e73..33f5023b78 100644 --- a/docs/deployment-guides/config-json/providers.mdx +++ b/docs/deployment-guides/config-json/providers.mdx @@ -256,6 +256,10 @@ When `value` is empty or omitted, Bifrost uses `DefaultAzureCredential` - which } ``` + +`aliases` values can also be objects, not just plain strings. The object form lets you tag each alias with a canonical `model_name` (improves pricing/log attribution when the wire ID is opaque), a `model_family` for routing, and per-alias provider overrides like `api_version` or `endpoint`. See [Aliasing Models](/providers/aliasing-models) for the full schema. + + **Multi-region failover** (two keys, different regions): ```json diff --git a/docs/deployment-guides/config-json/schema-reference.mdx b/docs/deployment-guides/config-json/schema-reference.mdx index 0f0c975201..aae662e12f 100644 --- a/docs/deployment-guides/config-json/schema-reference.mdx +++ b/docs/deployment-guides/config-json/schema-reference.mdx @@ -62,6 +62,7 @@ Controls the worker pool, logging pipeline, security, and SDK shims. All fields | `logging_headers` | array | `[]` | HTTP headers to capture in log metadata | | `enforce_auth_on_inference` | boolean | `false` | Require a virtual key on every `/v1/*` request | | `allowed_origins` | array | `["*"]` | CORS allowed origins | +| `allow_direct_keys` | boolean | `false` | Let callers bypass the key pool with `x-bf-direct-key: true` + a raw provider key | | `max_request_body_size_mb` | integer | `100` | Maximum request body in MB | | `whitelisted_routes` | array | `[]` | Routes that bypass auth middleware | | `allowed_headers` | array | `[]` | Additional headers permitted for CORS/WebSocket | diff --git a/docs/deployment-guides/helm/governance.mdx b/docs/deployment-guides/helm/governance.mdx index bf63e0022a..162803385d 100644 --- a/docs/deployment-guides/helm/governance.mdx +++ b/docs/deployment-guides/helm/governance.mdx @@ -384,6 +384,33 @@ bifrost: --- +## Complexity Router Configuration + +If you use `complexity_tier` in routing rules, you can seed the analyzer thresholds and keyword lists from Helm. The chart renders this block to `governance.complexity_analyzer_config` in `config.json`. + +Omit this block, or leave `complexityAnalyzerConfig: null`, to use the built-in defaults. + +```yaml +bifrost: + governance: + complexityAnalyzerConfig: + tier_boundaries: + simple_medium: 0.15 + medium_complex: 0.35 + complex_reasoning: 0.60 + keywords: + code_keywords: ["function", "class", "api", "debug", "deploy"] + reasoning_keywords: ["step by step", "explain why", "tradeoffs", "root cause analysis"] + technical_keywords: ["architecture", "kubernetes", "latency", "authentication"] + simple_keywords: ["hello", "hi", "thanks", "what is", "define"] +``` + + +When this block is present, it is reapplied from the generated `config.json` on restart. Runtime UI and API edits still hot-reload immediately, but Helm values remain the source of truth for the next rollout. + + +--- + ## Full Example ```yaml diff --git a/docs/docs.json b/docs/docs.json index e8b3951ebf..c86d954dce 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -229,7 +229,8 @@ "features/governance/budget-and-limits", "features/governance/model-limits", "features/governance/mcp-tools", - "features/governance/required-headers" + "features/governance/required-headers", + "features/governance/complexity-router" ] }, "features/telemetry", diff --git a/docs/enterprise/datadog-connector.mdx b/docs/enterprise/datadog-connector.mdx index 47e38c4504..b4a22b69da 100644 --- a/docs/enterprise/datadog-connector.mdx +++ b/docs/enterprise/datadog-connector.mdx @@ -62,8 +62,8 @@ This mode requires an API key but simplifies deployment by eliminating the need |-------|------|----------|---------|-------------| | `service_name` | `string` | No | `bifrost` | Service name displayed in Datadog APM | | `ml_app` | `string` | No | (uses `service_name`) | ML application name for LLM Observability grouping | -| `agent_addr` | `string` | No | `localhost:8126` | Datadog Agent address (agent mode only) | -| `dogstatsd_addr` | `string` | No | `localhost:8125` | DogStatsD server address (agent mode only) | +| `agent_addr` | `string` | No | `localhost:8126` | Datadog Agent address (agent mode only, supports `env.VAR_NAME`) | +| `dogstatsd_addr` | `string` | No | `localhost:8125` | DogStatsD server address (agent mode only, supports `env.VAR_NAME`) | | `env` | `string` | No | - | Environment tag (e.g., `production`, `staging`) | | `version` | `string` | No | - | Service version tag | | `custom_tags` | `object` | No | - | Additional tags for all traces and metrics | @@ -71,16 +71,18 @@ This mode requires an API key but simplifies deployment by eliminating the need | `enable_traces` | `bool` | No | `true` | Enable APM traces | | `enable_llm_obs` | `bool` | No | `true` | Enable LLM Observability | | `agentless` | `bool` | No | `false` | Use agentless mode (direct API) | -| `api_key` | `EnvVar` | Agentless only | - | Datadog API key (supports `env.VAR_NAME`) | +| `api_key` | `string` | Agentless only | - | Datadog API key (supports `env.VAR_NAME`) | | `site` | `string` | No | `datadoghq.com` | Datadog site/region | ### Environment Variable Substitution -The `api_key` and `custom_tags` fields support environment variable substitution using the `env.` prefix: +The `api_key`, `agent_addr`, `dogstatsd_addr`, and `custom_tags` fields support environment variable substitution using the `env.` prefix: ```json { "api_key": "env.DD_API_KEY", + "agent_addr": "env.DD_AGENT_ADDR", + "dogstatsd_addr": "env.DD_DOGSTATSD_ADDR", "custom_tags": { "team": "env.TEAM_NAME", "cost_center": "env.COST_CENTER" @@ -354,11 +356,27 @@ The plugin emits the following metrics to Datadog: | `bifrost.tokens.input` | Counter | Input/prompt tokens consumed | provider, model | | `bifrost.tokens.output` | Counter | Output/completion tokens generated | provider, model | | `bifrost.tokens.total` | Counter | Total tokens (input + output) | provider, model | -| `bifrost.cost.usd` | Gauge | Request cost in USD | provider, model | +| `bifrost.request.cost.usd` | Distribution | Per-request cost in USD | provider, model | | `bifrost.cache.hits` | Counter | Cache hits | provider, model, cache_type | | `bifrost.stream.first_token_latency` | Histogram | Time to first token (streaming) | provider, model | | `bifrost.stream.inter_token_latency` | Histogram | Inter-token latency (streaming) | provider, model | +### Migrating from `bifrost.cost.usd` + +The cost metric was renamed from `bifrost.cost.usd` to `bifrost.request.cost.usd`, and its type changed from **Gauge** to **Distribution**. The gauge was last-write-wins per flush window, so concurrent requests with the same tags collapsed to a single value and no query could recover the true total spend. The new name is required because Datadog permanently associates a metric name with its first-seen type per organization — orgs that previously received the gauge cannot receive the same name as a distribution. + +**Affected assets:** any dashboards, monitors, saved views, or alerts that query `bifrost.cost.usd`. + +**To migrate:** + +1. Replace `bifrost.cost.usd` with `bifrost.request.cost.usd` in all queries. +2. Update aggregations for Distribution semantics — each sample is one request's cost: + - Total spend: `sum:bifrost.request.cost.usd{*}` (do **not** append `.as_count()` or `.rollup(sum)`; the `sum:` aggregator already returns the additive total) + - Per-request statistics: `avg:`, `max:`, or percentile aggregators +3. Recreate monitors and alerts on the new metric, adjusting thresholds if they assumed gauge behavior (the gauge systematically under-reported under concurrent load). + +`bifrost.cost.usd` stops receiving data once the upgrade completes; during a rolling deploy both metrics receive data, so update dashboards at or shortly after the upgrade. Historical gauge data remains queryable under the old name for Datadog's standard retention window. + ### Custom Tags All metrics include your configured `custom_tags` plus automatic tags for: @@ -366,6 +384,7 @@ All metrics include your configured `custom_tags` plus automatic tags for: - `model` - Model name - `request_type` - Type of request (chat, embedding, etc.) - `env` - Environment from configuration +- `bifrost_node` - Per-instance identity (`BIFROST_NODE_ID` if set, otherwise `hostname-pid`) --- @@ -412,6 +431,51 @@ Each APM trace includes comprehensive LLM operation metadata: --- +## Plugin Span Filtering + +By default every plugin's pre- and post-hook execution generates a span, which can bloat APM traces when many plugins are active (e.g. 8 built-in plugins × 2 hooks = 16 plugin spans per request). Use `plugin_span_filter` inside the Datadog plugin config to control which plugin spans are exported. This affects only the exported APM trace spans — plugin execution and metrics are unchanged. + +**Via config.json** (inside the Datadog plugin config): + +```json +{ + "plugins": [ + { + "name": "datadog", + "enabled": true, + "config": { + "service_name": "bifrost", + "agent_addr": "localhost:8126", + "enable_traces": true, + "plugin_span_filter": { + "mode": "exclude", + "plugins": ["logging", "compat", "telemetry"] + } + } + } + ] +} +``` + +**Via the UI**: Open the **Observability** page, select the **Datadog** connector, and click **Configure Plugin Tracing**. Toggle individual plugins on or off and save. UI-saved settings persist across restarts unless `plugin_span_filter` is set in config.json with a higher `version` value. + +**Filter modes:** + +| Mode | Behaviour | +|------|-----------| +| `exclude` | Export spans for all plugins **except** those listed | +| `include` | Export spans **only** for the listed plugins | + +**Plugin names:** list each plugin using the exact name shown for it in the **Configure Plugin Tracing** sheet — this is the same name that appears in the span (`plugin..`), and it is what the filter matches against. Note that some plugins are registered under a different name than their config key: the enterprise prompts and governance plugins appear as `enterprise-prompts` and `enterprise-governance` (not `prompts`/`governance`). Common names include `telemetry`, `logging`, `otel`, `semantic_cache`, `compat`, `maxim`, `enterprise-prompts`, `enterprise-governance`, `datadog`, `bigquery`, `guardrails`, `adaptive-loadbalancer`, and `model-catalog-resolver`. The exact set depends on which plugins are loaded in your deployment. + +When a plugin span is filtered out, its children are automatically re-parented to the nearest exported ancestor so the trace hierarchy stays connected. The filter applies to APM trace spans only; it does not change DogStatsD metrics, which are never derived from plugin spans. + + + Each observability connector has its own independent `plugin_span_filter` — filtering plugin spans for Datadog does not affect OTEL, BigQuery, or any other connector. `plugin_span_filter` follows the standard plugin config precedence rules; to make a config.json value override UI-saved DB settings on restart, set a higher `version` on the Datadog plugin entry (e.g. `"version": 2`). See [Plugin Versioning](/deployment-guides/config-json/plugins) for details. + + +--- + ## Supported Request Types The Datadog plugin captures all Bifrost request types: diff --git a/docs/features/governance/complexity-router.mdx b/docs/features/governance/complexity-router.mdx new file mode 100644 index 0000000000..fddb5f0138 --- /dev/null +++ b/docs/features/governance/complexity-router.mdx @@ -0,0 +1,410 @@ +--- +title: "Complexity Router" +description: "Automatically classify incoming LLM requests into complexity tiers and route them to the right model." +icon: "sliders" +--- + +## Overview + +The Complexity Router analyzes each incoming request and assigns it one of four tiers - **Simple**, **Medium**, **Complex**, or **Reasoning** - based on the content of the latest user message, conversation history, and system prompt. The result is exposed as a flat string variable (`complexity_tier`) in Bifrost's CEL routing engine, so you can write routing rules like: + +```cel +complexity_tier == "REASONING" +complexity_tier in ["COMPLEX", "REASONING"] +``` + +This lets you route simple greetings to a fast, cheap model and deep reasoning tasks to a frontier model — automatically, with no changes to your application code. The algorithm is fast and deterministic: it runs entirely in-process using pre-compiled keyword matching, adds less than 1 ms to request latency, and makes zero external calls. + +![Complexity Router Configuration](../../media/architecture-complexity-router.png) + +--- + +## How it works + +### Scoring dimensions + +Every request produces a score between 0.0 and 1.0. The analyzer starts with a weighted score across five dimensions detected by scanning the last user message: + +| Dimension | Weight | What it measures | +|---|---|---| +| Code presence | 30% | Code, debugging, and programming artifacts | +| Reasoning markers | 25% | Analytical and multi-step reasoning language | +| Technical terms | 25% | Architecture, infra, and operational terminology | +| Token count | 10% | Prompt length (longer → higher score) | +| Simple indicators | −5% | Greetings, trivial queries (dampener, subtracted) | + +The simple indicators dimension subtracts from the score - it acts as a dampener, not a floor. This means a short, conversational prompt like "hi, how are you?" can reach a near-zero score even if it technically contains other weak signals. The dampener is reduced to near-zero when the prompt is long (≥30 words) or contains two or more other strong signals, so it does not suppress genuinely complex requests. + +### System prompt contribution + +The system prompt is scanned for code, technical, and simple signals, and its contribution is weighted at **25% of the user-message signal** for those three dimensions. This provides soft lexical context — for example, a system prompt describing a coding assistant nudges code scores up — but it never drives the token count, reasoning markers, or tier override. + +### Conversation context blending + +For multi-turn conversations, the score blends the current message with history from up to the last 10 user turns (recency-weighted: earlier turns count less): + +- **Default blend:** 60% last message + 40% conversation history +- **Referential follow-up blend:** 35% last message + 65% conversation history + +A message is treated as a referential follow-up when it is short (≤6 words), contains phrases like "do it", "retry", "continue", or "go ahead", and the conversation history has a meaningful complexity score. In that case, the follow-up inherits most of its score from prior context rather than being classified as Simple on its own. + +The final score is `max(last_message_score, weighted_blend)` — the current message always sets a floor. + +### Output complexity floor + +Some requests are hard not because the reasoning is especially deep, but because the output being asked for is broad or exhaustive. Prompts like "list every AWS service and explain each one with examples" can receive a built-in score floor even when the normal weighted score is only moderate. + +The analyzer looks for internal markers such as exhaustive enumeration ("list every", "all possible"), comprehensiveness cues ("comprehensive", "in detail"), and elaboration asks ("explain each", "with examples"). Limiting qualifiers like "briefly", "top 5", or "keep it short" reduce this boost. + +This output-complexity floor is built in — it is not currently exposed as a user-configurable keyword list. + +### Reasoning override + +When two or more **reasoning keywords** are detected in the last user message, the tier is forced to **Reasoning** regardless of the numeric score. The same override applies when one strong reasoning keyword appears alongside strong code or technical signals. + +This handles prompts like "step by step, explain why the authentication flow fails" that would score moderately on each individual dimension but clearly require deep reasoning. + +### Tier classification + +The final score maps to a tier using configurable boundaries (defaults shown): + +| Tier | Score range (defaults) | Typical requests | +|---|---|---| +| Simple | < 0.15 | Greetings, definitions, simple lookups | +| Medium | 0.15 – 0.35 | General questions, short explanations | +| Complex | 0.35 – 0.60 | Technical questions, code help, multi-step tasks | +| Reasoning | ≥ 0.60 (or override) | Analysis, architecture decisions, root-cause investigation | + +![Complexity Analyzer Pipeline](../../media/complexity-logic-architecture.png) + +--- + +## Configuration + +### Tier boundaries + +Adjust where the score thresholds fall to match your traffic and model lineup. + + + + +Navigate to **Complexity Router** in the sidebar. + +The **Complexity Spectrum** bar updates live as you type boundary values, so you can see how your traffic would be distributed before saving. + +![Complexity Router Tier Configuration](../../media/ui-complexity-router-config.png) + +1. Enter a value between 0 and 1 for each boundary. +2. Boundaries must be strictly increasing: `simple_medium` < `medium_complex` < `complex_reasoning`. +3. Click **Save changes** to apply immediately (hot-reloaded, no restart required). +4. Click **Restore defaults** to reset all boundaries and keyword lists to factory values. + + + + +```bash +# Get current configuration +curl http://localhost:8080/api/governance/complexity-analyzer-config + +# Update tier boundaries +curl -X PUT http://localhost:8080/api/governance/complexity-analyzer-config \ + -H "Content-Type: application/json" \ + -d '{ + "tier_boundaries": { + "simple_medium": 0.15, + "medium_complex": 0.35, + "complex_reasoning": 0.60 + }, + "keywords": { + "code_keywords": ["function", "class", "api", "debug"], + "reasoning_keywords": ["step by step", "explain why", "tradeoffs"], + "technical_keywords": ["architecture", "kubernetes", "latency"], + "simple_keywords": ["hello", "hi", "thanks", "what is"] + } + }' + +# Reset to factory defaults +curl -X POST http://localhost:8080/api/governance/complexity-analyzer-config/reset +``` + +**Response (GET / PUT):** +```json +{ + "tier_boundaries": { + "simple_medium": 0.15, + "medium_complex": 0.35, + "complex_reasoning": 0.60 + }, + "keywords": { + "code_keywords": ["function", "class", "..."], + "reasoning_keywords": ["step by step", "..."], + "technical_keywords": ["architecture", "..."], + "simple_keywords": ["hello", "hi", "..."] + } +} +``` + + + + +```json +{ + "governance": { + "complexity_analyzer_config": { + "tier_boundaries": { + "simple_medium": 0.15, + "medium_complex": 0.35, + "complex_reasoning": 0.60 + }, + "keywords": { + "code_keywords": ["function", "class", "api", "debug", "deploy"], + "reasoning_keywords": ["step by step", "explain why", "tradeoffs", "root cause analysis"], + "technical_keywords": ["architecture", "kubernetes", "latency", "authentication"], + "simple_keywords": ["hello", "hi", "thanks", "what is", "define"] + } + } + } +} +``` + +| Field | Type | Required | Default | Description | +|---|---|---|---|---| +| `tier_boundaries.simple_medium` | number | Yes | 0.15 | Score threshold between Simple and Medium (exclusive: 0 < value < 1) | +| `tier_boundaries.medium_complex` | number | Yes | 0.35 | Score threshold between Medium and Complex | +| `tier_boundaries.complex_reasoning` | number | Yes | 0.60 | Score threshold between Complex and Reasoning | +| `keywords.code_keywords` | string[] | Yes | built-in defaults | Signals for code/debugging/programming requests | +| `keywords.reasoning_keywords` | string[] | Yes | built-in defaults | Strong reasoning triggers — matches can force the Reasoning tier | +| `keywords.technical_keywords` | string[] | Yes | built-in defaults | Architecture/infra/operations signals | +| `keywords.simple_keywords` | string[] | Yes | built-in defaults | Phrases that dampen the complexity score | + + +Each keyword list requires at least one entry. Keywords are normalized to lowercase and deduplicated on save. Changes are hot-reloaded with no restart required. + + + +If `governance.complexity_analyzer_config` is present in `config.json`, the file remains authoritative on restart. UI and API edits apply immediately, but the file values are re-applied on the next startup unless you update `config.json` too. + + + + + +### Keyword lists + +Each list controls a different part of the scoring signal. Understanding what they do helps you tune routing for your domain. + + +The default keyword lists are tuned for common request patterns and are a good starting point for most deployments. For domain-specific traffic, add or remove keywords based on the prompts your users actually send so the tiers match your routing strategy. + + + + + +![Complexity Router Keyword Lists](../../media/ui-complexity-router-keywords.png) + +Type a keyword or phrase and press **Enter** to add it. Click the × on any tag to remove it. The entry count is shown next to each list label. + + + + +| List | Effect | When to customize | +|---|---|---| +| **Code keywords** | Each match contributes to the Code dimension (30% weight) | Add domain-specific tooling, frameworks, or file types your users frequently mention | +| **Reasoning keywords** | Strong triggers — two or more matches, or one match alongside strong code/technical signals, forces the Reasoning tier regardless of score | Narrow this list to the phrases that truly demand your most capable model. The default list is intentionally conservative. | +| **Technical keywords** | Each match contributes to the Technical dimension (25% weight) | Add industry-specific jargon (e.g. "SOC 2", "HIPAA", "proration") relevant to your product | +| **Simple keywords** | Each match subtracts from the score (dampener) | Add domain-specific phrases that signal trivial intent in your app context | + + +The **reasoning keywords** list gates the tier-override path, not just scoring. Adding broad terms like "explain" or "analyze" will push many requests to Reasoning. Prefer specific multi-word phrases like "step by step" or "root cause analysis". + + + + + +--- + +## Routing with `complexity_tier` + +Once the analyzer is configured, use `complexity_tier` as a variable in any CEL routing rule expression. Bifrost evaluates it as a plain string. + +`complexity_tier` is not a special standalone rule type. In the Routing Rules builder, it behaves like any other field, so you can combine it with headers, request type, team/customer scope, budgets, and other predicates in the same rule or nested rule group. + + +Complexity Router only exposes `complexity_tier`; it does not create rules automatically. Add rules for the tiers you want to route. For deterministic four-tier routing, create rules for Simple, Medium, Complex, and Reasoning. + + +### Available operators + +| Operator | CEL syntax | Example | +|---|---|---| +| Equal | `==` | `complexity_tier == "REASONING"` | +| Not equal | `!=` | `complexity_tier != "SIMPLE"` | +| In list | `in` | `complexity_tier in ["COMPLEX", "REASONING"]` | +| Not in list | `!(x in [...])` | `!(complexity_tier in ["SIMPLE", "MEDIUM"])` | + + +### Combining with other rule conditions + +You can mix complexity with any other routing condition the CEL builder supports: + +```cel +headers["x-tier"] == "premium" && complexity_tier == "REASONING" +headers["x-region"] == "us-east" && complexity_tier in ["COMPLEX", "REASONING"] +request_type == "chat_completion" && complexity_tier != "SIMPLE" +team_name == "ml-research" && headers["x-env"] == "prod" && complexity_tier == "REASONING" +``` + +### Setting up a complexity-based routing rule + +The best first rollout is usually a single **Reasoning** rule. It is easy to validate, has the smallest blast radius, and leaves Simple, Medium, and Complex traffic on your existing routing path. + +1. Go to **Routing Rules** in the sidebar. +2. Create a new rule and open the CEL builder. +3. Add a condition: field = **Complexity Tier**, operator = **=**, value = **Reasoning**. +4. Set the target provider and model to your strongest reasoning model. +5. Save and enable the rule. + +Once you are happy with the classifications, add complementary rules for Simple, Medium, and Complex if you want a full tier-based routing ladder. An example is shown below. + +![Routing Rule with Complexity Tier](../../media/ui-routing-rule-complexity.png) + +### Use case examples + +#### Start with a Reasoning carve-out + +Route only frontier-worthy requests to your strongest model and let everything else keep using your existing routing: + +```json +{ + "id": "complexity-reasoning", + "name": "Reasoning → Frontier model", + "enabled": true, + "cel_expression": "complexity_tier == \"REASONING\"", + "targets": [{ "provider": "anthropic", "model": "claude-opus-4-5", "weight": 1 }], + "scope": "global", + "priority": 0 +} +``` + +#### Full four-tier ladder + +Route every tier explicitly when you want deterministic model selection across the full spectrum: + +```json +[ + { + "id": "complexity-simple", + "name": "Simple → Fast model", + "enabled": true, + "cel_expression": "complexity_tier == \"SIMPLE\"", + "targets": [{ "provider": "groq", "model": "llama-3.1-8b-instant", "weight": 1 }], + "scope": "global", + "priority": 0 + }, + { + "id": "complexity-medium", + "name": "Medium → Balanced model", + "enabled": true, + "cel_expression": "complexity_tier == \"MEDIUM\"", + "targets": [{ "provider": "openai", "model": "gpt-4o-mini", "weight": 1 }], + "scope": "global", + "priority": 1 + }, + { + "id": "complexity-complex", + "name": "Complex → Strong general model", + "enabled": true, + "cel_expression": "complexity_tier == \"COMPLEX\"", + "targets": [{ "provider": "anthropic", "model": "claude-sonnet-4-5", "weight": 1 }], + "scope": "global", + "priority": 2 + }, + { + "id": "complexity-reasoning", + "name": "Reasoning → Frontier model", + "enabled": true, + "cel_expression": "complexity_tier == \"REASONING\"", + "targets": [{ "provider": "anthropic", "model": "claude-opus-4-5", "weight": 1 }], + "scope": "global", + "priority": 3 + } +] +``` + +#### Roll out to one team first + +Test complexity routing with a single team before enabling it globally: + +```json +{ + "id": "team-reasoning-pilot", + "name": "Team pilot — reasoning route", + "enabled": true, + "cel_expression": "complexity_tier == \"REASONING\"", + "targets": [{ "provider": "anthropic", "model": "claude-opus-4-5", "weight": 1 }], + "scope": "team", + "scope_id": "team-uuid-456", + "priority": 0 +} +``` + +--- + +## Observability + +Complexity analysis is recorded in the routing log for every request where analysis ran. In the log detail view, look at the **Routing Decision Logs** section backed by `routing_engine_logs`. + +![Routing Logs with Complexity Tier](../../media/ui-routing-logs-complexity.png) + +You will see log lines like `Complexity: tier=REASONING score=0.38 words=25`. Logs use the emitted uppercase tier string, while the UI displays the same tier as normal-case text. This lets you audit how traffic is being distributed and spot mis-classifications to tune thresholds or keyword lists. + +--- + +## Troubleshooting + +### Rule not matching when complexity_tier is set + +If the routing rule uses `complexity_tier` and the request is not matching, make sure the request contains analyzable user text. A system prompt by itself is not enough — the analyzer needs a text-bearing user prompt to classify. + +If analysis is unavailable (for example the body could not be parsed, or the user content is not text-only), `complexity_tier` is treated as **unknown** by the CEL evaluator. The rule does not match and evaluation falls through to the next rule. This is intentional: complexity rules silently degrade rather than blocking requests. + +### Which request types are supported + +Complexity routing currently runs only for **text-bearing** request families. Supported inputs include: + +- Chat Completions and other messages-style requests with text-only user content +- Text Completions requests using `prompt` +- Responses API requests using text-only `input` +- Anthropic Messages, Bedrock Converse, and Gemini `contents` / `systemInstruction` shapes when they carry text-only user input + +It does **not** run for: + +- Image generation, embeddings, rerank, OCR, audio/speech/transcription, video, or count-tokens requests +- Chat or Responses requests where user content mixes text with image, file, or audio blocks +- Requests that contain only system or developer text and no user text + +### All traffic classified as Reasoning + +The **reasoning keywords** list is the most common cause. Check if any broad single-word terms were added (e.g. "explain", "analyze"). The override gate fires when two or more strong reasoning keywords match — a broad list will trigger it on most prompts. Replace single-word terms with specific multi-word phrases. + +### Want to test threshold changes without affecting live traffic + +Use the **Discard changes** button to revert unsaved edits, or **Restore defaults** to return to factory settings. Changes only take effect on save. + +--- + +## Next Steps + + + + Full reference for CEL expressions, scope hierarchy, and rule chaining + + + Scope complexity routing rules to specific teams, customers, or virtual keys + + + Combine complexity routing with budget limits for cost-optimal routing + + + Understand how complexity routing fits into the full request routing pipeline + + diff --git a/docs/features/governance/routing.mdx b/docs/features/governance/routing.mdx index 9eee41336d..d0436ef715 100644 --- a/docs/features/governance/routing.mdx +++ b/docs/features/governance/routing.mdx @@ -23,6 +23,10 @@ This powerful feature enables key use cases like: - **Cost Management**: Route traffic to cheaper models or providers based on weights to optimize costs. - **Fine-grained Access Control**: Ensure that different teams or applications only use the models and API keys they are explicitly permitted to. + +This page covers **static governance routing** through Virtual Key provider configuration. If you want routing decisions to depend on runtime request attributes such as headers, budgets, or request-content complexity, use [Routing Rules](/providers/routing-rules). For complexity-based tiering specifically, see [Complexity Router](/features/governance/complexity-router). + + ## Provider/Model Restrictions Virtual Keys can be restricted to use only specific provider/models. When provider/model restrictions are configured, the VK can only access those designated provider/models, providing fine-grained control over which provider/models different users or applications can utilize. @@ -334,4 +338,4 @@ If you see warnings like this in your Bifrost logs during startup or provider up 1. Check that the provider is correctly configured and accessible 2. Verify network connectivity to the provider's API 3. Ensure API credentials are valid -4. Use `allowed_models: ["*"]` to allow all models, or specify an explicit list for critical providers \ No newline at end of file +4. Use `allowed_models: ["*"]` to allow all models, or specify an explicit list for critical providers diff --git a/docs/features/observability/otel.mdx b/docs/features/observability/otel.mdx index edaf84dee1..2cb011dfa2 100644 --- a/docs/features/observability/otel.mdx +++ b/docs/features/observability/otel.mdx @@ -941,6 +941,10 @@ These are the same **Prometheus-style metrics** from the telemetry plugin, pushe | `bifrost_stream_inter_token_latency_seconds` | Histogram | Inter-token latency | | `http_requests_total` | Counter | Total HTTP requests | | `http_request_duration_seconds` | Histogram | HTTP request duration | +| `http_request_size_bytes` | Histogram | HTTP request body size | +| `http_response_size_bytes` | Histogram | HTTP response body size | + +> **Note:** Size metrics are only recorded when the `Content-Length` header is present. Requests or responses without it (e.g., chunked transfer encoding, streaming responses) do not produce data points in these histograms. ### OTEL Collector Configuration @@ -1056,7 +1060,7 @@ By default every plugin's pre- and post-hook execution generates a span, which c } ``` -**Via the UI**: Open the Plugins page and click **Configure Plugin Tracing**. Toggle individual plugins on or off and save. UI-saved settings persist across restarts unless `plugin_span_filter` is set in config.json with a higher `version` value. +**Via the UI**: Open the **Observability** page, select the **Open Telemetry** connector, and click **Configure Plugin Tracing**. Toggle individual plugins on or off and save. UI-saved settings persist across restarts unless `plugin_span_filter` is set in config.json with a higher `version` value. **Filter modes:** @@ -1065,7 +1069,7 @@ By default every plugin's pre- and post-hook execution generates a span, which c | `exclude` | Export spans for all plugins **except** those listed | | `include` | Export spans **only** for the listed plugins | -**Built-in plugin names** (for reference): `telemetry`, `prompts`, `logging`, `governance`, `otel`, `semantic_cache`, `compat`, `maxim`. +**Plugin names:** list each plugin using the exact name shown for it in the **Configure Plugin Tracing** sheet — this is the same name that appears in the span (`plugin..`), and it is what the filter matches against. The built-in OSS plugins are `telemetry`, `prompts`, `logging`, `governance`, `otel`, `semantic_cache`, `compat`, and `maxim`. In enterprise deployments some plugins are registered under a different name than their config key — for example the prompts and governance plugins appear as `enterprise-prompts` and `enterprise-governance` — so always copy the name from the tracing sheet rather than assuming the config key. When a plugin span is filtered out, its children are automatically re-parented to the nearest exported ancestor so the trace hierarchy stays connected. diff --git a/docs/features/observability/prometheus.mdx b/docs/features/observability/prometheus.mdx index 2a77478274..6cd105025b 100644 --- a/docs/features/observability/prometheus.mdx +++ b/docs/features/observability/prometheus.mdx @@ -220,7 +220,7 @@ Most request-level Bifrost LLM metrics include these labels (the `bifrost_key_ro - `alias` - Alias resolved to this model (empty if none) - `method` - Request type (chat, completion, embedding, etc.) - `virtual_key_id` / `virtual_key_name` - Virtual key identifiers -- `routing_engine_used` - Comma-separated list of routing engines that contributed to the decision (e.g. `governance`, `routing-rule`, `loadbalancing`, `model-catalog`) +- `routing_engine_used` - Comma-separated list of routing engines that contributed to the decision (e.g. `governance`, `routing-rule`, `loadbalancing`, `model-catalog`, `core`). `core` is emitted when the Bifrost orchestrator itself makes a routing decision — fallback transitions or retry transitions. - `routing_rule_id` / `routing_rule_name` - Routing rule that matched the request - `selected_key_id` / `selected_key_name` - API key that successfully served the request (`""` when all attempts failed) - `fallback_index` - Fallback position diff --git a/docs/features/retries-and-fallbacks.mdx b/docs/features/retries-and-fallbacks.mdx index 821e59f613..2dfbd0264b 100644 --- a/docs/features/retries-and-fallbacks.mdx +++ b/docs/features/retries-and-fallbacks.mdx @@ -367,6 +367,32 @@ The retry budget is set per-provider in `network_config`. If your fallback provi --- +## Auditing retry and fallback decisions + +Every retry transition and every fallback transition is recorded on the request's **routing engine log trail** under the engine name `core`. This is the same per-request trail that plugins like `governance`, `loadbalancing`, `routing-rule`, and `model-catalog` write to when they make routing decisions — so the chain reads end-to-end: which engine picked the primary, what the primary failed with, what core retried with, and which fallback ultimately served the response. + +Entries core emits: + +| Phase | Level | Shape | +|---|---|---| +| Primary failed, entering fallback loop | Info | `Primary

/ failed ( HTTP ); evaluating N configured fallback(s)` | +| Each fallback iteration | Info | `Trying fallback i/N:

/ (previous attempt failed: HTTP )` | +| Fallback skipped (no provider config) | Warn | `Fallback

/ skipped: missing provider config` | +| Fallback succeeded | Info | `Request served by fallback

/ (attempt i/N)` | +| Fallback halted by short-circuit | Error | `Fallback

/ failed ( HTTP ); halting further fallbacks` | +| All fallbacks exhausted | Error | `All N fallback(s) exhausted; returning primary error ( HTTP )` | +| Retry transition (rotated key) | Info | `Retry n/N for

/ (previous attempt failed: HTTP ; rotated key=)` | +| Retry transition (same key) | Info | `Retry n/N for

/ (previous attempt failed: HTTP ; same key=)` | +| Retry transition (keyless provider) | Info | `Retry n/N for

/ (previous attempt failed: HTTP )` | +| Retries succeeded | Info | `Request to

/ succeeded after N retry attempt(s)` | +| Retries exhausted | Error | `Retries exhausted for

/ after N attempt(s); last error: HTTP ` | + +The failure context attached to each entry is intentionally categorical — only the error type (e.g. `rate_limit_error`) and HTTP status code. The upstream provider message is *never* included, since providers can echo back API keys, tokens, or user input. The key identifier surfaced in retry rotation notes is the user-set key **name**, not the secret value. + +When core emits at least one entry on a request, it also adds itself to the request log's `routing_engines_used` field (deduped — `core` appears at most once per request even if both the retry and fallback orchestrators were involved). + +--- + ## Real-world scenarios **Scenario 1: Rate limiting with key rotation** diff --git a/docs/features/semantic-caching.mdx b/docs/features/semantic-caching.mdx index d2eb9b2cba..c3f6f81b12 100644 --- a/docs/features/semantic-caching.mdx +++ b/docs/features/semantic-caching.mdx @@ -1,290 +1,323 @@ --- title: "Semantic Caching" -description: "Intelligent response caching based on semantic similarity. Reduce costs and latency by serving cached responses for semantically similar requests." +description: "Cache AI responses with exact-match hashing and semantic similarity search. Cut costs and latency by replaying answers for identical or semantically similar requests." icon: "database" --- ## Overview -Semantic caching uses vector similarity search to intelligently cache AI responses, serving cached results for semantically similar requests even when the exact wording differs. This dramatically reduces API costs and latency for repeated or similar queries. +Bifrost can cache LLM responses and replay them for repeated requests, avoiding a round-trip to the provider. It offers two complementary lookup paths: -**Key Benefits:** -- **Cost Reduction**: Avoid expensive LLM API calls for similar requests -- **Improved Performance**: Sub-millisecond cache retrieval vs multi-second API calls -- **Intelligent Matching**: Semantic similarity beyond exact text matching -- **Streaming Support**: Full streaming response caching with proper chunk ordering +- **Direct (hash) matching** — deterministic, exact-match replay. The request is normalized and hashed; an identical request is served instantly. No embeddings required. +- **Semantic (similarity) matching** — embedding-based lookup that serves a cached answer when a *new* request is close enough to a previous one, even if the wording differs. + +Both paths can run together (direct first, semantic on miss), or you can run direct-only with no embedding provider at all. + + +In the Web UI this feature is labeled **Local Cache** (under **Settings → Caching**). "Semantic caching" refers to the embedding-based mode; "direct" mode is the embedding-free path. They are the same plugin (`semantic_cache`). + + +**Key benefits:** +- **Cost reduction** — skip paid LLM calls for repeated or similar prompts. +- **Lower latency** — sub-millisecond cache reads vs. multi-second provider calls. +- **Two modes** — exact-match deduplication (direct) or fuzzy similarity (semantic). +- **Streaming support** — streamed responses are cached and replayed chunk-by-chunk. --- -## Core Features +## How it works + +```mermaid +graph LR + A[Request] --> B{Cache key present?} + B -- No --> P[Skip cache, call provider] + B -- Yes --> C[Direct hash lookup
exact match, no threshold] + C -- Exact hit --> R[Serve cached response] + C -- Miss --> D{Semantic enabled?} + D -- No --> P + D -- Yes --> E[Embed + similarity search] + E -- similarity >= threshold --> R + E -- below threshold --> P + P --> W[Store response async, TTL applied] +``` + +A few things that trip up first-time users — read these before configuring: -- **Dual-Layer Caching**: Exact hash matching + semantic similarity search (customizable threshold) -- **Vector-Powered Intelligence**: Uses embeddings to find semantically similar requests -- **Dynamic Configuration**: Per-request TTL and threshold overrides via headers/context -- **Model/Provider Isolation**: Separate caching per model and provider combination +1. **A cache key is mandatory.** Caching only engages when a request carries a cache key (the `x-bf-cache-key` header, or the `CacheKey` context value in the Go SDK). Without one — and without a configured `default_cache_key` — the request bypasses the cache entirely. This is the single most common reason "nothing is being cached." +2. **Direct runs before semantic.** When both paths are enabled, a direct hash hit is served first; the semantic search only runs on a direct miss. You can narrow a request to one path with the `x-bf-cache-type` header. +3. **Writes are asynchronous.** On a cache miss, Bifrost returns the provider's response immediately and stores it in the background, so the *first* request never blocks on a cache write. +4. **Entries persist across restarts.** Cache entries live in your vector store with a per-entry expiry (`expires_at`). They are **not** purged when Bifrost shuts down — a restart keeps serving warm cache (see [Cache lifecycle](#lifecycle--cleanup)). + +**What gets cached:** chat completions, text completions, the Responses API (including WebSocket), embeddings, transcriptions, speech, and image generation — including their streaming variants. + + +**Latency overhead.** The cache lookup itself adds latency to every cache-enabled request, and the cost differs per path: + +- **Direct lookup** — one vector store round-trip per request, hit or miss. Sub-millisecond to a few milliseconds with a local Redis/Valkey; higher with remote or managed stores. (Computing the request hash itself is in-process and takes microseconds — the round-trip is the only real cost.) +- **Semantic lookup** — runs on every direct miss, and must embed the incoming request *before* it can search. That means one embedding API call to your provider (typically tens to a few hundred milliseconds) plus a vector similarity search, paid upfront regardless of the outcome. A semantic **hit** therefore costs roughly an embedding round-trip — not the near-instant replay of a direct hit — and a semantic **miss** pays the embedding call *on top of* the full LLM call, making it slower than running without the cache. +- **Cache writes** — asynchronous; they add no latency to the response. + --- -## Vector Store Setup +## Prerequisites -Semantic caching requires a configured vector store. Bifrost supports the following vector databases: +1. **A vector store** is required as the storage backend for *both* modes — even direct-only mode stores its entries there. Bifrost supports: + + In-memory, RediSearch-compatible. Recommended for direct-only mode. + Production-ready vector database with gRPC support. - - High-performance in-memory vector store using RediSearch-compatible APIs. - Rust-based vector search engine with advanced filtering. - Managed vector database service with serverless options. + Managed, serverless vector database service. +2. **An embedding-capable provider** — only if you want semantic mode. Direct-only mode needs no provider. + -For detailed setup instructions and configuration options for each vector store, see the [Vector Store documentation](/architecture/framework/vector-store). +See the [Vector Store documentation](/architecture/framework/vector-store) for per-store setup. The vector store must be enabled in `config.json` before the **Enable Caching** toggle becomes available in the UI. -**Quick Example (Weaviate):** - - - - - -```go -import ( - "context" - "github.com/maximhq/bifrost/framework/vectorstore" -) - -// Configure vector store (example: Weaviate) -vectorConfig := &vectorstore.Config{ - Enabled: true, - Type: vectorstore.VectorStoreTypeWeaviate, - Config: vectorstore.WeaviateConfig{ - Scheme: "http", - Host: "localhost:8080", - }, -} - -// Create vector store -store, err := vectorstore.NewVectorStore(context.Background(), vectorConfig, logger) -if err != nil { - log.Fatal("Failed to create vector store:", err) -} -``` - - - - +**Minimal vector store config (Redis/Valkey):** ```json { "vector_store": { "enabled": true, - "type": "weaviate", + "type": "redis", "config": { - "host": "localhost:8080", - "scheme": "http" + "addr": "localhost:6379" } } } ``` - - - + +For Valkey, keep `vector_store.type` as `"redis"` and point `config.addr` at your Valkey endpoint. + --- -## Semantic Cache Configuration +## Configuration -> **UI Note**: The current Web UI flow configures provider-backed semantic caching. If you want direct-only mode (`dimension: 1` with no `provider`), configure it through `config.json`. + - + - +![Local Cache configuration page](../media/ui-semantic-cache-config.png) -```go -import ( - "github.com/maximhq/bifrost/plugins/semanticcache" - "github.com/maximhq/bifrost/core/schemas" -) +1. Configure and enable a **vector store** in `config.json` (see [Prerequisites](#prerequisites)). Without it, the toggle stays disabled. +2. In the Bifrost UI, go to **Settings → Caching**. You'll see the **Local Cache** panel. +3. Flip **Enable Caching** on. The plugin loads live — no server restart needed. +4. Pick a **Cache Mode** using the tabs at the top of the panel: + - **Direct only** — exact-match caching. No provider or embeddings. Cheapest path; ideal for stable, repeated prompts. + - **Direct + Semantic** — adds vector similarity on top of direct matching. Requires an embedding-capable provider. (This tab is disabled until at least one embedding-capable provider is configured.) -// Configure semantic cache plugin -cacheConfig := &semanticcache.Config{ - // Embedding model configuration (Required) - Provider: schemas.OpenAI, - EmbeddingModel: "text-embedding-3-small", - Dimension: 1536, - - // Cache behavior - TTL: 5 * time.Minute, // Time to live for cached responses (default: 5 minutes) - Threshold: 0.8, // Similarity threshold for cache lookup (default: 0.8) - CleanUpOnShutdown: true, // Clean up cache on shutdown (default: false) - - // Conversation behavior - ConversationHistoryThreshold: 5, // Skip caching if conversation has > N messages (default: 3) - ExcludeSystemPrompt: bifrost.Ptr(false), // Exclude system messages from cache key (default: false) - - // Advanced options - CacheByModel: bifrost.Ptr(true), // Include model in cache key (default: true) - CacheByProvider: bifrost.Ptr(true), // Include provider in cache key (default: true) -} +5. **For semantic mode**, fill in the embedding provider, model, and dimension that appear below the tabs: + - **Configured Providers** — an embedding-capable provider already set up in Bifrost. Its API keys are inherited automatically. + - **Embedding Model** — e.g. `text-embedding-3-small`. + - **Dimension** — the vector size the model produces. **Must match the model exactly** (e.g. `1536` for `text-embedding-3-small`, `3072` for `text-embedding-3-large`, `768` for many Cohere/Voyage models). -// Create plugin -plugin, err := semanticcache.Init(context.Background(), cacheConfig, logger, store) -if err != nil { - log.Fatal("Failed to create semantic cache plugin:", err) -} - -// Add to Bifrost config -bifrostConfig := schemas.BifrostConfig{ - LLMPlugins: []schemas.LLMPlugin{plugin}, - // ... other config -} -``` +6. Tune **Cache Settings**, **Storage & Cache Key**, **Conversation Settings**, and **Cache Key Composition** (all explained in the [field reference](#field-reference) below). +7. Click **Save Changes**. Config changes mutate the live plugin in place. +8. Send a request with an `x-bf-cache-key` header to start caching (see [Triggering the cache](#triggering-the-cache)). - - -![Semantic Cache Plugin Configuration](../media/ui-semantic-cache-config.png) - -**Prerequisites**: A vector store must be configured and enabled in `config.json`, and at least one provider must be configured, before the toggle becomes available. + -1. **Navigate to the Config page** in the Bifrost UI and find the **Plugins** section. +The cache is the `semantic_cache` plugin, managed through the plugins API. The `config` object takes the same fields as the [field reference](#field-reference) below. -2. **Toggle** the **Enable Semantic Caching** switch to enable it. The configuration form expands below. +**Create (enable) the plugin:** -3. **Fill in the fields** across the four sections: +```bash +curl -X POST http://localhost:8080/api/plugins \ + -H "Content-Type: application/json" \ + -d '{ + "name": "semantic_cache", + "enabled": true, + "path": "", + "config": { + "provider": "openai", + "embedding_model": "text-embedding-3-small", + "dimension": 1536, + "ttl": "5m", + "threshold": 0.8, + "conversation_history_threshold": 3, + "exclude_system_prompt": false, + "cache_by_model": true, + "cache_by_provider": true, + "vector_store_namespace": "BifrostSemanticCachePlugin", + "default_cache_key": "" + } + }' +``` -**Provider and Model Settings** (required for semantic mode): -- **Configured Providers**: Dropdown of providers already set up in Bifrost. The selected provider's API keys are inherited automatically. -- **Embedding Model**: The embedding model to use (e.g. `text-embedding-3-small`). +**Update config or toggle on/off** (changes apply to the live plugin, no restart): -**Cache Settings**: -- **TTL (seconds)**: How long cached responses are kept (default: 300 s). -- **Similarity Threshold**: Cosine similarity cutoff for a cache hit (0–1, default: 0.8). -- **Dimension**: Vector size produced by the embedding model — must match the model exactly. Common values: `1536` for OpenAI `text-embedding-3-small`, `3072` for `text-embedding-3-large`, `768` for many Cohere/Voyage models. Use `1` only in direct-only mode (no provider). +```bash +curl -X PUT http://localhost:8080/api/plugins/semantic_cache \ + -H "Content-Type: application/json" \ + -d '{ + "enabled": true, + "path": "", + "config": { "ttl": "10m", "threshold": 0.85, "dimension": 1536, "provider": "openai", "embedding_model": "text-embedding-3-small" } + }' +``` -> **Heads up**: a vector store namespace can only hold vectors of *one* dimension. Whenever you change the embedding **provider**, **model**, or **dimension**, make sure the new dimension still matches what the model produces — otherwise writes to the existing namespace will fail and reads will silently miss. The namespace is **not** recreated automatically; either point `vector_store_namespace` at a fresh name or drop the existing class/index in your vector store before saving. +**Read current config / disable:** -**Conversation Settings**: -- **Conversation History Threshold**: Skip caching when the conversation has more than this many messages (default: 3). -- **Exclude System Prompt** (toggle): Exclude system messages from cache-key generation. +```bash +# Inspect the current plugin config and status +curl http://localhost:8080/api/plugins/semantic_cache -**Cache Behavior**: -- **Cache by Model** (toggle): Include the model name in the cache key (default: on). -- **Cache by Provider** (toggle): Include the provider name in the cache key (default: on). +# Disable without deleting the saved config +curl -X PUT http://localhost:8080/api/plugins/semantic_cache \ + -H "Content-Type: application/json" \ + -d '{ "enabled": false, "path": "", "config": { "dimension": 1 } }' +``` -4. Click **Save**. Changes are persisted and applied immediately for enabled plugins via the API reload path; other plugin changes (e.g. via `config.json`) may still require a restart. + +A vector store must be enabled in `config.json` first — the plugin has nowhere to store entries otherwise. For **direct-only mode**, send `"dimension": 1` and omit `provider`/`embedding_model`. + ```json -{ +{ + "vector_store": {...}, "plugins": [ { "enabled": true, "name": "semantic_cache", - "config": { + "config": { "provider": "openai", "embedding_model": "text-embedding-3-small", "dimension": 1536, - + "ttl": "5m", "threshold": 0.8, - + "conversation_history_threshold": 3, "exclude_system_prompt": false, - + "cache_by_model": true, - "cache_by_provider": true + "cache_by_provider": true, + + "vector_store_namespace": "BifrostSemanticCachePlugin", + "default_cache_key": "" } } ] } ``` -> **Note**: Provider API keys are inherited automatically from the global provider configuration. You do not need to (and cannot) specify keys inside the plugin config. +> **Note:** Provider API keys are inherited automatically from the global provider configuration. You do not need to (and cannot) specify keys inside the plugin config. -**TTL Format Options:** +**TTL format options:** - Duration strings: `"30s"`, `"5m"`, `"1h"`, `"24h"` - Numeric seconds: `300` (5 minutes), `3600` (1 hour) - + ---- +```go +import ( + "time" -## Direct Hash Mode (Embedding-Free) + bifrost "github.com/maximhq/bifrost/core" + "github.com/maximhq/bifrost/core/schemas" + "github.com/maximhq/bifrost/plugins/semanticcache" +) -Direct hash mode provides exact-match caching without requiring an embedding provider. Each request is hashed deterministically based on its normalized input, parameters, and stream flag. Identical requests produce cache hits; different wording is a cache miss. +cacheConfig := &semanticcache.Config{ + // Embedding settings (semantic mode only) + Provider: schemas.OpenAI, + EmbeddingModel: "text-embedding-3-small", + Dimension: 1536, // use 1 for direct-only mode -Exact-match direct entries are stored and retrieved using a deterministic cache ID. This keeps repeated direct cache lookups fast and consistent across retries, streaming responses, and restarts. + // Cache behavior + TTL: 5 * time.Minute, // default: 5m + Threshold: 0.8, // default: 0.8 -**When to use direct hash mode:** -- You only need exact-match deduplication (no fuzzy/semantic matching) -- You cannot or do not want to call an external embedding API -- You want the lowest possible latency with zero embedding overhead -- Cost-sensitive environments where embedding API calls add up + // Conversation behavior + ConversationHistoryThreshold: 3, // default: 3 + ExcludeSystemPrompt: bifrost.Ptr(false), -### Setup + // Cache key composition + CacheByModel: bifrost.Ptr(true), + CacheByProvider: bifrost.Ptr(true), -To enable direct-only mode globally, set `dimension: 1` and omit the `provider` and `embedding_model` fields from the plugin config. The plugin will automatically fall back to direct search only. + // Storage & default key (optional) + VectorStoreNamespace: "BifrostSemanticCachePlugin", + DefaultCacheKey: "", +} -> **Important**: If you specify `dimension: 1` and also provide a `provider`, Bifrost treats the config as provider-backed semantic mode, not direct-only mode. To use direct-only mode, omit the `provider` field entirely. +plugin, err := semanticcache.Init(context.Background(), cacheConfig, logger, vectorStore) +if err != nil { + log.Fatal("Failed to create semantic cache plugin:", err) +} - -A vector store is still required as the storage backend, even in direct hash mode. See [Recommended Vector Store](#recommended-vector-store) below for the best choice. - +bifrostConfig := schemas.BifrostConfig{ + LLMPlugins: []schemas.LLMPlugin{plugin}, + // ... other config +} +``` - + - + -```go -import ( - "github.com/maximhq/bifrost/plugins/semanticcache" -) +### Field reference + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `provider` | string | — | Embedding provider. **Required for semantic mode**; omit for direct-only. | +| `embedding_model` | string | — | Embedding model name. Required when `provider` is set. | +| `dimension` | integer | — | Vector size. Use `1` for direct-only mode; the embedding model's real dimension (`> 1`) for semantic mode. **Required.** | +| `ttl` | duration / seconds | `5m` (300s) | How long entries live before they expire. Accepts a duration string (`"5m"`) or numeric seconds (`300`). | +| `threshold` | number (0–1) | `0.8` | Minimum cosine similarity for a semantic hit. Semantic mode only. | +| `conversation_history_threshold` | integer | `3` | Skip caching when a conversation has **more than** this many messages. UI range: 1–50. | +| `exclude_system_prompt` | boolean | `false` | Exclude system messages from cache-key generation. | +| `cache_by_model` | boolean | `true` | Include the model name in the cache key (different models won't share entries). | +| `cache_by_provider` | boolean | `true` | Include the provider name in the cache key (different providers won't share entries). | +| `vector_store_namespace` | string | `BifrostSemanticCachePlugin` | Bucket/index where entries live. Changing it points the plugin at a different (possibly empty) bucket; old entries aren't deleted, just no longer queried. | +| `default_cache_key` | string | `""` (empty) | Fallback cache key used when a request doesn't send `x-bf-cache-key`. **Left empty, caching is disabled for any request without the header.** | -cacheConfig := &semanticcache.Config{ - // No Provider or EmbeddingModel -- direct hash mode only - Dimension: 1, // Placeholder; entries are stored as metadata-only (no embedding vectors). Change dimension before switching to dual-layer mode to avoid mixed-dimension issues. +--- - TTL: 5 * time.Minute, - CleanUpOnShutdown: true, - CacheByModel: bifrost.Ptr(true), - CacheByProvider: bifrost.Ptr(true), -} +## Direct vs. semantic mode -plugin, err := semanticcache.Init(ctx, cacheConfig, logger, store) -``` +| | Direct only | Direct + Semantic | +|---|---|---| +| **Matches** | Exact (normalized) request | Exact **and** semantically similar | +| **Embedding provider** | Not needed | Required | +| **Cost per miss** | Zero embedding cost | One embedding call per miss | +| **Added latency** | One vector store round-trip per request | Store round-trip, plus an embedding call + similarity search on every direct miss | +| **Best for** | Stable, repeated prompts; strict dedup | Paraphrased / varied user queries | +| **`dimension`** | `1` | The model's real vector size (`> 1`) | - +### Direct-only setup - +Direct mode hashes each request deterministically from its normalized input, parameters, and stream flag. Identical requests hit; any difference is a miss. The deterministic cache ID keeps repeated lookups consistent across retries, streaming, and restarts. -```yaml -bifrost: - plugins: - semanticCache: - enabled: true - config: - dimension: 1 - ttl: "5m" - cache_by_model: true - cache_by_provider: true -``` +To enable direct-only mode, set `dimension: 1` and **omit** `provider` and `embedding_model`. In the UI, pick the **Direct only** tab. - + +If you set `dimension: 1` **and** also provide a `provider`, Bifrost treats the config as semantic mode, not direct-only. To use direct-only mode, omit `provider` entirely. + + + @@ -307,131 +340,93 @@ bifrost: - - -When initialized this way, all requests automatically use direct hash matching regardless of the `x-bf-cache-type` header. No embeddings are generated, and no embedding provider credentials are needed. + -### Recommended Vector Store +```go +cacheConfig := &semanticcache.Config{ + // No Provider or EmbeddingModel -- direct hash mode only. + Dimension: 1, // entries are stored as metadata-only (no embedding vectors). -**Redis/Valkey-compatible stores** are recommended for direct hash mode. They do not require vectors for metadata-only entries, and all cache fields are indexed as TAG fields for fast exact-match lookups. + TTL: 5 * time.Minute, + CacheByModel: bifrost.Ptr(true), + CacheByProvider: bifrost.Ptr(true), +} - -Qdrant and Pinecone are not compatible with direct hash mode when no embedding provider is configured. These stores require a vector for every entry; the plugin's zero-vector placeholder codepath requires an initialised embedding client, so storage will fail if no provider is set. Weaviate requires a vector per entry as well and is therefore also not recommended for direct-only mode. - +plugin, err := semanticcache.Init(ctx, cacheConfig, logger, store) +``` - + ```yaml -vectorStore: - enabled: true - type: redis - redis: - external: +bifrost: + plugins: + semanticCache: enabled: true - host: "redis-or-valkey.example.com" - port: 6379 - password: "your-redis-password" -``` - - - - - -```json -{ - "vector_store": { - "enabled": true, - "type": "redis", - "config": { - "addr": "localhost:6379" - } - } -} + config: + dimension: 1 + ttl: "5m" + cache_by_model: true + cache_by_provider: true ``` - -For Valkey deployments, keep `vector_store.type` as `"redis"` and point `config.addr` to your Valkey endpoint. - - -### Per-Request Cache Type Override +In direct-only mode, all requests use hash matching regardless of the `x-bf-cache-type` header — no embeddings are generated and no embedding credentials are needed. -When the plugin is initialized **without** an embedding provider (direct-only mode), all requests use direct hash matching automatically. The `x-bf-cache-type` header has no effect. +### Recommended vector store for direct-only mode -When the plugin is initialized **with** an embedding provider (dual-layer mode), you can force direct-only matching on specific requests using the `x-bf-cache-type: direct` header. See [Cache Type Control](#cache-type-control) for details. +**Redis/Valkey-compatible stores** are recommended for direct-only mode. They don't require a vector for metadata-only entries, and all cache fields are indexed as TAG fields for fast exact-match lookups. + + +**Qdrant, Pinecone, and Weaviate are not suitable for direct-only mode.** They require a vector for every entry; the plugin's zero-vector placeholder codepath needs an initialized embedding client, so storage fails when no provider is configured. Use Redis/Valkey for direct-only. + --- -## Cache Triggering +## Triggering the cache -**Cache Key is mandatory**: Semantic caching only activates when a cache key is provided. Without a cache key, requests bypass caching entirely. +**A cache key is mandatory.** Caching only activates when a request carries a cache key. Without one (and without a configured `default_cache_key`), the request bypasses caching entirely. - +The cache key is the **partition** every lookup and write is scoped to — it's part of the cache entry's identity alongside the model and provider. It exists for two reasons: - -Must set cache key in request context: +- **Isolation (no cross-talk).** Entries are only ever matched within the same key. A request under `tenant-A` can never be served a response cached under `tenant-B`, even if the prompts are identical. This prevents one user, tenant, or feature from leaking cached answers to another — the key is how you draw that boundary (per user, per session, per feature, per tenant, etc.). +- **Explicit opt-in.** Caching changes behavior — a response can be replayed instead of freshly generated. Requiring a key makes that a deliberate choice per request (or per deployment via `default_cache_key`), so you never accidentally serve a cached answer where you wanted a live one. -```go -// This request WILL be cached -ctx = context.WithValue(ctx, semanticcache.CacheKey, "session-123") -response, err := client.ChatCompletionRequest(schemas.NewBifrostContext(ctx, schemas.NoDeadline), request) +Pick a key granularity that matches how much you want to share: a coarse key (e.g. a feature name) maximizes hit rate across users; a fine key (e.g. a per-user or per-session ID) keeps caches private at the cost of fewer hits. -// This request will NOT be cached (no context value) -response, err := client.ChatCompletionRequest(schemas.NewBifrostContext(context.Background(), schemas.NoDeadline), request) -``` - - + -Must set cache key in request header `x-bf-cache-key`: + +Set the cache key in the `x-bf-cache-key` header: ```bash # This request WILL be cached curl -H "x-bf-cache-key: session-123" ... -# This request will NOT be cached (no header) +# This request will NOT be cached (no header, no default_cache_key) curl ... ``` - - -## Per-Request Overrides - -Override default TTL and similarity threshold per request: - - - -You can set TTL and threshold in the request context using the semantic cache context keys: +Set the cache key in the request context: ```go -// Go SDK: Custom TTL and threshold +// This request WILL be cached ctx = context.WithValue(ctx, semanticcache.CacheKey, "session-123") -ctx = context.WithValue(ctx, semanticcache.CacheTTLKey, 30*time.Second) -ctx = context.WithValue(ctx, semanticcache.CacheThresholdKey, 0.9) -``` - - - - - -You can set TTL and threshold in the request headers `x-bf-cache-ttl` and `x-bf-cache-threshold`: +response, err := client.ChatCompletionRequest(schemas.NewBifrostContext(ctx, schemas.NoDeadline), request) -```bash -# HTTP: Custom TTL and threshold -curl -H "x-bf-cache-key: session-123" \ - -H "x-bf-cache-ttl: 30s" \ - -H "x-bf-cache-threshold: 0.9" ... +// This request will NOT be cached (no context value) +response, err := client.ChatCompletionRequest(schemas.NewBifrostContext(context.Background(), schemas.NoDeadline), request) ``` @@ -440,158 +435,96 @@ curl -H "x-bf-cache-key: session-123" \ --- -## Advanced Cache Control +## Per-request overrides -### Cache Type Control +Every plugin default can be overridden per request via headers (HTTP) or context keys (Go SDK). -Control which caching mechanism to use per request: +| Header | Context key (Go) | Value | Effect | +|--------|------------------|-------|--------| +| `x-bf-cache-key` | `CacheKey` | string | Scope this request to a cache partition. Required (or `default_cache_key`) for caching to engage. | +| `x-bf-cache-ttl` | `CacheTTLKey` | duration string or seconds | Override TTL for this request. Invalid values are ignored. | +| `x-bf-cache-threshold` | `CacheThresholdKey` | float (0–1) | Override the semantic similarity threshold. Clamped to `[0,1]`. | +| `x-bf-cache-type` | `CacheTypeKey` | `direct` or `semantic` | Limit lookup to a single path. | +| `x-bf-cache-no-store` | `CacheNoStoreKey` | `true` | Skip writing the response (still serves cached hits). | - - - - -```go -// Use only direct hash matching (fastest) -ctx = context.WithValue(ctx, semanticcache.CacheKey, "session-123") -ctx = context.WithValue(ctx, semanticcache.CacheTypeKey, semanticcache.CacheTypeDirect) - -// Use only semantic similarity search -ctx = context.WithValue(ctx, semanticcache.CacheKey, "session-123") -ctx = context.WithValue(ctx, semanticcache.CacheTypeKey, semanticcache.CacheTypeSemantic) - -// Default behavior: Direct + semantic fallback (if not specified) -ctx = context.WithValue(ctx, semanticcache.CacheKey, "session-123") -``` - - + ```bash -# Direct hash matching only +# Custom TTL and threshold curl -H "x-bf-cache-key: session-123" \ - -H "x-bf-cache-type: direct" ... + -H "x-bf-cache-ttl: 30s" \ + -H "x-bf-cache-threshold: 0.9" ... -# Semantic similarity search only +# Force direct-only matching curl -H "x-bf-cache-key: session-123" \ - -H "x-bf-cache-type: semantic" ... + -H "x-bf-cache-type: direct" ... -# Default: Both (if header not specified) -curl -H "x-bf-cache-key: session-123" ... +# Read from cache but don't store the response +curl -H "x-bf-cache-key: session-123" \ + -H "x-bf-cache-no-store: true" ... ``` - - -### No-Store Control - -Disable response caching while still allowing cache reads: - - - ```go -// Read from cache but don't store the response ctx = context.WithValue(ctx, semanticcache.CacheKey, "session-123") -ctx = context.WithValue(ctx, semanticcache.CacheNoStoreKey, true) -``` - - +ctx = context.WithValue(ctx, semanticcache.CacheTTLKey, 30*time.Second) +ctx = context.WithValue(ctx, semanticcache.CacheThresholdKey, 0.9) - +// Force a single lookup path +ctx = context.WithValue(ctx, semanticcache.CacheTypeKey, semanticcache.CacheTypeDirect) +// or semanticcache.CacheTypeSemantic -```bash -# Read from cache but don't store response -curl -H "x-bf-cache-key: session-123" \ - -H "x-bf-cache-no-store: true" ... +// Read-only: serve from cache but don't write this response +ctx = context.WithValue(ctx, semanticcache.CacheNoStoreKey, true) ``` ---- - -## Conversation Configuration - -### History Threshold Logic - -The `ConversationHistoryThreshold` setting skips caching for conversations with many messages to prevent false positives: - -**Why this matters:** -- **Semantic False Positives**: Long conversation histories have high probability of semantic matches with unrelated conversations due to topic overlap -- **Direct Cache Inefficiency**: Long conversations rarely have exact hash matches, making direct caching less effective -- **Performance**: Reduces vector store load by filtering out low-value caching scenarios - -```json -{ - "conversation_history_threshold": 3 // Skip caching if > 3 messages in conversation -} -``` - -**Recommended Values:** -- **1-2**: Very conservative (may miss valuable caching opportunities) -- **3-5**: Balanced approach (default: 3) -- **10+**: Cache longer conversations (higher false positive risk) - -### System Prompt Handling - -Control whether system messages are included in cache key generation: - -```json -{ - "exclude_system_prompt": false // Include system messages in cache key (default) -} -``` - -**When to exclude (`true`):** -- System prompts change frequently but content is similar -- Multiple system prompt variations for same use case -- Focus caching on user content similarity - -**When to include (`false`):** -- System prompts significantly change response behavior -- Each system prompt requires distinct cached responses -- Strict response consistency requirements + +In direct-only mode (no embedding provider), `x-bf-cache-type` and `x-bf-cache-threshold` have no effect — every request uses direct matching. + --- -## Cache Management - -### Cache Metadata Location - -When responses are served from semantic cache, 3 key variables are automatically added to the response: +## Cache management -**Location**: `response.ExtraFields.CacheDebug` (as a JSON object) +Every cached or cache-checked response carries debug metadata so you can confirm caching is working and capture the entry's ID for management. -**Fields**: -- `CacheHit` (boolean): `true` if the response was served from the cache, `false` when lookup fails. -- `HitType` (string): `"semantic"` for similarity match, `"direct"` for hash match -- `CacheID` (string): Unique cache entry ID for management operations (present only for cache hits) +**Location:** `response.ExtraFields.CacheDebug` +| Field | When present | Description | +|-------|--------------|-------------| +| `cache_hit` | always | `true` if served from cache, `false` otherwise. | +| `cache_id` | always | Storage ID of the entry — use it to invalidate later. | +| `hit_type` | on hit | `"direct"` or `"semantic"`. | +| `threshold` | on semantic hit | Similarity threshold used. | +| `similarity` | on semantic hit | Actual cosine similarity score. | +| `provider_used` | when semantic search ran | Embedding provider used. | +| `model_used` | when semantic search ran | Embedding model used. | +| `input_tokens` | when semantic search ran | Tokens consumed computing the embedding. | -**Semantic Cache Only**: -- `ProviderUsed` (string): Provider used for the calculating semantic match embedding. (present for both cache hits and misses) -- `ModelUsed` (string): Model used for the calculating semantic match embedding. (present for both cache hits and misses) -- `InputTokens` (number): Number of tokens extracted from the request for the semantic match embedding calculation. (present for both cache hits and misses) -- `Threshold` (number): Similarity threshold used for the match. (present only for cache hits) -- `Similarity` (number): Similarity score for the match. (present only for cache hits) - -Example HTTP Response: +**Examples:** ```json +// Direct hit { "extra_fields": { "cache_debug": { "cache_hit": true, "hit_type": "direct", - "cache_id": "550e8500-e29b-41d4-a725-446655440001", + "cache_id": "550e8500-e29b-41d4-a725-446655440001" } } } +// Semantic hit { "extra_fields": { "cache_debug": { @@ -601,49 +534,51 @@ Example HTTP Response: "threshold": 0.8, "similarity": 0.95, "provider_used": "openai", - "model_used": "gpt-4o-mini", + "model_used": "text-embedding-3-small", "input_tokens": 100 } } } +// Miss (semantic search ran but found nothing close enough) { "extra_fields": { "cache_debug": { "cache_hit": false, "cache_id": "550e8500-e29b-41d4-a725-446655440001", "provider_used": "openai", - "model_used": "gpt-4o-mini", + "model_used": "text-embedding-3-small", "input_tokens": 20 } } } ``` -`cache_debug` is populated on both hits and misses. `cache_id` is the storage ID of the entry — use it to invalidate the entry later. The embedding-related fields (`provider_used`, `model_used`, `input_tokens`) are only present when semantic search actually ran. + +On a streamed response, only the **final** chunk carries the full `cache_debug` payload. + -### Clear Specific Cache Entry +Cache outcomes also surface in **Logs** without inspecting the raw response: -Use the `cache_id` from `cache_debug` to clear a specific entry: +![Log detail sheet showing the Semantic Cache badge and Caching Details block](../media/ui-semantic-cache-log-details.png) - +- **Hit-type badge** — a cache hit is tagged with a **Direct Cache** or **Semantic Cache** badge on the log entry. +- **Cache row** — each cached request shows a `Cache (hit)` / `Cache (miss)` row with the copyable `cache_id`. +- **Caching Details block** — expands to the `cache_debug` fields: cache type, and for semantic hits the embedding provider, embedding model, threshold, similarity score, and embedding input tokens. +- **Local Caching filter** — the logs filter sidebar lets you filter requests by hit type (**Direct cache** / **Semantic cache**). - +--- -```go -// Clear specific entry by cache ID (read from response.ExtraFields.CacheDebug.CacheID) -err := plugin.ClearCacheForCacheID("550e8500-e29b-41d4-a725-446655440001") +### Invalidation -// Clear all entries for a cache key -err := plugin.ClearCacheForKey("support-session-456") -``` +Use the `cache_id` from `cache_debug` to invalidate entries. - + ```bash -# Clear specific cached entry by cache ID +# Clear a specific cached entry by cache ID curl -X DELETE http://localhost:8080/api/cache/clear/550e8500-e29b-41d4-a725-446655440001 # Clear all entries for a cache key @@ -652,31 +587,75 @@ curl -X DELETE http://localhost:8080/api/cache/clear-by-key/support-session-456 - + + +```go +// Clear a specific entry by cache ID +err := plugin.ClearCacheForCacheID("550e8500-e29b-41d4-a725-446655440001") + +// Clear all entries for a cache key +err := plugin.ClearCacheForKey("support-session-456") +``` -### Cache Lifecycle & Cleanup + -The semantic cache automatically handles cleanup to prevent storage bloat: + -**Automatic Cleanup:** -- **TTL Expiration**: Entries are automatically removed when TTL expires -- **Shutdown Cleanup**: All cache entries are cleared from the vector store namespace and the namespace itself when Bifrost client shuts down -- **Namespace Isolation**: Each Bifrost instance uses isolated vector store namespaces to prevent conflicts +### Lifecycle & Cleanup -**Manual Cleanup Options:** -- Clear specific entries by cache ID (see examples above) -- Clear all entries for a cache key -- Restart Bifrost to clear all cache data +- **TTL expiration** — every entry is stored with an `expires_at` timestamp. Expired entries are no longer served and are swept out over time. +- **Entries persist across restarts** — cache data lives in your vector store and is **not** purged when Bifrost shuts down. A restart resumes serving the existing (unexpired) cache. To wipe entries, use the [cache-clear APIs](#cache-management) or clear the namespace in your vector store directly. +- **Namespace isolation** — each `vector_store_namespace` is an independent cache pool. Use distinct namespaces to keep separate caches from colliding. -**Dimension / Provider / Model Changes**: A vector store namespace can only hold vectors of **one** dimension. If you change `dimension` (or switch to an embedding `provider`/`model` that produces a different vector size), the existing namespace is **not** recreated automatically — `CreateNamespace` is a no-op when the class/collection already exists. Subsequent writes will fail (vector-size mismatch) and reads will silently miss. Before saving the change, either: +**Changing `dimension`, `provider`, or `embedding_model`:** a vector store namespace can hold vectors of **one** dimension only. The namespace is **not** recreated automatically — `CreateNamespace` is a no-op when the class/collection already exists. If the new embedding model produces a different vector size, subsequent writes fail (size mismatch) and reads silently miss. Before saving such a change, either: -- point `vector_store_namespace` at a fresh name, or -- drop the existing class/index in your vector store, or +- point `vector_store_namespace` at a fresh name, **or** +- drop the existing class/index in your vector store. --- - -**Vector Store Requirement**: Semantic caching requires a configured vector store. Bifrost supports Weaviate, Redis/Valkey-compatible endpoints, Qdrant, and Pinecone. See the [Vector Store documentation](/architecture/framework/vector-store) for setup details. - +## Troubleshooting + + + + +**Most common cause:** no cache key. Caching only engages when a request sends `x-bf-cache-key` (or you've set a `default_cache_key`). Confirm the header is present, then check `cache_debug` on the response. + + + +Expected. The cache is populated *after* the first response is returned (writes are asynchronous). Send the same request again to see a hit. + + + +- Verify `dimension` exactly matches your embedding model's output size. +- Lower the `threshold` (e.g. `0.8` → `0.75`) if genuinely-similar prompts aren't matching. +- Check `cache_debug.similarity` on a miss to see how close you got. + + + +You changed `dimension`/`provider`/`embedding_model` against an existing namespace. See the [dimension-change warning](#lifecycle--cleanup) — use a fresh namespace or drop the old class/index. + + + +No embedding-capable provider is configured. Add one under **Providers** first; its keys are inherited automatically. + + + +No vector store is enabled. Configure and enable one in `config.json` (see [Prerequisites](#prerequisites)). + + + +You're likely using Qdrant, Pinecone, or Weaviate, which require a vector per entry. Switch to Redis/Valkey for direct-only mode. + + + + +--- + +## Next steps + +- **[Vector Store setup](/architecture/framework/vector-store)** — configure Weaviate, Redis/Valkey, Qdrant, or Pinecone. +- **[Plugins overview](/features/plugins)** — how Bifrost's plugin pipeline works. +- **[Providers](/providers)** — configure the embedding provider used for semantic mode. diff --git a/docs/features/telemetry.mdx b/docs/features/telemetry.mdx index 9a6b02347d..2d0cd1bc78 100644 --- a/docs/features/telemetry.mdx +++ b/docs/features/telemetry.mdx @@ -70,7 +70,7 @@ Base Labels: - `method`: Request type (`chat`, `text`, `embedding`, `speech`, `transcription`) - `virtual_key_id`: Virtual key ID - `virtual_key_name`: Virtual key name -- `routing_engine_used`: Comma-separated routing engines used (`routing-rule`, `governance`, `loadbalancing`, `model-catalog`) +- `routing_engine_used`: Comma-separated routing engines used (`routing-rule`, `governance`, `loadbalancing`, `model-catalog`, `core`). `core` is emitted when the Bifrost orchestrator itself makes a routing decision — i.e. a fallback transition or a retry transition. - `routing_rule_id`: Routing rule ID that matched the request - `routing_rule_name`: Routing rule name that matched the request - `selected_key_id`: ID of the key that successfully served the request (empty string `""` on final errors) diff --git a/docs/media/architecture-complexity-router.png b/docs/media/architecture-complexity-router.png new file mode 100644 index 0000000000..5bb33bc406 Binary files /dev/null and b/docs/media/architecture-complexity-router.png differ diff --git a/docs/media/complexity-logic-architecture.png b/docs/media/complexity-logic-architecture.png new file mode 100644 index 0000000000..c28ecab0eb Binary files /dev/null and b/docs/media/complexity-logic-architecture.png differ diff --git a/docs/media/ui-complexity-router-config.png b/docs/media/ui-complexity-router-config.png new file mode 100644 index 0000000000..9d5bd8f443 Binary files /dev/null and b/docs/media/ui-complexity-router-config.png differ diff --git a/docs/media/ui-complexity-router-keywords.png b/docs/media/ui-complexity-router-keywords.png new file mode 100644 index 0000000000..7d3cd1efb5 Binary files /dev/null and b/docs/media/ui-complexity-router-keywords.png differ diff --git a/docs/media/ui-routing-logs-complexity.png b/docs/media/ui-routing-logs-complexity.png new file mode 100644 index 0000000000..8fa73cc45c Binary files /dev/null and b/docs/media/ui-routing-logs-complexity.png differ diff --git a/docs/media/ui-routing-rule-complexity.png b/docs/media/ui-routing-rule-complexity.png new file mode 100644 index 0000000000..92858997f5 Binary files /dev/null and b/docs/media/ui-routing-rule-complexity.png differ diff --git a/docs/media/ui-semantic-cache-config.png b/docs/media/ui-semantic-cache-config.png index b1b2ba6a7c..4f312caeb8 100644 Binary files a/docs/media/ui-semantic-cache-config.png and b/docs/media/ui-semantic-cache-config.png differ diff --git a/docs/media/ui-semantic-cache-log-details.png b/docs/media/ui-semantic-cache-log-details.png new file mode 100644 index 0000000000..08ebb09670 Binary files /dev/null and b/docs/media/ui-semantic-cache-log-details.png differ diff --git a/docs/openapi/openapi.json b/docs/openapi/openapi.json index 903d90c3c3..88fc1c6799 100644 --- a/docs/openapi/openapi.json +++ b/docs/openapi/openapi.json @@ -40043,7 +40043,7 @@ "get": { "operationId": "getVirtualKeyQuota", "summary": "Get virtual key quota", - "description": "Returns the overall budget and rate limit quota for the authenticated virtual key,\nas well as per-provider budgets and rate limits.\nThis is a self-service endpoint - no admin authentication required.\nThe virtual key value itself (provided via header) is the credential.\n", + "description": "Returns the overall budget and rate limit quota for the authenticated virtual key,\nas well as per-provider and per-model budgets and rate limits (with current usage).\nEach budget also carries the actual per-model usage for its current cycle.\nThis is a self-service endpoint - no admin authentication required.\nThe virtual key value itself (provided via header) is the credential.\n", "tags": [ "Governance" ], @@ -49027,6 +49027,445 @@ } } }, + "/api/governance/complexity-analyzer-config": { + "get": { + "operationId": "getComplexityAnalyzerConfig", + "summary": "Get complexity analyzer config", + "description": "Returns the full complexity analyzer runtime config, including tier thresholds and editable keyword lists. Returns built-in defaults if none have been configured.", + "tags": [ + "Governance" + ], + "responses": { + "200": { + "description": "Complexity analyzer config retrieved successfully", + "content": { + "application/json": { + "schema": { + "type": "object", + "description": "Full runtime configuration for complexity routing analysis.", + "additionalProperties": false, + "required": [ + "tier_boundaries", + "keywords" + ], + "properties": { + "tier_boundaries": { + "type": "object", + "description": "Score thresholds for complexity tier classification. All values must satisfy 0 < simple_medium < medium_complex < complex_reasoning < 1.", + "additionalProperties": false, + "required": [ + "simple_medium", + "medium_complex", + "complex_reasoning" + ], + "properties": { + "simple_medium": { + "type": "number", + "description": "Score boundary between SIMPLE and MEDIUM tiers", + "exclusiveMinimum": 0, + "exclusiveMaximum": 1 + }, + "medium_complex": { + "type": "number", + "description": "Score boundary between MEDIUM and COMPLEX tiers", + "exclusiveMinimum": 0, + "exclusiveMaximum": 1 + }, + "complex_reasoning": { + "type": "number", + "description": "Score boundary between COMPLEX and REASONING tiers", + "exclusiveMinimum": 0, + "exclusiveMaximum": 1 + } + } + }, + "keywords": { + "type": "object", + "description": "Editable keyword lists used by the complexity analyzer. Only the four user-facing\ndimensions are exposed; `reasoning_keywords` entries drive the reasoning tier\noverride. Matching is normalized to lowercase and duplicates are removed on save.\n", + "additionalProperties": false, + "required": [ + "code_keywords", + "reasoning_keywords", + "technical_keywords", + "simple_keywords" + ], + "properties": { + "code_keywords": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + }, + "reasoning_keywords": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + }, + "technical_keywords": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + }, + "simple_keywords": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + } + } + } + } + } + } + } + }, + "500": { + "description": "Internal server error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BifrostError" + } + } + } + }, + "503": { + "$ref": "#/components/responses/ConfigStoreUnavailable" + } + } + }, + "put": { + "operationId": "updateComplexityAnalyzerConfig", + "summary": "Update complexity analyzer config", + "description": "Replaces the full complexity analyzer runtime config and hot-reloads the governance plugin. Tier thresholds must satisfy 0 < simple_medium < medium_complex < complex_reasoning < 1.", + "tags": [ + "Governance" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "description": "Full runtime configuration for complexity routing analysis.", + "additionalProperties": false, + "required": [ + "tier_boundaries", + "keywords" + ], + "properties": { + "tier_boundaries": { + "type": "object", + "description": "Score thresholds for complexity tier classification. All values must satisfy 0 < simple_medium < medium_complex < complex_reasoning < 1.", + "additionalProperties": false, + "required": [ + "simple_medium", + "medium_complex", + "complex_reasoning" + ], + "properties": { + "simple_medium": { + "type": "number", + "description": "Score boundary between SIMPLE and MEDIUM tiers", + "exclusiveMinimum": 0, + "exclusiveMaximum": 1 + }, + "medium_complex": { + "type": "number", + "description": "Score boundary between MEDIUM and COMPLEX tiers", + "exclusiveMinimum": 0, + "exclusiveMaximum": 1 + }, + "complex_reasoning": { + "type": "number", + "description": "Score boundary between COMPLEX and REASONING tiers", + "exclusiveMinimum": 0, + "exclusiveMaximum": 1 + } + } + }, + "keywords": { + "type": "object", + "description": "Editable keyword lists used by the complexity analyzer. Only the four user-facing\ndimensions are exposed; `reasoning_keywords` entries drive the reasoning tier\noverride. Matching is normalized to lowercase and duplicates are removed on save.\n", + "additionalProperties": false, + "required": [ + "code_keywords", + "reasoning_keywords", + "technical_keywords", + "simple_keywords" + ], + "properties": { + "code_keywords": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + }, + "reasoning_keywords": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + }, + "technical_keywords": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + }, + "simple_keywords": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + } + } + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Complexity analyzer config updated successfully", + "content": { + "application/json": { + "schema": { + "type": "object", + "description": "Full runtime configuration for complexity routing analysis.", + "additionalProperties": false, + "required": [ + "tier_boundaries", + "keywords" + ], + "properties": { + "tier_boundaries": { + "type": "object", + "description": "Score thresholds for complexity tier classification. All values must satisfy 0 < simple_medium < medium_complex < complex_reasoning < 1.", + "additionalProperties": false, + "required": [ + "simple_medium", + "medium_complex", + "complex_reasoning" + ], + "properties": { + "simple_medium": { + "type": "number", + "description": "Score boundary between SIMPLE and MEDIUM tiers", + "exclusiveMinimum": 0, + "exclusiveMaximum": 1 + }, + "medium_complex": { + "type": "number", + "description": "Score boundary between MEDIUM and COMPLEX tiers", + "exclusiveMinimum": 0, + "exclusiveMaximum": 1 + }, + "complex_reasoning": { + "type": "number", + "description": "Score boundary between COMPLEX and REASONING tiers", + "exclusiveMinimum": 0, + "exclusiveMaximum": 1 + } + } + }, + "keywords": { + "type": "object", + "description": "Editable keyword lists used by the complexity analyzer. Only the four user-facing\ndimensions are exposed; `reasoning_keywords` entries drive the reasoning tier\noverride. Matching is normalized to lowercase and duplicates are removed on save.\n", + "additionalProperties": false, + "required": [ + "code_keywords", + "reasoning_keywords", + "technical_keywords", + "simple_keywords" + ], + "properties": { + "code_keywords": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + }, + "reasoning_keywords": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + }, + "technical_keywords": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + }, + "simple_keywords": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + } + } + } + } + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BifrostError" + } + } + } + }, + "500": { + "description": "Internal server error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BifrostError" + } + } + } + }, + "503": { + "$ref": "#/components/responses/ConfigStoreUnavailable" + } + } + } + }, + "/api/governance/complexity-analyzer-config/reset": { + "post": { + "operationId": "resetComplexityAnalyzerConfig", + "summary": "Reset complexity analyzer config", + "description": "Restores the built-in complexity analyzer runtime config defaults and hot-reloads the governance plugin.", + "tags": [ + "Governance" + ], + "responses": { + "200": { + "description": "Complexity analyzer config reset successfully", + "content": { + "application/json": { + "schema": { + "type": "object", + "description": "Full runtime configuration for complexity routing analysis.", + "additionalProperties": false, + "required": [ + "tier_boundaries", + "keywords" + ], + "properties": { + "tier_boundaries": { + "type": "object", + "description": "Score thresholds for complexity tier classification. All values must satisfy 0 < simple_medium < medium_complex < complex_reasoning < 1.", + "additionalProperties": false, + "required": [ + "simple_medium", + "medium_complex", + "complex_reasoning" + ], + "properties": { + "simple_medium": { + "type": "number", + "description": "Score boundary between SIMPLE and MEDIUM tiers", + "exclusiveMinimum": 0, + "exclusiveMaximum": 1 + }, + "medium_complex": { + "type": "number", + "description": "Score boundary between MEDIUM and COMPLEX tiers", + "exclusiveMinimum": 0, + "exclusiveMaximum": 1 + }, + "complex_reasoning": { + "type": "number", + "description": "Score boundary between COMPLEX and REASONING tiers", + "exclusiveMinimum": 0, + "exclusiveMaximum": 1 + } + } + }, + "keywords": { + "type": "object", + "description": "Editable keyword lists used by the complexity analyzer. Only the four user-facing\ndimensions are exposed; `reasoning_keywords` entries drive the reasoning tier\noverride. Matching is normalized to lowercase and duplicates are removed on save.\n", + "additionalProperties": false, + "required": [ + "code_keywords", + "reasoning_keywords", + "technical_keywords", + "simple_keywords" + ], + "properties": { + "code_keywords": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + }, + "reasoning_keywords": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + }, + "technical_keywords": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + }, + "simple_keywords": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + } + } + } + } + } + } + } + }, + "500": { + "description": "Internal server error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BifrostError" + } + } + } + }, + "503": { + "$ref": "#/components/responses/ConfigStoreUnavailable" + } + } + } + }, "/api/logs": { "get": { "operationId": "getLogs", @@ -57500,6 +57939,16 @@ } } } + }, + "ConfigStoreUnavailable": { + "description": "Config store not available", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BifrostError" + } + } + } } }, "schemas": { @@ -68031,7 +68480,9 @@ "type": "boolean" }, "disable_auth_on_inference": { - "type": "boolean" + "type": "boolean", + "deprecated": true, + "description": "Deprecated and ignored. Use client_config.enforce_auth_on_inference instead." } } }, @@ -68296,7 +68747,9 @@ "type": "boolean" }, "disable_auth_on_inference": { - "type": "boolean" + "type": "boolean", + "deprecated": true, + "description": "Deprecated and ignored. Use client_config.enforce_auth_on_inference instead." } } } @@ -70566,7 +71019,8 @@ "is_active", "budgets", "rate_limit", - "provider_configs" + "provider_configs", + "model_configs" ], "properties": { "virtual_key_name": { @@ -70582,9 +71036,60 @@ "array", "null" ], - "description": "Overall budget quotas assigned to this virtual key", + "description": "Overall budget quotas for this virtual key. Each budget also carries the actual per-model spend (from request logs) accumulated in its current cycle.\n", "items": { - "$ref": "#/components/schemas/Budget" + "description": "A virtual key budget with the actual per-model spend (from request logs) accumulated in its current cycle [last_reset, now]. The per-model totals reconcile with current_usage. The models list is empty when the logging plugin is not enabled.\n", + "allOf": [ + { + "$ref": "#/components/schemas/Budget" + }, + { + "type": "object", + "required": [ + "per_model_usage" + ], + "properties": { + "per_model_usage": { + "type": "array", + "description": "Per-model actual usage within this budget's current cycle", + "items": { + "type": "object", + "description": "One model's actual usage (from request logs) within a budget cycle", + "required": [ + "model", + "total_requests", + "total_tokens", + "total_cost" + ], + "properties": { + "model": { + "type": "string", + "description": "The model name" + }, + "provider": { + "type": "string", + "description": "Provider that served the model; omitted when unknown" + }, + "total_requests": { + "type": "integer", + "format": "int64", + "description": "Number of requests to this model in the cycle" + }, + "total_tokens": { + "type": "integer", + "format": "int64", + "description": "Total tokens consumed by this model in the cycle" + }, + "total_cost": { + "type": "number", + "description": "Total cost in dollars for this model in the cycle" + } + } + } + } + } + } + ] } }, "rate_limit": { @@ -70606,6 +71111,47 @@ "items": { "$ref": "#/components/schemas/VirtualKeyProviderConfig" } + }, + "model_configs": { + "type": "array", + "description": "Per-model budget quotas and rate limits configured for this virtual key (model-level limits). Always an array — the endpoint fails closed (500) rather than returning null when the model-config lookup errors.\n", + "items": { + "type": "object", + "description": "Per-model budgets and rate limit (with current usage) for a model governed under a virtual key", + "required": [ + "model_name" + ], + "properties": { + "model_name": { + "type": "string", + "description": "The model this usage applies to" + }, + "provider": { + "type": "string", + "description": "Provider this model config is scoped to; omitted when it applies to all providers" + }, + "budgets": { + "type": [ + "array", + "null" + ], + "description": "Budget quotas for this model", + "items": { + "$ref": "#/components/schemas/Budget" + } + }, + "rate_limit": { + "oneOf": [ + { + "$ref": "#/components/schemas/RateLimit" + }, + { + "type": "null" + } + ] + } + } + } } } }, @@ -73883,4 +74429,4 @@ } } } -} \ No newline at end of file +} diff --git a/docs/openapi/openapi.yaml b/docs/openapi/openapi.yaml index 20cb7c412f..65bf2ee9ae 100644 --- a/docs/openapi/openapi.yaml +++ b/docs/openapi/openapi.yaml @@ -918,6 +918,12 @@ paths: /api/mcp/tool-groups/{id}: $ref: './paths/management/mcptoolgroups.yaml#/mcp-tool-groups-by-id' + # Governance - Complexity Analyzer Config + /api/governance/complexity-analyzer-config: + $ref: './paths/management/governance.yaml#/complexity' + /api/governance/complexity-analyzer-config/reset: + $ref: './paths/management/governance.yaml#/complexity-reset' + # Logging /api/logs: $ref: './paths/management/logging.yaml#/logs' @@ -1086,6 +1092,12 @@ components: application/json: schema: $ref: './schemas/inference/common.yaml#/BifrostError' + ConfigStoreUnavailable: + description: Config store not available + content: + application/json: + schema: + $ref: './schemas/inference/common.yaml#/BifrostError' schemas: # ==================== Common ==================== diff --git a/docs/openapi/paths/management/governance.yaml b/docs/openapi/paths/management/governance.yaml index c126891c1e..7159fccfaa 100644 --- a/docs/openapi/paths/management/governance.yaml +++ b/docs/openapi/paths/management/governance.yaml @@ -109,7 +109,8 @@ virtual-keys-quota: summary: Get virtual key quota description: | Returns the overall budget and rate limit quota for the authenticated virtual key, - as well as per-provider budgets and rate limits. + as well as per-provider and per-model budgets and rate limits (with current usage). + Each budget also carries the actual per-model usage for its current cycle. This is a self-service endpoint - no admin authentication required. The virtual key value itself (provided via header) is the credential. tags: @@ -1221,3 +1222,69 @@ pricing-overrides-by-id: $ref: '../../schemas/management/common.yaml#/MessageResponse' '500': $ref: '../../openapi.yaml#/components/responses/InternalError' + +# Complexity Analyzer Config + +complexity: + get: + operationId: getComplexityAnalyzerConfig + summary: Get complexity analyzer config + description: Returns the full complexity analyzer runtime config, including tier thresholds and editable keyword lists. Returns built-in defaults if none have been configured. + tags: + - Governance + responses: + '200': + description: Complexity analyzer config retrieved successfully + content: + application/json: + schema: + $ref: '../../schemas/management/governance.yaml#/ComplexityAnalyzerConfig' + '500': + $ref: '../../openapi.yaml#/components/responses/InternalError' + '503': + $ref: '../../openapi.yaml#/components/responses/ConfigStoreUnavailable' + + put: + operationId: updateComplexityAnalyzerConfig + summary: Update complexity analyzer config + description: Replaces the full complexity analyzer runtime config and hot-reloads the governance plugin. Tier thresholds must satisfy 0 < simple_medium < medium_complex < complex_reasoning < 1. + tags: + - Governance + requestBody: + required: true + content: + application/json: + schema: + $ref: '../../schemas/management/governance.yaml#/ComplexityAnalyzerConfig' + responses: + '200': + description: Complexity analyzer config updated successfully + content: + application/json: + schema: + $ref: '../../schemas/management/governance.yaml#/ComplexityAnalyzerConfig' + '400': + $ref: '../../openapi.yaml#/components/responses/BadRequest' + '500': + $ref: '../../openapi.yaml#/components/responses/InternalError' + '503': + $ref: '../../openapi.yaml#/components/responses/ConfigStoreUnavailable' + +complexity-reset: + post: + operationId: resetComplexityAnalyzerConfig + summary: Reset complexity analyzer config + description: Restores the built-in complexity analyzer runtime config defaults and hot-reloads the governance plugin. + tags: + - Governance + responses: + '200': + description: Complexity analyzer config reset successfully + content: + application/json: + schema: + $ref: '../../schemas/management/governance.yaml#/ComplexityAnalyzerConfig' + '500': + $ref: '../../openapi.yaml#/components/responses/InternalError' + '503': + $ref: '../../openapi.yaml#/components/responses/ConfigStoreUnavailable' diff --git a/docs/openapi/schemas/management/config.yaml b/docs/openapi/schemas/management/config.yaml index a398d1a964..d2f3eca9c8 100644 --- a/docs/openapi/schemas/management/config.yaml +++ b/docs/openapi/schemas/management/config.yaml @@ -137,6 +137,8 @@ AuthConfig: type: boolean disable_auth_on_inference: type: boolean + deprecated: true + description: "Deprecated and ignored. Use client_config.enforce_auth_on_inference instead." HeaderFilterConfig: type: object diff --git a/docs/openapi/schemas/management/governance.yaml b/docs/openapi/schemas/management/governance.yaml index 9a97529d8f..4c6d1e02a4 100644 --- a/docs/openapi/schemas/management/governance.yaml +++ b/docs/openapi/schemas/management/governance.yaml @@ -474,6 +474,7 @@ VirtualKeyQuotaResponse: - budgets - rate_limit - provider_configs + - model_configs properties: virtual_key_name: type: string @@ -483,9 +484,11 @@ VirtualKeyQuotaResponse: description: Whether the virtual key is active budgets: type: [array, 'null'] - description: Overall budget quotas assigned to this virtual key + description: > + Overall budget quotas for this virtual key. Each budget also carries the actual + per-model spend (from request logs) accumulated in its current cycle. items: - $ref: '#/Budget' + $ref: '#/VirtualKeyBudgetUsage' rate_limit: oneOf: - $ref: '#/RateLimit' @@ -495,6 +498,80 @@ VirtualKeyQuotaResponse: description: Per-provider budget quotas and rate limits (provider-level splits and limits) items: $ref: '#/VirtualKeyProviderConfig' + model_configs: + type: array + description: > + Per-model budget quotas and rate limits configured for this virtual key (model-level + limits). Always an array — the endpoint fails closed (500) rather than returning null + when the model-config lookup errors. + items: + $ref: '#/VirtualKeyModelUsage' + +VirtualKeyBudgetUsage: + description: > + A virtual key budget with the actual per-model spend (from request logs) accumulated in + its current cycle [last_reset, now]. The per-model totals reconcile with current_usage. + The models list is empty when the logging plugin is not enabled. + allOf: + - $ref: '#/Budget' + - type: object + required: + - per_model_usage + properties: + per_model_usage: + type: array + description: Per-model actual usage within this budget's current cycle + items: + $ref: '#/VirtualKeyModelSpend' + +VirtualKeyModelSpend: + type: object + description: One model's actual usage (from request logs) within a budget cycle + required: + - model + - total_requests + - total_tokens + - total_cost + properties: + model: + type: string + description: The model name + provider: + type: string + description: Provider that served the model; omitted when unknown + total_requests: + type: integer + format: int64 + description: Number of requests to this model in the cycle + total_tokens: + type: integer + format: int64 + description: Total tokens consumed by this model in the cycle + total_cost: + type: number + description: Total cost in dollars for this model in the cycle + +VirtualKeyModelUsage: + type: object + description: Per-model budgets and rate limit (with current usage) for a model governed under a virtual key + required: + - model_name + properties: + model_name: + type: string + description: The model this usage applies to + provider: + type: string + description: Provider this model config is scoped to; omitted when it applies to all providers + budgets: + type: [array, 'null'] + description: Budget quotas for this model + items: + $ref: '#/Budget' + rate_limit: + oneOf: + - $ref: '#/RateLimit' + - type: 'null' VirtualKeyResponse: type: object @@ -1698,3 +1775,77 @@ ListPricingOverridesResponse: count: type: integer description: Total number of overrides returned + +# Complexity Tier Boundaries + +ComplexityTierBoundaries: + type: object + description: Score thresholds for complexity tier classification. All values must satisfy 0 < simple_medium < medium_complex < complex_reasoning < 1. + additionalProperties: false + required: + - simple_medium + - medium_complex + - complex_reasoning + properties: + simple_medium: + type: number + description: Score boundary between SIMPLE and MEDIUM tiers + exclusiveMinimum: 0 + exclusiveMaximum: 1 + medium_complex: + type: number + description: Score boundary between MEDIUM and COMPLEX tiers + exclusiveMinimum: 0 + exclusiveMaximum: 1 + complex_reasoning: + type: number + description: Score boundary between COMPLEX and REASONING tiers + exclusiveMinimum: 0 + exclusiveMaximum: 1 + +ComplexityKeywordConfig: + type: object + description: | + Editable keyword lists used by the complexity analyzer. Only the four user-facing + dimensions are exposed; `reasoning_keywords` entries drive the reasoning tier + override. Matching is normalized to lowercase and duplicates are removed on save. + additionalProperties: false + required: + - code_keywords + - reasoning_keywords + - technical_keywords + - simple_keywords + properties: + code_keywords: + type: array + items: + type: string + minItems: 1 + reasoning_keywords: + type: array + items: + type: string + minItems: 1 + technical_keywords: + type: array + items: + type: string + minItems: 1 + simple_keywords: + type: array + items: + type: string + minItems: 1 + +ComplexityAnalyzerConfig: + type: object + description: Full runtime configuration for complexity routing analysis. + additionalProperties: false + required: + - tier_boundaries + - keywords + properties: + tier_boundaries: + $ref: '#/ComplexityTierBoundaries' + keywords: + $ref: '#/ComplexityKeywordConfig' diff --git a/docs/plugins/getting-started.mdx b/docs/plugins/getting-started.mdx index c84b5fba42..6eb025fdf9 100644 --- a/docs/plugins/getting-started.mdx +++ b/docs/plugins/getting-started.mdx @@ -54,8 +54,9 @@ This generates a `.so` file that exports specific functions matching Bifrost's p - `GetName() string` - Return the plugin name - `HTTPTransportPreHook()` - Intercept HTTP requests before they enter Bifrost core (HTTP transport only) - `HTTPTransportPostHook()` - Intercept HTTP responses after they exit Bifrost core (HTTP transport only) - - `PreLLMHook()` - Intercept requests before they reach providers - - `PostLLMHook()` - Process responses after provider calls + - `PreRequestHook()` v1.6.x+ - Once-per-request routing phase: decide provider/model/fallbacks + - `PreLLMHook()` - Intercept requests before they reach providers (runs per provider attempt) + - `PostLLMHook()` - Process responses after provider calls (runs per provider attempt) - `Cleanup() error` - Clean up resources on shutdown @@ -83,7 +84,7 @@ This means if you're running Bifrost on Linux AMD64, you must build your plugin 1. **Load** - Bifrost loads the `.so` file using Go's `plugin.Open()` 2. **Initialize** - Calls `Init()` with configuration from `config.json` -3. **Hook Execution** - Calls `PreLLMHook()` and `PostLLMHook()` for each request +3. **Hook Execution** - Calls `PreRequestHook()`, `PreLLMHook()` and `PostLLMHook()` for each request 4. **Cleanup** - Calls `Cleanup()` when Bifrost shuts down Plugins execute in a specific order: @@ -91,10 +92,11 @@ Plugins execute in a specific order: 1. `HTTPTransportPreHook` - Intercept HTTP requests (HTTP transport only) - 2. `PreLLMHook`/`PreMCPHook` - Executes in registration order, can short-circuit requests - 3. Provider call (if not short-circuited) - 4. `PostLLMHook`/`PostMCPHook` - Executes in reverse order of PreHooks - 5. `HTTPTransportPostHook` - Intercept HTTP responses (HTTP transport only, reverse order) + 2. `PreRequestHook` v1.6.x+ - **Once per request**, before any provider call. Routing decisions (provider/model/fallbacks) happen here and propagate to every attempt. + 3. `PreLLMHook`/`PreMCPHook` - Per provider attempt, registration order, can short-circuit requests + 4. Provider call (if not short-circuited) + 5. `PostLLMHook`/`PostMCPHook` - Per provider attempt, reverse order of PreHooks + 6. `HTTPTransportPostHook` - Intercept HTTP responses (HTTP transport only, reverse order) 1. `TransportInterceptor` - Modifies raw HTTP requests (HTTP transport only) diff --git a/docs/plugins/sequencing.mdx b/docs/plugins/sequencing.mdx index e261028b28..0355c5dc3f 100644 --- a/docs/plugins/sequencing.mdx +++ b/docs/plugins/sequencing.mdx @@ -32,6 +32,20 @@ graph LR Post-hooks execute in **reverse order** of pre-hooks (LIFO pattern). This means a `pre_builtin` plugin's `PreLLMHook` runs first, but its `PostLLMHook` runs last - ensuring proper cleanup and state unwinding. +### Routing layer order (PreRequestHook) + +`PreRequestHook` is the per-request **routing phase**. All routing-capable plugins fire here in registration order, and each one sees the routing decisions of those that ran before it. Built-in routing plugins are sequenced as follows within the `builtin` group: + +| Order | Plugin | Role | +|-------|--------|------| +| 4 | governance | Routing rules (CEL) + VK-scoped weighted load balancing | +| Higher | adaptive-loadbalancer (Enterprise) | Performance-based provider selection across the model catalog | +| 9 (last) | model-catalog-resolver | **Final fallback** — fills in `req.Provider` from the model catalog for unprefixed models when no earlier plugin picked one | + +The resolver runs last so CEL routing rules can match on `provider == ""` (the unresolved state) and earlier plugins always get the canonical bare model. After `PreRequestHook` returns, the core validates that `req.Provider` is non-empty — an unresolvable request returns a 400 with a clear error. + +Custom routing plugins can slot into this chain via `placement` + `order` like any other plugin. Place them in `pre_builtin` to override governance, or `post_builtin` to act as a custom fallback after all built-ins. + ### Ordering within a group Within each placement group, plugins are sorted by their `order` value (lower executes earlier). Plugins with the same order preserve their registration order. @@ -185,6 +199,6 @@ When in doubt, use the default `post_builtin` placement. Most custom plugins - l ## Next steps -- **[Writing a Go plugin](./writing-go-plugin)** - Build your first custom plugin with `PreLLMHook` and `PostLLMHook` +- **[Writing a Go plugin](./writing-go-plugin)** - Build your first custom plugin with `PreRequestHook`, `PreLLMHook`, and `PostLLMHook` - **[Writing a WASM plugin](./writing-wasm-plugin)** - Build a portable WASM plugin - **[Plugin architecture](../architecture/core/plugins)** - Deep dive into the plugin lifecycle and hook execution model diff --git a/docs/plugins/writing-go-plugin.mdx b/docs/plugins/writing-go-plugin.mdx index e9b2a514a7..d2462aa85f 100644 --- a/docs/plugins/writing-go-plugin.mdx +++ b/docs/plugins/writing-go-plugin.mdx @@ -152,6 +152,16 @@ func HTTPTransportStreamChunkHook(ctx *schemas.BifrostContext, req *schemas.HTTP } +// PreRequestHook is called once per top-level request (NOT per fallback attempt). +// This is the routing phase — use it for provider/model/fallback decisions. +// Mutations to req.Provider/req.Model/req.Fallbacks commit and propagate to every attempt. +// Errors are non-blocking (logged + skipped). +func PreRequestHook(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) error { + ctx.Log(schemas.LogLevelInfo, "PreRequestHook called") + // Plugins that don't participate in routing should just return nil + return nil +} + // PreLLMHook is called before the request is sent to the provider // This is where you can modify requests or short-circuit the flow func PreLLMHook(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) (*schemas.BifrostRequest, *schemas.LLMPluginShortCircuit, error) { @@ -268,6 +278,13 @@ func HTTPTransportStreamChunkHook(ctx *schemas.BifrostContext, req *schemas.HTTP } +// PreRequestHook is called once per top-level request (routing phase) +// Mutations to req.Provider/req.Model/req.Fallbacks commit across fallbacks +func PreRequestHook(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) error { + fmt.Println("PreRequestHook called") + return nil +} + // PreLLMHook is called before the request is sent to the provider // This is where you can modify requests or short-circuit the flow func PreLLMHook(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) (*schemas.BifrostRequest, *schemas.LLMPluginShortCircuit, error) { @@ -547,6 +564,58 @@ This function is **only called** when using `bifrost-http`. It's **not invoked** +#### `PreRequestHook(...)` v1.6.x+ + +Called **once per top-level request**, before any provider call and before `PreLLMHook`. This is the **routing phase**: it's where plugins decide which provider, model, and fallbacks the request should be sent to. + +Use this for: +- **Routing decisions**: governance rules, virtual-key load balancing, geo/tier routing +- **Provider resolution**: filling in `req.Provider` for unprefixed model names (the built-in `model-catalog-resolver` does this as the last routing layer) +- **Fallback chain construction**: populating `req.Fallbacks` based on policy + +**Why a separate hook from `PreLLMHook`:** + +| Aspect | `PreRequestHook` | `PreLLMHook` | +|---|---|---| +| Runs | Once per request | Once per provider attempt (re-runs on each fallback) | +| Provider/Model mutations | **Commit and propagate** to every attempt | No-op for `Provider`/`Model` (overwritten by core) | +| Use for | Routing decisions | Request transforms, caching, validation | +| Short-circuit | No | Yes (can return a synthetic response) | +| Error semantics | Non-blocking (logged, pipeline continues) | Non-blocking (logged, pipeline continues) | + +**Routing Example:** + +```go +func PreRequestHook(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) error { + provider, model, _ := req.GetRequestFields() + + // Route premium-tier requests to a faster provider + if tier := ctx.Value(schemas.BifrostContextKey("x-tier")); tier == "premium" && provider == "openai" { + req.SetProvider(schemas.Anthropic) + req.SetModel("claude-3-5-sonnet") + // Emit a routing-engine log entry so users can see why this decision was made + ctx.AppendRoutingEngineLog(schemas.RoutingEngineRoutingRule, schemas.LogLevelInfo, + fmt.Sprintf("Routed %s to anthropic/claude-3-5-sonnet (tier=premium)", model)) + schemas.AppendToContextList(ctx, schemas.BifrostContextKeyRoutingEnginesUsed, + schemas.RoutingEngineRoutingRule) + } + return nil +} +``` + +**Two helpers worth knowing when writing routing logic:** + +- `ctx.AppendRoutingEngineLog(engine, level, message)` — emits a structured log entry visible in observability tools. Use it to explain *why* a routing decision was made. +- `schemas.AppendToContextList(ctx, schemas.BifrostContextKeyRoutingEnginesUsed, engineName)` — records which routing engine(s) participated. Surfaces in telemetry as `routing.engines_used`. + + +**Plugin order matters for routing.** Built-in routing plugins run in this order within `PreRequestHook`: governance routing rules → governance VK load balancing → enterprise load balancer → model-catalog-resolver (final fallback). Custom plugins can slot in via `placement` + `order` — see [Plugin Sequencing](./sequencing). + + + +**Errors are non-blocking.** Returning a non-nil error from `PreRequestHook` logs a warning but does NOT fail the request — the pipeline continues with the next plugin. The core validates `req.Provider` after all `PreRequestHook` plugins have run; an unresolved provider returns a 400 to the caller. + + #### `PreLLMHook(...)` Called before each provider request. Use this to: diff --git a/docs/providers/aliasing-models.mdx b/docs/providers/aliasing-models.mdx index 532ff9da9e..99a8a11126 100644 --- a/docs/providers/aliasing-models.mdx +++ b/docs/providers/aliasing-models.mdx @@ -75,15 +75,52 @@ curl -X POST http://localhost:8080/api/providers/openai/keys \ }' ``` -The `aliases` field is a flat `string → string` map. The key is what your application sends; the value is what gets forwarded to the provider. There are no restrictions on what either side can be - deployments, ARNs, model IDs, version hashes, fine-tune IDs, anything. +Each alias maps the name your application sends to either a plain target string (the simplest form) or an object that additionally tags the alias with the canonical name used for pricing/logs, the model family used for provider routing, and any provider-specific overrides. The shorthand and the rich object form are interchangeable - both are accepted on the wire. + +```json +{ + "aliases": { + "fast-model": "gpt-4o-mini", + "best-model": { + "model_id": "12345-azure-deployment", + "model_name": "claude-sonnet-4-5", + "model_family": "anthropic", + "description": "Claude Sonnet 4.5 on our Azure tenant", + "api_version": "2024-10-21", + "endpoint": "env.AZURE_SECONDARY_ENDPOINT" + } + } +} +``` + +| Field | Type | Description | +|-------|------|-------------| +| `model_id` | string | **Required.** The wire identifier forwarded to the provider — deployment name, inference profile ARN, fine-tune ID, anything. | +| `model_name` | string | Canonical model name used by pricing and logs. Set this when `model_id` is opaque (e.g. an Azure deployment ID) so cost attribution still hits the catalog. | +| `model_family` | enum | Forces the family used for provider routing decisions (request shape, response parsing, etc.). One of: `anthropic`, `openai`, `mistral`, `cohere`, `gemini`, `gemma`, `llama`, `imagen`, `veo`, `nova`, `titan`. When unset, the family is auto-detected by substring matching against `model_name`, `model_id`, and the alias name. | +| `description` | string | Free-form note. Surfaced in the UI; not used for routing. | +| `region` | string / EnvVar | Per-alias region override (Bedrock + Vertex). Falls back to the key-level region when unset. | + +**Provider-specific overrides** (only set on aliases whose owning key matches the provider; validation rejects mismatches): + +| Provider | Field | Description | +|----------|-------|-------------| +| Azure | `api_version` | Overrides the `api-version` query parameter | +| Azure | `anthropic_version` | Overrides the `anthropic-version` header for Claude-on-Azure deployments | +| Azure | `endpoint` | Overrides the key-level Azure endpoint — useful when one credential spans deployments on multiple Azure resources | +| Vertex | `project_id` | Per-alias GCP project ID | +| Vertex | `project_number` | Per-alias GCP project number (required for fine-tuned models) | +| Bedrock | `inference_profile_arn` | Cross-region inference profile ARN invoked instead of the model ID | +| Replicate | `use_deployments_endpoint` | Routes the alias through Replicate's deployments endpoint | ### Validation rules Bifrost rejects an aliases map that violates any of these: -- **No empty strings** - both the alias name and its target must be non-empty +- **No empty strings** - both the alias name and `model_id` must be non-empty - **No leading or trailing whitespace** on either side - **No duplicate alias names** (checked case-insensitively) - `"GPT-4o"` and `"gpt-4o"` cannot both be keys in the same map +- **No provider mismatch on sub-configs** - an Azure-specific override (`api_version`, `endpoint`, …) can only appear on an alias whose owning key is an Azure key, and likewise for Vertex / Bedrock / Replicate ### Case-insensitive matching @@ -91,19 +128,38 @@ Alias lookup is case-insensitive. If your map has `"GPT-4O": "gpt-4o-2024-11-20" ### Tracking in responses -Every response includes both the original name and the resolved identifier in `extra_fields`: +Every response includes a `routing_info` block in `extra_fields` describing the routing decisions Bifrost made: ```json { "extra_fields": { - "original_model_requested": "best-model", - "resolved_model_used": "gpt-4o-2024-11-20", - "provider": "openai" + "routing_info": { + "provider": "openai", + "model": "best-model", + "key": "production-key", + "resolved_key_alias": { + "model_id": "gpt-4o-2024-11-20", + "model_name": "gpt-4o" + } + } } } ``` -If no alias matches, `resolved_model_used` equals `original_model_requested`. +| Field | Description | +|-------|-------------| +| `routing_info.provider` | Provider that handled the request | +| `routing_info.model` | Model name the caller sent (the LHS of the alias map when an alias matched) | +| `routing_info.key` | Human-friendly name of the key used | +| `routing_info.resolved_key_alias` | Present only when an alias matched. Carries the wire `model_id`, the canonical `model_name` (if set), and the resolved `model_family` (if set). | +| `routing_info.is_fallback` | `true` when the request was served by a fallback attempt rather than the primary | +| `routing_info.primary_provider` / `routing_info.primary_model` | Populated on fallback attempts with the primary attempt's provider/model | + +When no alias matches, `resolved_key_alias` is omitted and `model` carries the wire identifier directly. + + +`extra_fields.provider`, `extra_fields.original_model_requested`, and `extra_fields.resolved_model_used` are deprecated but still populated for backward compatibility. New consumers should read from `routing_info`. + --- @@ -333,13 +389,18 @@ ml-team → model="best-model" → Anthropic receives model="claude-3-5-sonnet-20241022" ``` -**Response `extra_fields` for tech-team + premium-vk:** +**Response `extra_fields.routing_info` for tech-team + premium-vk:** ```json { - "original_model_requested": "best-model", - "resolved_model_used": "gpt-5", - "provider": "openai" + "routing_info": { + "provider": "openai", + "model": "best-model", + "key": "high-tier-key", + "resolved_key_alias": { + "model_id": "gpt-5" + } + } } ``` -`original_model_requested` is always what the client originally sent. `resolved_model_used` is the final identifier that reached the provider API - after both routing and alias resolution. +`routing_info.model` is what the client sent (after any routing rule rewrites). `routing_info.resolved_key_alias.model_id` is the final identifier that reached the provider API - after both routing and key-level alias resolution. When no key-level alias matches, `resolved_key_alias` is omitted and the wire model equals `routing_info.model`. diff --git a/docs/providers/custom-pricing.mdx b/docs/providers/custom-pricing.mdx index eebfee4cdd..752e540947 100644 --- a/docs/providers/custom-pricing.mdx +++ b/docs/providers/custom-pricing.mdx @@ -106,6 +106,22 @@ For wildcard patterns, append a `*` at the end of the prefix. For example, `clau --- +## Lookup precedence + +When pricing resolves a request, it tries lookup candidates in this order against the catalog (built-in entries + your overrides): + +1. The alias's canonical `model_name` (`routing_info.resolved_key_alias.model_name`) +2. The alias's wire `model_id` (`routing_info.resolved_key_alias.model_id`) +3. The model the caller sent (`routing_info.model`) + +The first non-empty candidate that matches a catalog entry wins. The precedence solves the **opaque deployment ID** case: when an admin aliases an unrecognisable wire ID (e.g. an Azure deployment `12345-azure-prod`) to a catalog-known canonical name (e.g. `claude-sonnet-4-5`) via [Static Aliasing](/providers/aliasing-models), pricing hits the catalog via the canonical name even though the wire identifier wouldn't. + +When no key-level alias matches, candidates (1) and (2) are absent and the lookup falls straight through to the model the caller sent — preserving pre-alias behavior. + +**Overrides** are matched against the wire model (`model_id` when an alias matched, otherwise the caller-sent model) so per-deployment override pricing stays addressable regardless of how the catalog entry was found. + +--- + ## Request type filtering `request_types` is **required** and must contain at least one value. Only request types that have pricing support are accepted. Stream variants are treated identically to their base type - specifying `chat_completion` covers both streaming and non-streaming chat requests. diff --git a/docs/providers/provider-routing.mdx b/docs/providers/provider-routing.mdx index 3f74aed602..6e1d9960d8 100644 --- a/docs/providers/provider-routing.mdx +++ b/docs/providers/provider-routing.mdx @@ -836,20 +836,21 @@ This is how Bifrost achieves **intelligent cross-provider routing** without manu v1.5.0-prerelease7 and above**. -When a request includes a bare model name without a `provider/` prefix (e.g., `"model": "gpt-4o"` instead of `"model": "openai/gpt-4o"`), Bifrost automatically resolves the provider using the Model Catalog. Note that this default behavior is applied **after all other routing engines** have run. +When a request includes a bare model name without a `provider/` prefix (e.g., `"model": "gpt-4o"` instead of `"model": "openai/gpt-4o"`), Bifrost automatically resolves the provider using the Model Catalog. This default behavior is applied **after all other routing engines** have run — the built-in `model-catalog-resolver` PreRequestHook plugin is registered as the last routing layer (order 9 within `builtin`), so governance routing rules, VK load balancing, and enterprise LB all get first crack. ### How It Works 1. **Request arrives** without a provider prefix (e.g., `"model": "gpt-4o"`) -2. **Catalog lookup**: Bifrost calls `GetProvidersForModel("gpt-4o")` to find all providers that support the model -3. **Provider selected**: A provider from the catalog's available list is used (e.g., `openai`) -4. **Request continues**: The resolved `provider/model` string is used for load balancing and fallback handling +2. Governance, VK LB, and enterprise LB all run first; if any of them sets `req.Provider`, the resolver no-ops +3. **Catalog lookup** (if `req.Provider` is still empty): Bifrost calls `GetProvidersForModel("gpt-4o")` to find all providers that support the model +4. **Provider selected**: If the request came in via an integration route (OpenAI / Anthropic / GenAI / Bedrock / Cohere) and the catalog includes that integration's canonical provider in the candidate list, it is preferred. Otherwise the first candidate is selected. +5. **Request continues**: The resolved `provider/model` is used for the provider call, fallback handling, and Level 2 key selection. This is logged as the **`model-catalog`** routing engine in telemetry and routing logs, with a message like: ``` No provider specified for model gpt-4o, found 3 options in model catalog: -[openai, azure, groq], selecting first: openai +[openai, azure, groq], selected: openai ``` ### Example @@ -873,6 +874,18 @@ curl -X POST http://localhost:8080/v1/chat/completions \ prefix. +### Routing allowlist enforcement + +When a Virtual Key has `provider_configs`, governance publishes the VK's allowed-provider set to the request context (`BifrostContextKeyRoutingAllowedProviders`). The constraint is then enforced at **two levels**: + +1. **Cooperative filtering (observability-first):** Enterprise LB and the model-catalog-resolver intersect their catalog candidates with the allowlist before picking a provider. This produces clean routing-engine logs explaining *why* a candidate was excluded ("filtered N catalog candidates by routing allowlist"). + +2. **Hard enforcement in core:** After all `PreRequestHook` plugins have run, the core validates the final `req.Provider` against the allowlist. If `req.Provider` isn't in the allowlist, the request fails with HTTP 400. Fallbacks that target non-allowed providers are silently filtered out. + +**Why two levels:** cooperative filtering surfaces routing decisions in observability; core enforcement makes the constraint a *guarantee* that no plugin (or user-specified `provider/model` prefix) can bypass. A user request for `model: "anthropic/claude-3"` against a VK that allows only `[openai, azure]` is rejected by core enforcement even though the user provided an explicit prefix. + +Custom routing plugins can set the same context key to constrain downstream routing for any reason — geo restrictions, A/B test cohorts, tier-based gating, etc. The semantics are **fail-closed**: setting `BifrostContextKeyRoutingAllowedProviders` to an empty slice means "no provider is permitted for this request" → HTTP 400. + --- ## Governance-based Routing @@ -1189,60 +1202,67 @@ This means key-level optimization works regardless of how the provider was chose flowchart TD Start["Request: gpt-4o"] - subgraph Governance["Governance Plugin (HTTPTransportIntercept)"] + subgraph PreReq["PreRequestHook Phase (once per request, registration order)"] HasVK{"Has VK with
provider_configs?"} - GovRoute["Provider Selection:
Weighted random"] - AddPrefix["Add prefix:
azure/gpt-4o"] - end - - subgraph LB1["Load Balancer Level 1 (Middleware)"] - PrefixCheck{"Has provider
prefix?"} - LBProvider["Provider Selection:
Performance-based"] - AddLBPrefix["Add prefix:
openai/gpt-4o"] + GovRoute["Governance:
Routing rules + VK weighted random"] + AddPrefix["Set req.Provider/Model:
azure/gpt-4o"] + PrefixCheck{"req.Provider
already set?"} + LBProvider["Enterprise LB:
Performance-based selection"] + AddLBPrefix["Set req.Provider/Model:
openai/gpt-4o"] + Resolver["model-catalog-resolver:
Fill from catalog (last fallback)"] end - subgraph LB2["Load Balancer Level 2 (Key Selector)"] + subgraph LB2["Load Balancer Level 2 (Key Selector, in core)"] GetKeys["Get available keys
for selected provider"] ScoreKeys["Score keys by
performance metrics"] SelectKey["Select best key"] end Start --> HasVK - HasVK -->|Yes| GovRoute --> AddPrefix + HasVK -->|Yes| GovRoute --> AddPrefix --> PrefixCheck HasVK -->|No| PrefixCheck - AddPrefix --> PrefixCheck - PrefixCheck -->|Yes, skip Level 1| GetKeys - PrefixCheck -->|No| LBProvider --> AddLBPrefix --> GetKeys + PrefixCheck -->|Yes, skip LB Level 1| Resolver + PrefixCheck -->|No| LBProvider --> AddLBPrefix --> Resolver + Resolver --> GetKeys GetKeys --> ScoreKeys --> SelectKey --> Execute["Execute request
with selected provider + key"] ``` ### Execution Order -1. **HTTPTransportIntercept** (Governance Plugin - Provider Level) - - Runs first in the request pipeline - - Checks if Virtual Key has `provider_configs` - - If yes: adds provider prefix (e.g., `azure/gpt-4o`) - - **Result**: Provider is selected by governance rules - -2. **Middleware** (Load Balancing Plugin - Provider Level / Direction) - - Runs after HTTPTransportIntercept - - Checks if model string contains "/" - - If yes: **skips provider selection** (already determined by governance or user) - - If no: performs performance-based provider selection - - **Result**: Provider prefix added if not already present - -3. **KeySelector** (Load Balancing - Key Level / Route) - - **Always runs** during request execution in Bifrost core - - Gets all keys for the selected provider - - Filters keys based on model restrictions +All three routing layers (governance, enterprise LB Level 1, model-catalog-resolver) now run inside a single **PreRequestHook** phase that fires **once per top-level request**, before any provider call and before per-attempt hooks. Within that phase, plugins execute in placement + order: + +1. **Governance Plugin** (PreRequestHook, builtin order 4) + - Evaluates routing rules (CEL expressions, scope hierarchy) + - If Virtual Key has `provider_configs`: performs weighted random provider selection + - **Result**: `req.Provider`/`req.Model` set; `req.Fallbacks` populated + +2. **Enterprise Load Balancer Level 1** (PreRequestHook, builtin) + - Runs after governance + - If `req.Provider` is already set (by governance or by an explicit `provider/model` prefix from the user): **skips provider selection** + - If not: performs performance-based provider selection across catalog providers + - **Result**: `req.Provider`/`req.Model` set if previously empty + +3. **model-catalog-resolver** (PreRequestHook, builtin order 9 — final fallback) + - Runs last + - If `req.Provider` is still empty: looks up the model in the catalog and picks a provider (preferring the integration's canonical provider when the request came in via an integration route) + - Emits a `model-catalog` routing-engine log entry + - **Result**: Always leaves `req.Provider` populated when the catalog knows about the model + +4. **Empty-provider validation** (core, after PreRequestHook) + - If `req.Provider` is still empty: returns 400 to the caller with a clear error + +5. **Load Balancer Level 2** (Key Selector — core, per provider attempt) + - **Always runs** during request execution + - Gets all keys for the selected provider, filters by model restrictions - Scores each key by performance metrics - Selects best key using weighted random + exploration - **Result**: Optimal key selected within the provider - **Important**: Even when governance specifies `azure/gpt-4o`, load balancing - **still optimizes which Azure key to use** based on performance metrics. This - is the power of the two-level architecture! + **Important**: Even when governance specifies `azure/gpt-4o` in PreRequestHook, + load balancing Level 2 **still optimizes which Azure key to use** based on + performance metrics. The two-level architecture is preserved — only the + *layer* where Level 1 runs has moved from a middleware to PreRequestHook. ### Example Scenarios @@ -1395,35 +1415,37 @@ Routing Rules provide sophisticated, expression-based control over request routi flowchart TD Start["Request: model + provider"] - subgraph Rules["1. Routing Rules Layer (Evaluated First)"] - RuleMatch{"CEL Expression
Matches?"} - RuleDecision["Override:
New provider/model/fallbacks"] - NoMatch["No match:
Continue to Governance"] - end - - subgraph Gov["2. Governance Layer (if no routing rule matched)"] - VKValidation["Virtual Key Validation"] - GovRouting["Provider Governance Routing
(weighted random)"] + subgraph PreReq["PreRequestHook Phase (once per request)"] + direction TB + subgraph Gov["Governance Plugin"] + RuleMatch{"CEL Routing Rule
Matches?"} + RuleDecision["Override:
provider/model/fallbacks"] + VKValidation["Virtual Key Validation"] + GovRouting["VK Provider Selection
(weighted random)"] + end + LB1["Enterprise LB Level 1:
Provider Selection
(skipped if provider already set)"] + Resolver["model-catalog-resolver:
Fill provider from catalog
(final fallback)"] end - subgraph LB["3. Load Balancing Layer"] - LB1["Level 1: Provider Selection"] - LB2["Level 2: Key Selection"] - end + LB2["LB Level 2: Key Selection
(core, per attempt)"] Start --> RuleMatch RuleMatch -->|Yes| RuleDecision --> LB1 - RuleMatch -->|No| NoMatch --> VKValidation --> GovRouting --> LB1 - LB1 --> LB2 --> Execute["Execute with
selected provider + key"] + RuleMatch -->|No| VKValidation --> GovRouting --> LB1 + LB1 --> Resolver --> LB2 --> Execute["Execute with
selected provider + key"] ``` ### How It Works +All routing layers below execute inside the **PreRequestHook** phase in registration order; routing rules run first within the governance plugin's hook body, before VK load balancing: + 1. **Routing rules evaluate first** in scope precedence order (VirtualKey → Team → Customer → Global) -2. **If a routing rule matches**: provider/model/fallbacks are overridden, governance provider_configs are skipped -3. **If no routing rule matches**: governance provider selection runs (weighted random) -4. **Load balancing Level 1**: skipped if provider already determined (has "/" prefix) -5. **Load balancing Level 2** (key selection): always runs to select the best key within the determined provider +2. **If a routing rule matches**: provider/model/fallbacks are overridden, the VK `provider_configs` weighted selection is skipped +3. **If no routing rule matches**: VK provider selection runs (weighted random) +4. **Enterprise LB Level 1**: skipped if `req.Provider` is already set; otherwise performs performance-based selection +5. **model-catalog-resolver**: last fallback — fills `req.Provider` from the catalog if no earlier plugin set it +6. **Empty-provider validation** (core): returns 400 if `req.Provider` is still empty after the phase +7. **Load balancing Level 2** (key selection, core, per attempt): always runs to select the best key within the determined provider ### Available CEL Variables @@ -1501,6 +1523,10 @@ Within each scope, rules are sorted by **priority** (ascending: 0 before 10). | **Priority Ordering** | Lower priority evaluated first within same scope | | **Capacity Awareness** | Access real-time budget and rate limit usage percentages | + +For complexity-based routing driven by request content, see [Complexity Router](/features/governance/complexity-router). It adds a `complexity_tier` CEL variable that lets routing rules steer SIMPLE, MEDIUM, COMPLEX, and REASONING requests to different models. + + ### Integration with Governance Routing Rules execute **before** governance provider selection and can override it: diff --git a/docs/providers/request-options.mdx b/docs/providers/request-options.mdx index 7fa85b3fef..f74c4e86ef 100644 --- a/docs/providers/request-options.mdx +++ b/docs/providers/request-options.mdx @@ -13,6 +13,7 @@ Bifrost provides request options that control behavior, enable features, and pas | `BifrostContextKeyVirtualKey` | `x-bf-vk` | `string` | Virtual key identifier for governance | | `BifrostContextKeyAPIKeyName` | `x-bf-api-key` | `string` | Explicit API key name selection | | `BifrostContextKeyAPIKeyID` | `x-bf-api-key-id` | `string` | Explicit API key ID selection (takes priority over name) | +| `BifrostContextKeyDirectKey` | `x-bf-direct-key` (+ `Authorization` / `x-api-key` / `x-goog-api-key`) | `schemas.Key` | Use a caller-supplied raw provider key directly, bypassing the registered key pool. On the gateway, requires `allow_direct_keys` server-side | | `BifrostContextKeySessionID` | `x-bf-session-id` | `string` | Session ID for key stickiness (requires KV store) | | `BifrostContextKeySessionTTL` | `x-bf-session-ttl` | `time.Duration` | Session-to-key cache TTL (duration string or seconds) | | `BifrostContextKeyRequestID` | `x-request-id` | `string` | Custom request ID for tracking | @@ -161,6 +162,59 @@ Input: messages,
+### Direct API Key + +**Context Key:** `BifrostContextKeyDirectKey` +**Header:** `x-bf-direct-key` (plus the raw key in `Authorization`, `x-api-key`, or `x-goog-api-key`) +**Type:** `schemas.Key` +**Required:** No + +Supply a raw provider API key with the request and have Bifrost use it directly, bypassing the registered key pool entirely. Unlike [API Key Selection](#api-key-selection) (which references a *stored* key by ID or name), this passes the secret itself — useful for multi-tenant setups where each caller brings their own provider credentials. + +On the **gateway**, this is double-gated and off by default: + +1. The server admin must enable `allow_direct_keys` (see [client config](../deployment-guides/config-json/client)). +2. The caller must send `x-bf-direct-key: true` on the request **and** the raw provider key in one of `Authorization: Bearer `, `x-api-key`, or `x-goog-api-key`. + +Both conditions must hold; neither alone takes effect. Virtual keys (`sk-bf-*`) in those headers are **not** treated as direct keys — they continue to resolve as virtual keys. + +In the **Go SDK** there is no flag: set `BifrostContextKeyDirectKey` to a `schemas.Key` and it is used as-is. + + +A direct key bypasses Bifrost's key management entirely — **no governance** (virtual key budgets, rate limits, provider/model allow-lists, routing rules), **no weighted selection**, and **no rotation or fallback** across alternate keys. The key is used exactly as supplied. Enable `allow_direct_keys` only when you intend callers to manage their own provider credentials. + + + + +```bash +curl --location 'http://localhost:8080/v1/chat/completions' \ +--header 'x-bf-direct-key: true' \ +--header 'Authorization: Bearer sk-your-real-openai-key' \ +--header 'Content-Type: application/json' \ +--data '{ + "model": "openai/gpt-4o-mini", + "messages": [{"role": "user", "content": "Hello!"}] +}' +``` + + +```go +ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) +ctx.SetValue(schemas.BifrostContextKeyDirectKey, schemas.Key{ + Value: schemas.EnvVar{Val: "sk-your-real-openai-key"}, + Models: []string{}, + Weight: 1.0, +}) + +response, err := client.ChatCompletionRequest(ctx, &schemas.BifrostChatRequest{ + Provider: schemas.OpenAI, + Model: "gpt-4o-mini", + Input: messages, +}) +``` + + + ### Session Stickiness (Session ID) **Context Key:** `BifrostContextKeySessionID` @@ -1233,6 +1287,7 @@ Bifrost maintains a security denylist of headers that are **never** forwarded to - `x-goog-api-key` (when used via `x-bf-eh-*`) - `x-bf-api-key` (when used via `x-bf-eh-*`) - `x-bf-vk` (when used via `x-bf-eh-*`) +- `x-bf-direct-key` (when used via `x-bf-eh-*`) ## Internal Context Keys diff --git a/docs/providers/routing-rules.mdx b/docs/providers/routing-rules.mdx index 20fc98809a..ea51bb41b1 100644 --- a/docs/providers/routing-rules.mdx +++ b/docs/providers/routing-rules.mdx @@ -116,6 +116,28 @@ tokens_used < 50 // Route to fast provider when below 50% token limit request > 90 // Switch providers when request limit near max ``` +#### Complexity Routing + +```cel +complexity_tier // Automatically-classified request tier: "SIMPLE", "MEDIUM", "COMPLEX", or "REASONING" +``` + +Bifrost analyzes each request's prompt content and assigns a tier before routing rules are evaluated. This lets you route cheap/fast requests to small models and deep reasoning tasks to frontier models with no application-side changes. + + +If complexity analysis is unavailable for a request (e.g. unparseable body), `complexity_tier` is treated as **unknown** by the evaluator — the rule does not match and evaluation falls through. Rules using other variables are unaffected. + + +**Examples:** +```cel +complexity_tier == "REASONING" // Only frontier-worthy tasks +complexity_tier in ["COMPLEX", "REASONING"] // Complex and above +!(complexity_tier in ["SIMPLE", "MEDIUM"]) // Same as above, negated form +complexity_tier == "REASONING" && team_name == "research" // Scoped to a team +``` + +See [Complexity Router](/features/governance/complexity-router) for how tiers are computed and how to tune the thresholds and keyword lists. + #### How Capacity Metrics Are Resolved Each variable reflects **current usage as a percentage of the configured limit** for the request's provider and model combination. Values above 100 mean the limit is exhausted. diff --git a/docs/providers/supported-providers/openai.mdx b/docs/providers/supported-providers/openai.mdx index 3a4d7075ef..ce7a2a71a0 100644 --- a/docs/providers/supported-providers/openai.mdx +++ b/docs/providers/supported-providers/openai.mdx @@ -523,6 +523,8 @@ Operations: GET `/v1/models` - Lists available models with metadata. Model IDs in Bifrost responses are prefixed with `openai/` (e.g., `openai/gpt-4o`). Results are aggregated from all configured API keys. No request body or parameters required. +When the upstream provider returns richer model objects, Bifrost preserves those top-level fields in the native `/v1/models` response, including fields such as `name`, `description`, `context_length`, `architecture`, `supported_parameters`, and provider-specific `pricing`. Clients can use this metadata for model pickers, context limits, modality or tool support, and pricing displays without a separate `metadata` wrapper. + --- # 13. Video Generation diff --git a/docs/providers/supported-providers/openrouter.mdx b/docs/providers/supported-providers/openrouter.mdx index e6e2f0949a..64a5ebb69f 100644 --- a/docs/providers/supported-providers/openrouter.mdx +++ b/docs/providers/supported-providers/openrouter.mdx @@ -188,6 +188,8 @@ Lists 100+ models available through OpenRouter, including: - Mistral - And many more +Bifrost preserves OpenRouter's richer top-level model metadata in the native `/v1/models` response, including fields such as context limits, architecture, supported parameters, provider-specific pricing, and other upstream model attributes that UI pickers and agent runtimes commonly read. + --- # 5. Embeddings diff --git a/docs/quickstart/go-sdk/context-keys.mdx b/docs/quickstart/go-sdk/context-keys.mdx index 0f7101a679..b8c3ccb6a4 100644 --- a/docs/quickstart/go-sdk/context-keys.mdx +++ b/docs/quickstart/go-sdk/context-keys.mdx @@ -78,6 +78,22 @@ Skip key selection entirely and pass an empty key to the provider. Useful for pr bfCtx.SetValue(schemas.BifrostContextKeySkipKeySelection, true) ``` +### Direct Key + +Supply a raw provider API key to use directly, bypassing the registered key pool. Set `BifrostContextKeyDirectKey` to a `schemas.Key` and Bifrost uses it as-is — there is no flag to enable in the SDK (the gateway's `allow_direct_keys` setting only gates the HTTP header path that populates this same key). + +```go +bfCtx.SetValue(schemas.BifrostContextKeyDirectKey, schemas.Key{ + Value: schemas.EnvVar{Val: "sk-your-real-openai-key"}, + Models: []string{}, + Weight: 1.0, +}) +``` + + +A direct key bypasses governance, weighted selection, and rotation/fallback — the key is used exactly as supplied. Prefer [API Key Selection](#api-key-selection) when you want to reference a Bifrost-managed key instead of passing the secret. + + ### Session Stickiness (Session ID) Bind a session to a specific API key so that requests with the same session ID consistently use the same key. Useful for predictable rate-limit buckets, cost attribution per user, and consistent model routing per session. @@ -344,6 +360,7 @@ func makeRequest(client *bifrost.Bifrost) { | `BifrostContextKeyVirtualKey` | `string` | Set | Virtual key identifier for governance | | `BifrostContextKeyAPIKeyName` | `string` | Set | Explicit API key name selection | | `BifrostContextKeyAPIKeyID` | `string` | Set | Explicit API key ID selection (priority over name) | +| `BifrostContextKeyDirectKey` | `schemas.Key` | Set | Use a caller-supplied raw provider key directly, bypassing the key pool | | `BifrostContextKeyRequestID` | `string` | Set | Custom request ID for tracking | | `BifrostContextKeyFallbackRequestID` | `string` | Read | Request ID used for fallback attempt | | `BifrostContextKeySkipKeySelection` | `bool` | Set | Skip key selection entirely | diff --git a/examples/plugins/hello-world/main.go b/examples/plugins/hello-world/main.go index 2f464e2a7d..a08fe013f0 100644 --- a/examples/plugins/hello-world/main.go +++ b/examples/plugins/hello-world/main.go @@ -57,6 +57,10 @@ func HTTPTransportStreamChunkHook(ctx *schemas.BifrostContext, req *schemas.HTTP return chunk, nil } +func PreRequestHook(_ *schemas.BifrostContext, _ *schemas.BifrostRequest) error { + return nil +} + func PreLLMHook(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) (*schemas.BifrostRequest, *schemas.LLMPluginShortCircuit, error) { value1 := ctx.Value(transportPreHookKey) fmt.Println("value1:", value1) diff --git a/examples/plugins/llm-only/main.go b/examples/plugins/llm-only/main.go index 65d47c2020..90ec9da184 100644 --- a/examples/plugins/llm-only/main.go +++ b/examples/plugins/llm-only/main.go @@ -67,6 +67,11 @@ func GetName() string { return "llm-only" } +// PreRequestHook is the per-request routing phase. This example plugin doesn't route. +func PreRequestHook(_ *schemas.BifrostContext, _ *schemas.BifrostRequest) error { + return nil +} + // PreLLMHook is called before the LLM provider is invoked // This example demonstrates request modification and logging func PreLLMHook(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) (*schemas.BifrostRequest, *schemas.LLMPluginShortCircuit, error) { diff --git a/examples/plugins/multi-interface/main.go b/examples/plugins/multi-interface/main.go index d593b778ef..e8b3a14292 100644 --- a/examples/plugins/multi-interface/main.go +++ b/examples/plugins/multi-interface/main.go @@ -155,6 +155,11 @@ func HTTPTransportPostHook(ctx *schemas.BifrostContext, req *schemas.HTTPRequest // LLMPlugin Interface // ============================================================================ +// PreRequestHook is the per-request routing phase. This example plugin doesn't route. +func PreRequestHook(_ *schemas.BifrostContext, _ *schemas.BifrostRequest) error { + return nil +} + // PreLLMHook is called before the LLM provider is invoked func PreLLMHook(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) (*schemas.BifrostRequest, *schemas.LLMPluginShortCircuit, error) { if !pluginConfig.EnableLLMHooks { diff --git a/flake.lock b/flake.lock index d968f41e45..7cab390cf7 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1779952895, - "narHash": "sha256-j0P9+h7HX67KNlGki6puFfx8xO6wx4Jz23jXg3dpfCw=", + "lastModified": 1780965770, + "narHash": "sha256-g13MEx3zFQG2HiYiEuM3QzLvGwnYiCuzRrIVYrTeSro=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "7bbe929cc678c8d32f0c23e2dffb5b4f2e68a9b5", + "rev": "6a817a86f2e8684a19f9982b4598b3b65a3d38e6", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index ecb94e87e3..b6ae37956c 100644 --- a/flake.nix +++ b/flake.nix @@ -17,6 +17,18 @@ "aarch64-darwin" # 64-bit ARM macOS ]; + # Temporary workaround until nixpkgs includes Go 1.26.4. + go_1_26_4_overlay = final: prev: { + go_1_26 = prev.go_1_26.overrideAttrs (oldAttrs: rec { + version = "1.26.4"; + src = final.fetchurl { + url = "https://go.dev/dl/go${version}.src.tar.gz"; + sha256 = "0bb089d2bfszfc8r4cra94qsdb8x5y69dyw1m3k344gwzcr8lrjg"; + }; + }); + go = final.go_1_26; + }; + # Helper for providing system-specific attributes forEachSupportedSystem = f: @@ -27,6 +39,7 @@ # Provides a system-specific, configured Nixpkgs pkgs = import inputs.nixpkgs { inherit system; + overlays = [ go_1_26_4_overlay ]; # Enable using unfree packages config.allowUnfree = true; }; diff --git a/framework/configstore/clientconfig.go b/framework/configstore/clientconfig.go index 80874ee674..aeb880b322 100644 --- a/framework/configstore/clientconfig.go +++ b/framework/configstore/clientconfig.go @@ -1386,14 +1386,42 @@ type frameworkConfigHashPayload struct { PricingSyncInterval *int64 `json:"pricing_sync_interval"` } +type frameworkConfigHashPayloadWithMCP struct { + PricingURL *string `json:"pricing_url"` + ModelParametersURL *string `json:"model_parameters_url"` + PricingSyncInterval *int64 `json:"pricing_sync_interval"` + MCPLibraryURL *string `json:"mcp_library_url"` + MCPLibrarySyncInterval *int64 `json:"mcp_library_sync_interval"` +} + +// FrameworkConfigHashOptions adds optional framework config fields to the +// config.json change-detection hash while preserving the legacy pricing-only +// hash when omitted. +type FrameworkConfigHashOptions struct { + MCPLibraryURL *string + MCPLibrarySyncInterval *int64 +} + // GenerateFrameworkConfigHash generates a SHA256 hash for a framework config. // This is used to detect changes to framework config between config.json and database. -func GenerateFrameworkConfigHash(pricingURL *string, modelParametersURL *string, pricingSyncInterval *int64) (string, error) { - data, err := sonic.Marshal(frameworkConfigHashPayload{ - PricingURL: pricingURL, - ModelParametersURL: modelParametersURL, - PricingSyncInterval: pricingSyncInterval, - }) +func GenerateFrameworkConfigHash(pricingURL *string, modelParametersURL *string, pricingSyncInterval *int64, opts ...FrameworkConfigHashOptions) (string, error) { + var data []byte + var err error + if len(opts) > 0 { + data, err = sonic.Marshal(frameworkConfigHashPayloadWithMCP{ + PricingURL: pricingURL, + ModelParametersURL: modelParametersURL, + PricingSyncInterval: pricingSyncInterval, + MCPLibraryURL: opts[0].MCPLibraryURL, + MCPLibrarySyncInterval: opts[0].MCPLibrarySyncInterval, + }) + } else { + data, err = sonic.Marshal(frameworkConfigHashPayload{ + PricingURL: pricingURL, + ModelParametersURL: modelParametersURL, + PricingSyncInterval: pricingSyncInterval, + }) + } if err != nil { return "", err } @@ -1403,10 +1431,9 @@ func GenerateFrameworkConfigHash(pricingURL *string, modelParametersURL *string, // AuthConfig represents configured auth config for Bifrost dashboard type AuthConfig struct { - AdminUserName *schemas.EnvVar `json:"admin_username"` - AdminPassword *schemas.EnvVar `json:"admin_password"` - IsEnabled bool `json:"is_enabled"` - DisableAuthOnInference bool `json:"disable_auth_on_inference"` + AdminUserName *schemas.EnvVar `json:"admin_username"` + AdminPassword *schemas.EnvVar `json:"admin_password"` + IsEnabled bool `json:"is_enabled"` } // ConfigMap maps provider names to their configurations. @@ -1415,14 +1442,15 @@ type ConfigMap map[schemas.ModelProvider]ProviderConfig // GovernanceConfig contains governance entities loaded from the config store or // reconciled from config.json. type GovernanceConfig struct { - VirtualKeys []tables.TableVirtualKey `json:"virtual_keys"` - Teams []tables.TableTeam `json:"teams"` - Customers []tables.TableCustomer `json:"customers"` - Budgets []tables.TableBudget `json:"budgets"` - RateLimits []tables.TableRateLimit `json:"rate_limits"` - ModelConfigs []tables.TableModelConfig `json:"model_configs"` - Providers []tables.TableProvider `json:"providers"` - RoutingRules []tables.TableRoutingRule `json:"routing_rules"` - PricingOverrides []tables.TablePricingOverride `json:"pricing_overrides,omitempty"` - AuthConfig *AuthConfig `json:"auth_config,omitempty"` + VirtualKeys []tables.TableVirtualKey `json:"virtual_keys"` + Teams []tables.TableTeam `json:"teams"` + Customers []tables.TableCustomer `json:"customers"` + Budgets []tables.TableBudget `json:"budgets"` + RateLimits []tables.TableRateLimit `json:"rate_limits"` + ModelConfigs []tables.TableModelConfig `json:"model_configs"` + Providers []tables.TableProvider `json:"providers"` + RoutingRules []tables.TableRoutingRule `json:"routing_rules"` + PricingOverrides []tables.TablePricingOverride `json:"pricing_overrides,omitempty"` + AuthConfig *AuthConfig `json:"auth_config,omitempty"` + ComplexityAnalyzerConfig *ComplexityAnalyzerConfig `json:"complexity_analyzer_config,omitempty"` } diff --git a/framework/configstore/complexityconfig.go b/framework/configstore/complexityconfig.go new file mode 100644 index 0000000000..0bc0079630 --- /dev/null +++ b/framework/configstore/complexityconfig.go @@ -0,0 +1,130 @@ +package configstore + +import ( + "encoding/json" + "fmt" + "sort" + "strings" +) + +// ComplexityTierBoundaries defines score thresholds for complexity tier classification. +type ComplexityTierBoundaries struct { + SimpleMedium float64 `json:"simple_medium"` + MediumComplex float64 `json:"medium_complex"` + ComplexReasoning float64 `json:"complex_reasoning"` +} + +// Validate checks that tier boundaries are ordered and inside the analyzer score range. +func (b *ComplexityTierBoundaries) Validate() error { + if b == nil { + return nil + } + if !(0 < b.SimpleMedium && + b.SimpleMedium < b.MediumComplex && + b.MediumComplex < b.ComplexReasoning && + b.ComplexReasoning < 1) { + return fmt.Errorf( + "tier boundaries must satisfy 0 < simple_medium (%.4f) < medium_complex (%.4f) < complex_reasoning (%.4f) < 1", + b.SimpleMedium, b.MediumComplex, b.ComplexReasoning, + ) + } + return nil +} + +// ComplexityEditableKeywordConfig contains the user-editable keyword lists. +type ComplexityEditableKeywordConfig struct { + CodeKeywords []string `json:"code_keywords"` + ReasoningKeywords []string `json:"reasoning_keywords"` + TechnicalKeywords []string `json:"technical_keywords"` + SimpleKeywords []string `json:"simple_keywords"` +} + +// ComplexityAnalyzerConfig is the persisted runtime configuration for the complexity analyzer. +type ComplexityAnalyzerConfig struct { + TierBoundaries ComplexityTierBoundaries `json:"tier_boundaries"` + Keywords ComplexityEditableKeywordConfig `json:"keywords"` +} + +// Validate checks that the config is internally consistent. +func (c *ComplexityAnalyzerConfig) Validate() error { + if c == nil { + return nil + } + if err := c.TierBoundaries.Validate(); err != nil { + return err + } + + var missing []string + if len(c.Keywords.CodeKeywords) == 0 { + missing = append(missing, "code_keywords") + } + if len(c.Keywords.ReasoningKeywords) == 0 { + missing = append(missing, "reasoning_keywords") + } + if len(c.Keywords.TechnicalKeywords) == 0 { + missing = append(missing, "technical_keywords") + } + if len(c.Keywords.SimpleKeywords) == 0 { + missing = append(missing, "simple_keywords") + } + if len(missing) > 0 { + return fmt.Errorf("keyword lists must be non-empty: %s", strings.Join(missing, ", ")) + } + return nil +} + +// Normalized returns a canonical copy suitable for persistence and runtime use. +func (c *ComplexityAnalyzerConfig) Normalized() ComplexityAnalyzerConfig { + if c == nil { + return ComplexityAnalyzerConfig{} + } + return ComplexityAnalyzerConfig{ + TierBoundaries: c.TierBoundaries, + Keywords: ComplexityEditableKeywordConfig{ + CodeKeywords: normalizeComplexityKeywordList(c.Keywords.CodeKeywords), + ReasoningKeywords: normalizeComplexityKeywordList(c.Keywords.ReasoningKeywords), + TechnicalKeywords: normalizeComplexityKeywordList(c.Keywords.TechnicalKeywords), + SimpleKeywords: normalizeComplexityKeywordList(c.Keywords.SimpleKeywords), + }, + } +} + +// DecodeComplexityAnalyzerConfig decodes raw JSON into a normalized, validated config. +func DecodeComplexityAnalyzerConfig(data []byte) (*ComplexityAnalyzerConfig, error) { + if len(data) == 0 { + return nil, nil + } + + var cfg ComplexityAnalyzerConfig + if err := json.Unmarshal(data, &cfg); err != nil { + return nil, fmt.Errorf("failed to unmarshal complexity analyzer config: %w", err) + } + + normalized := cfg.Normalized() + if err := normalized.Validate(); err != nil { + return nil, fmt.Errorf("invalid complexity analyzer config: %w", err) + } + return &normalized, nil +} + +func normalizeComplexityKeywordList(values []string) []string { + if len(values) == 0 { + return nil + } + + seen := make(map[string]struct{}, len(values)) + out := make([]string, 0, len(values)) + for _, value := range values { + normalized := strings.ToLower(strings.TrimSpace(value)) + if normalized == "" { + continue + } + if _, ok := seen[normalized]; ok { + continue + } + seen[normalized] = struct{}{} + out = append(out, normalized) + } + sort.Strings(out) + return out +} diff --git a/framework/configstore/config.go b/framework/configstore/config.go index d40ba51bd3..3a2fd631d5 100644 --- a/framework/configstore/config.go +++ b/framework/configstore/config.go @@ -27,7 +27,7 @@ func (c *Config) UnmarshalJSON(data []byte) error { type TempConfig struct { Enabled bool `json:"enabled"` Type ConfigStoreType `json:"type"` - Config json.RawMessage `json:"config"` // Keep as raw JSON + Config json.RawMessage `json:"config"` } var temp TempConfig diff --git a/framework/configstore/encryption_test.go b/framework/configstore/encryption_test.go index 4b5c553904..6a795c6fca 100644 --- a/framework/configstore/encryption_test.go +++ b/framework/configstore/encryption_test.go @@ -775,7 +775,7 @@ func TestEncryptPlaintextKeys_BedrockFields_EncryptsAndDecryptsCorrectly(t *test assert.Equal(t, "us-west-2", found.BedrockKeyConfig.Region.GetValue()) require.NotNil(t, found.BedrockKeyConfig.ARN) assert.Equal(t, "arn:aws:iam::123456789:role/bedrock", found.BedrockKeyConfig.ARN.GetValue()) - assert.Equal(t, "profile-claude", found.Aliases["claude-3"]) + assert.Equal(t, "profile-claude", found.Aliases["claude-3"].ModelID) require.NotNil(t, found.BedrockKeyConfig.BatchS3Config) require.Len(t, found.BedrockKeyConfig.BatchS3Config.Buckets, 1) assert.Equal(t, "my-bucket", found.BedrockKeyConfig.BatchS3Config.Buckets[0].BucketName) diff --git a/framework/configstore/keyhash_alias_test.go b/framework/configstore/keyhash_alias_test.go new file mode 100644 index 0000000000..604ab03fbe --- /dev/null +++ b/framework/configstore/keyhash_alias_test.go @@ -0,0 +1,76 @@ +package configstore + +import ( + "testing" + + "github.com/bytedance/sonic" + "github.com/maximhq/bifrost/core/schemas" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestGenerateKeyHash_LegacyAliasesPreserveByteShape proves that an +// unenriched alias (only ModelID set) marshals into the hasher as the legacy +// {"k":"v"} string-valued shape, which is what keeps config_hash byte-stable +// across the refactor. If MarshalJSON ever stops emitting the legacy form for +// ModelID-only entries, this test fires. +// +// Strategy: hash the same Key with two equivalent alias representations — the +// rich KeyAliases{"k": {ModelID: "v"}} and (a hand-rolled JSON for) the +// legacy "k": "v" shape — and confirm both feed identical bytes into the +// hasher by checking that the marshaled outputs match. We don't recompute the +// full SHA256 since GenerateKeyHash composes many field bytes; the marshaling +// stability of the alias map alone is the regression-prone surface. +func TestGenerateKeyHash_LegacyAliasesPreserveByteShape(t *testing.T) { + key := schemas.Key{ + Name: "openai-key", + Value: *schemas.NewEnvVar("sk-test"), + Weight: 1.0, + Aliases: schemas.KeyAliases{"best-model": {ModelID: "gpt-4o-deployment"}}, + } + + gotMarshal, err := sonic.Marshal(key.Aliases) + require.NoError(t, err) + assert.Equal(t, + `{"best-model":"gpt-4o-deployment"}`, + string(gotMarshal), + "unenriched alias should marshal to the legacy string-valued wire shape; otherwise GenerateKeyHash drifts from pre-refactor rows", + ) + + // And GenerateKeyHash itself runs cleanly with the new types. + hash, err := GenerateKeyHash(key) + require.NoError(t, err) + assert.NotEmpty(t, hash) +} + +// TestGenerateKeyHash_RichAliasesProduceDifferentHash sanity-checks the other +// side: enriching an alias with ModelName/Family/etc. *does* change the hash, +// so genuine config changes are still detected. +func TestGenerateKeyHash_RichAliasesProduceDifferentHash(t *testing.T) { + canonical := "gpt-4o" + family := schemas.ModelFamilyOpenAI + + legacy := schemas.Key{ + Name: "k", + Value: *schemas.NewEnvVar("sk"), + Weight: 1.0, + Aliases: schemas.KeyAliases{"x": {ModelID: "y"}}, + } + rich := schemas.Key{ + Name: "k", + Value: *schemas.NewEnvVar("sk"), + Weight: 1.0, + Aliases: schemas.KeyAliases{"x": { + ModelID: "y", + ModelName: &canonical, + ModelFamily: &family, + }}, + } + + legacyHash, err := GenerateKeyHash(legacy) + require.NoError(t, err) + richHash, err := GenerateKeyHash(rich) + require.NoError(t, err) + + assert.NotEqual(t, legacyHash, richHash, "enriching an alias must change the key hash so config diffs are detected") +} diff --git a/framework/configstore/migrations.go b/framework/configstore/migrations.go index 4449946908..55a5bf06ed 100644 --- a/framework/configstore/migrations.go +++ b/framework/configstore/migrations.go @@ -188,7 +188,6 @@ type legacyBudgetTeam struct { // TableName returns the governance_teams table name for legacyBudgetTeam. func (legacyBudgetTeam) TableName() string { return "governance_teams" } - // sqliteColumnInfo holds the information about a SQLite column. type sqliteColumnInfo struct { Name string `gorm:"column:name"` @@ -866,6 +865,24 @@ func triggerMigrations(ctx context.Context, db *gorm.DB) error { if err := migrationAddModelConfigBudgetsFKConstraint(ctx, db); err != nil { return err } + if err := migrationAddMCPLibraryTable(ctx, db); err != nil { + return err + } + if err := migrationAddMCPLibraryConfigColumns(ctx, db); err != nil { + return err + } + if err := migrationAddMCPLibrarySourceColumns(ctx, db); err != nil { + return err + } + if err := migrationAddFastModePricingColumns(ctx, db); err != nil { + return err + } + if err := migrationAddCustomerNameUniqueConstraint(ctx, db); err != nil { + return err + } + if err := migrationNullLegacyCustomerBudgetID(ctx, db); err != nil { + return err + } return nil } @@ -7820,6 +7837,54 @@ func migrationAddFlexTierPricingColumns(ctx context.Context, db *gorm.DB) error return nil } +// migrationAddFastModePricingColumns adds pricing columns for Anthropic fast mode +// (research preview, speed:"fast" on Opus 4.6/4.7/4.8). +func migrationAddFastModePricingColumns(ctx context.Context, db *gorm.DB) error { + m := migrator.New(db, migrator.DefaultOptions, []*migrator.Migration{{ + ID: "add_fast_mode_pricing_columns", + Migrate: func(tx *gorm.DB) error { + tx = tx.WithContext(ctx) + mg := tx.Migrator() + + columns := []string{ + "input_cost_per_token_fast", + "output_cost_per_token_fast", + } + + for _, field := range columns { + if !mg.HasColumn(&tables.TableModelPricing{}, field) { + if err := mg.AddColumn(&tables.TableModelPricing{}, field); err != nil { + return fmt.Errorf("failed to add column %s: %w", field, err) + } + } + } + return nil + }, + Rollback: func(tx *gorm.DB) error { + tx = tx.WithContext(ctx) + mg := tx.Migrator() + + columns := []string{ + "input_cost_per_token_fast", + "output_cost_per_token_fast", + } + + for _, field := range columns { + if mg.HasColumn(&tables.TableModelPricing{}, field) { + if err := mg.DropColumn(&tables.TableModelPricing{}, field); err != nil { + return fmt.Errorf("failed to drop column %s: %w", field, err) + } + } + } + return nil + }, + }}) + if err := m.Migrate(); err != nil { + return fmt.Errorf("error while running fast mode pricing columns migration: %s", err.Error()) + } + return nil +} + // migrationAddWhitelistedRoutesJSONColumn adds the whitelisted_routes_json column to the config_client table func migrationAddWhitelistedRoutesJSONColumn(ctx context.Context, db *gorm.DB) error { m := migrator.New(db, migrator.DefaultOptions, []*migrator.Migration{{ @@ -9775,3 +9840,322 @@ func migrationAddCustomerBudgetsToBudgetsTable(ctx context.Context, db *gorm.DB) } return nil } + +// migrationNullLegacyCustomerBudgetID clears the legacy governance_customers.budget_id +// values left behind by migrationAddCustomerBudgetsToBudgetsTable. The column and its +// FK (fk_governance_customers_budget) are intentionally kept — dropping either is +// deferred to a major release — but rows still holding a value make DeleteCustomer's +// `DELETE FROM governance_budgets WHERE customer_id = ?` fail that FK check. Ownership +// already lives on governance_budgets.customer_id, so after a defensive backfill the +// legacy values can be nulled; a null reference satisfies the FK unconditionally. +func migrationNullLegacyCustomerBudgetID(ctx context.Context, db *gorm.DB) error { + m := migrator.New(db, migrator.DefaultOptions, []*migrator.Migration{{ + ID: "null_legacy_customer_budget_id_refs", + Migrate: func(tx *gorm.DB) error { + tx = tx.WithContext(ctx) + legacyExists, err := hasColumn(tx, "governance_customers", "budget_id") + if err != nil { + return fmt.Errorf("failed to introspect governance_customers for budget_id: %w", err) + } + if !legacyExists { + return nil + } + // Customers the defensive backfill below will attach a budget to. + // GenerateCustomerHash includes sorted budget IDs, so their stored + // config_hash goes stale once the budget is linked and must be refreshed. + var affectedCustomerIDs []string + if err := tx.Raw(` + SELECT DISTINCT c.id + FROM governance_customers c + JOIN governance_budgets b ON b.id = c.budget_id + WHERE b.customer_id IS NULL + AND b.virtual_key_id IS NULL AND b.team_id IS NULL + AND b.provider_config_id IS NULL AND b.model_config_id IS NULL + `).Scan(&affectedCustomerIDs).Error; err != nil { + return fmt.Errorf("failed to identify customers affected by budget backfill: %w", err) + } + // Defensive backfill (same shape as migrationAddCustomerBudgetsToBudgetsTable) + // in case a budget_id was written after that migration ran, e.g. by an older + // instance in a mixed-version cluster. Only claims budgets with no owner yet. + if err := tx.Exec(` + UPDATE governance_budgets SET customer_id = ( + SELECT id FROM governance_customers + WHERE governance_customers.budget_id = governance_budgets.id + ) WHERE customer_id IS NULL + AND virtual_key_id IS NULL AND team_id IS NULL + AND provider_config_id IS NULL AND model_config_id IS NULL + AND EXISTS ( + SELECT 1 FROM governance_customers + WHERE governance_customers.budget_id = governance_budgets.id + ) + `).Error; err != nil { + return fmt.Errorf("failed to backfill customer budget customer_id: %w", err) + } + // Refresh config_hash for customers whose budgets just got linked, keeping + // migration and runtime hash generation in parity (same as + // migrationAddCustomerBudgetsToBudgetsTable). + for _, customerID := range affectedCustomerIDs { + var customer tables.TableCustomer + if err := tx.Preload("Budgets").First(&customer, "id = ?", customerID).Error; err != nil { + return fmt.Errorf("failed to reload customer %s for hash refresh: %w", customerID, err) + } + hash, err := GenerateCustomerHash(customer) + if err != nil { + return fmt.Errorf("failed to generate hash for customer %s: %w", customerID, err) + } + if err := tx.Model(&tables.TableCustomer{}).Where("id = ?", customerID).Update("config_hash", hash).Error; err != nil { + return fmt.Errorf("failed to update hash for customer %s: %w", customerID, err) + } + } + if err := tx.Exec(`UPDATE governance_customers SET budget_id = NULL WHERE budget_id IS NOT NULL`).Error; err != nil { + return fmt.Errorf("failed to clear legacy governance_customers.budget_id values: %w", err) + } + return nil + }, + // Best-effort inverse: repopulate budget_id from governance_budgets.customer_id. + // The legacy column held a single value while the new model allows several + // budgets per customer, so for multi-budget customers the oldest budget is + // picked — for any customer that predates the pivot that is the original + // legacy budget, since later additions sort newer. + Rollback: func(tx *gorm.DB) error { + tx = tx.WithContext(ctx) + legacyExists, err := hasColumn(tx, "governance_customers", "budget_id") + if err != nil { + return fmt.Errorf("failed to introspect governance_customers for budget_id: %w", err) + } + if !legacyExists { + return nil + } + if err := tx.Exec(` + UPDATE governance_customers SET budget_id = ( + SELECT id FROM governance_budgets + WHERE governance_budgets.customer_id = governance_customers.id + ORDER BY created_at ASC, id ASC + LIMIT 1 + ) WHERE budget_id IS NULL AND EXISTS ( + SELECT 1 FROM governance_budgets + WHERE governance_budgets.customer_id = governance_customers.id + ) + `).Error; err != nil { + return fmt.Errorf("failed to restore legacy governance_customers.budget_id values: %w", err) + } + return nil + }, + }}) + if err := m.Migrate(); err != nil { + return fmt.Errorf("error running null_legacy_customer_budget_id_refs migration: %s", err.Error()) + } + return nil +} + +// migrationAddMCPLibraryTable creates the mcp_library table, the synced-only +// catalog of discoverable MCP servers. Rows are populated from the external MCP +// library datasheet on a configurable interval (mirroring the model-pricing +// sync), so this migration only stands up the schema; no rows exist yet. +func migrationAddMCPLibraryTable(ctx context.Context, db *gorm.DB) error { + m := migrator.New(db, migrator.DefaultOptions, []*migrator.Migration{{ + ID: "add_mcp_library_table", + Migrate: func(tx *gorm.DB) error { + tx = tx.WithContext(ctx) + mg := tx.Migrator() + if !mg.HasTable(&tables.TableMCPLibrary{}) { + if err := mg.CreateTable(&tables.TableMCPLibrary{}); err != nil { + return fmt.Errorf("create mcp_library table: %w", err) + } + } + return nil + }, + Rollback: func(tx *gorm.DB) error { + tx = tx.WithContext(ctx) + mg := tx.Migrator() + if mg.HasTable(&tables.TableMCPLibrary{}) { + if err := mg.DropTable(&tables.TableMCPLibrary{}); err != nil { + return fmt.Errorf("drop mcp_library table: %w", err) + } + } + return nil + }, + }}) + if err := m.Migrate(); err != nil { + return fmt.Errorf("error running add_mcp_library_table migration: %s", err.Error()) + } + return nil +} + +// migrationAddMCPLibraryConfigColumns adds the mcp_library_url and +// mcp_library_sync_interval columns to framework_configs. These store the sync +// source + interval for the MCP server library catalog, mirroring pricing_url / +// pricing_sync_interval. Idempotent via HasColumn guards. +func migrationAddMCPLibraryConfigColumns(ctx context.Context, db *gorm.DB) error { + m := migrator.New(db, migrator.DefaultOptions, []*migrator.Migration{{ + ID: "add_mcp_library_config_columns", + Migrate: func(tx *gorm.DB) error { + tx = tx.WithContext(ctx) + mg := tx.Migrator() + if !mg.HasColumn(&tables.TableFrameworkConfig{}, "mcp_library_url") { + if err := mg.AddColumn(&tables.TableFrameworkConfig{}, "MCPLibraryURL"); err != nil { + return fmt.Errorf("add mcp_library_url column: %w", err) + } + } + if !mg.HasColumn(&tables.TableFrameworkConfig{}, "mcp_library_sync_interval") { + if err := mg.AddColumn(&tables.TableFrameworkConfig{}, "MCPLibrarySyncInterval"); err != nil { + return fmt.Errorf("add mcp_library_sync_interval column: %w", err) + } + } + return nil + }, + Rollback: func(tx *gorm.DB) error { + tx = tx.WithContext(ctx) + mg := tx.Migrator() + if mg.HasColumn(&tables.TableFrameworkConfig{}, "mcp_library_url") { + if err := mg.DropColumn(&tables.TableFrameworkConfig{}, "MCPLibraryURL"); err != nil { + return fmt.Errorf("drop mcp_library_url column: %w", err) + } + } + if mg.HasColumn(&tables.TableFrameworkConfig{}, "mcp_library_sync_interval") { + if err := mg.DropColumn(&tables.TableFrameworkConfig{}, "MCPLibrarySyncInterval"); err != nil { + return fmt.Errorf("drop mcp_library_sync_interval column: %w", err) + } + } + return nil + }, + }}) + if err := m.Migrate(); err != nil { + return fmt.Errorf("error running add_mcp_library_config_columns migration: %s", err.Error()) + } + return nil +} + +// migrationAddMCPLibrarySourceColumns adds the source and deleted_at columns to +// mcp_library. `source` marks a row as remote-synced or org-internal ("custom") +// so the sync can protect custom rows; `deleted_at` is a soft-delete tombstone +// so a user-hidden row (remote or custom) is never resurrected by the next sync. +// Idempotent via HasColumn guards. +func migrationAddMCPLibrarySourceColumns(ctx context.Context, db *gorm.DB) error { + m := migrator.New(db, migrator.DefaultOptions, []*migrator.Migration{{ + ID: "add_mcp_library_source_columns", + Migrate: func(tx *gorm.DB) error { + tx = tx.WithContext(ctx) + mg := tx.Migrator() + if !mg.HasColumn(&tables.TableMCPLibrary{}, "source") { + if err := mg.AddColumn(&tables.TableMCPLibrary{}, "Source"); err != nil { + return fmt.Errorf("add source column: %w", err) + } + } + if !mg.HasColumn(&tables.TableMCPLibrary{}, "deleted_at") { + if err := mg.AddColumn(&tables.TableMCPLibrary{}, "DeletedAt"); err != nil { + return fmt.Errorf("add deleted_at column: %w", err) + } + } + // Create indexes on the new columns (AddColumn doesn't create indexes + // from struct tags). `deleted_at IS NULL` is the leading predicate on + // every paginated library query, so the index avoids a full table scan. + if !mg.HasIndex(&tables.TableMCPLibrary{}, "idx_mcp_library_source") { + if err := mg.CreateIndex(&tables.TableMCPLibrary{}, "Source"); err != nil { + return fmt.Errorf("create index on mcp_library.source: %w", err) + } + } + if !mg.HasIndex(&tables.TableMCPLibrary{}, "idx_mcp_library_deleted_at") { + if err := mg.CreateIndex(&tables.TableMCPLibrary{}, "DeletedAt"); err != nil { + return fmt.Errorf("create index on mcp_library.deleted_at: %w", err) + } + } + return nil + }, + Rollback: func(tx *gorm.DB) error { + // Rollback is intentionally a no-op: dropping `source` and + // `deleted_at` would destroy custom-row protection markers and + // soft-delete tombstones, letting the next sync resurrect rows the + // user hid. This migration is one-way. + return nil + }, + }}) + if err := m.Migrate(); err != nil { + return fmt.Errorf("error running add_mcp_library_source_columns migration: %s", err.Error()) + } + return nil +} + +// migrationAddCustomerNameUniqueConstraint deduplicates governance_customers by +// appending -1, -2, … to later occurrences of the same name (ordered by +// created_at then id), then adds a unique index on the name column. +func migrationAddCustomerNameUniqueConstraint(ctx context.Context, db *gorm.DB) error { + const idxName = "idx_governance_customers_name" + + // Step 1 (transactional): rename duplicate customer names so the later + // CREATE UNIQUE INDEX cannot fail due to pre-existing duplicates. + if err := RunSingleMigration(ctx, nil, db, &migrator.Migration{ + ID: "add_customer_name_unique_constraint_dedup", + Migrate: func(tx *gorm.DB) error { + tx = tx.WithContext(ctx) + + // Fetch all customers in a stable order so the earliest-created row + // always keeps the original name and later duplicates receive suffixes. + var customers []tables.TableCustomer + if err := tx.Order("created_at ASC, id ASC").Find(&customers).Error; err != nil { + return fmt.Errorf("failed to fetch customers: %w", err) + } + + // taken tracks every name that is currently (or will be) in use so + // suffix search never collides with an existing original name. + taken := make(map[string]bool, len(customers)) + for _, c := range customers { + taken[c.Name] = true + } + + firstSeen := make(map[string]bool, len(customers)) + for _, c := range customers { + if !firstSeen[c.Name] { + firstSeen[c.Name] = true + continue // earliest occurrence keeps its name + } + // Find the lowest suffix whose candidate name is not already taken. + suffix := 1 + candidate := fmt.Sprintf("%s-%d", c.Name, suffix) + for taken[candidate] { + suffix++ + candidate = fmt.Sprintf("%s-%d", c.Name, suffix) + } + taken[candidate] = true + if err := tx.Model(&tables.TableCustomer{}).Where("id = ?", c.ID).Update("name", candidate).Error; err != nil { + return fmt.Errorf("failed to rename customer %s to %q: %w", c.ID, candidate, err) + } + } + return nil + }, + Rollback: func(tx *gorm.DB) error { + return nil // name renames are not reversed; dropping the index in step 2 restores the invariant + }, + }); err != nil { + return err + } + + // Step 2 (non-transactional): create the unique index. + // UseTransaction must be false because CREATE INDEX CONCURRENTLY cannot + // execute inside a transaction block. IF NOT EXISTS makes this step safe + // to re-run if the process crashes after the index is built but before + // the migration record is written. + noTxOpts := *migrator.DefaultOptions + noTxOpts.UseTransaction = false + return RunSingleMigration(ctx, &noTxOpts, db, &migrator.Migration{ + ID: "add_customer_name_unique_constraint_index", + Migrate: func(tx *gorm.DB) error { + tx = tx.WithContext(ctx) + // SQLite does not support CONCURRENTLY; use the plain form there. + var stmt string + if tx.Dialector.Name() == "sqlite" { + stmt = "CREATE UNIQUE INDEX IF NOT EXISTS " + idxName + " ON governance_customers (name)" + } else { + stmt = "CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS " + idxName + " ON governance_customers (name)" + } + if err := tx.Exec(stmt).Error; err != nil { + return fmt.Errorf("failed to create unique index on governance_customers.name: %w", err) + } + return nil + }, + Rollback: func(tx *gorm.DB) error { + tx = tx.WithContext(ctx) + return tx.Exec("DROP INDEX IF EXISTS " + idxName).Error + }, + }) +} diff --git a/framework/configstore/migrations_test.go b/framework/configstore/migrations_test.go index f522c7cb95..5a274d533b 100644 --- a/framework/configstore/migrations_test.go +++ b/framework/configstore/migrations_test.go @@ -1090,8 +1090,8 @@ func TestMigrationDropDeploymentColumnsAndAddAliases_BedrockEncrypted(t *testing // Verify the aliases contain the original deployment data (not double-encrypted) aliases := keys[0].Aliases assert.Contains(t, aliases, "claude") - assert.Equal(t, "dep-claude", aliases["claude"]) - assert.Equal(t, "dep-instant", aliases["claude-instant"]) + assert.Equal(t, "dep-claude", aliases["claude"].ModelID) + assert.Equal(t, "dep-instant", aliases["claude-instant"].ModelID) } // ============================================================================ diff --git a/framework/configstore/rdb.go b/framework/configstore/rdb.go index d3ef71ca05..1198d20a4a 100644 --- a/framework/configstore/rdb.go +++ b/framework/configstore/rdb.go @@ -1609,7 +1609,247 @@ func (s *RDBConfigStore) GetMCPClientsPaginated(ctx context.Context, params MCPC return clients, totalCount, nil } -// GetMCPClientByID retrieves an MCP client by ID from the database. +// mcpLibrarySortColumns whitelists the columns the MCP library list endpoint +// may sort by. Restricting to a fixed set keeps the ORDER BY clause free of +// caller-supplied identifiers. +var mcpLibrarySortColumns = map[string]string{ + "name": "name", + "category": "category", + "publisher": "publisher", + "created_at": "created_at", + "updated_at": "updated_at", +} + +// GetMCPLibraryPaginated retrieves MCP library catalog entries with optional +// search, filtering, sorting, and pagination. Returns the page of rows and the +// total count matching the filters (before pagination). +func (s *RDBConfigStore) GetMCPLibraryPaginated(ctx context.Context, params MCPLibraryQueryParams) ([]tables.TableMCPLibrary, int64, error) { + baseQuery := s.DB().WithContext(ctx).Model(&tables.TableMCPLibrary{}).Where("deleted_at IS NULL") + + if params.Search != "" { + search := "%" + strings.ToLower(params.Search) + "%" + baseQuery = baseQuery.Where( + "LOWER(name) LIKE ? OR LOWER(description) LIKE ? OR LOWER(publisher) LIKE ?", + search, search, search, + ) + } + if len(params.Categories) > 0 { + baseQuery = baseQuery.Where("category IN ?", params.Categories) + } + if len(params.ConnectionTypes) > 0 { + baseQuery = baseQuery.Where("connection_type IN ?", params.ConnectionTypes) + } + if len(params.AuthTypes) > 0 { + baseQuery = baseQuery.Where("auth_type IN ?", params.AuthTypes) + } + // Tags are stored as a JSON-encoded array string; match rows whose JSON + // contains any requested tag as a quoted token. This is a substring match + // over the serialized array, which is sufficient for the catalog's small, + // well-formed tag values and avoids a DB-specific JSON operator. LIKE + // metacharacters in the tag are escaped (with an explicit ESCAPE clause) so + // a tag containing % or _ matches literally instead of as a wildcard. + likeEscaper := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`) + for _, tag := range params.Tags { + if tag == "" { + continue + } + escapedTag := likeEscaper.Replace(tag) + baseQuery = baseQuery.Where(`tags LIKE ? ESCAPE '\'`, `%"`+escapedTag+`"%`) + } + + var totalCount int64 + if err := baseQuery.Count(&totalCount).Error; err != nil { + return nil, 0, err + } + + limit := params.Limit + offset := params.Offset + if limit <= 0 { + limit = 25 + } else if limit > 100 { + limit = 100 + } + if offset < 0 { + offset = 0 + } + + sortColumn := "name" + if col, ok := mcpLibrarySortColumns[params.SortBy]; ok { + sortColumn = col + } + dir := "ASC" + if strings.EqualFold(params.Order, "desc") { + dir = "DESC" + } + // id as a stable tiebreaker so paging is deterministic across equal keys. + orderClause := fmt.Sprintf("%s %s, id ASC", sortColumn, dir) + + var entries []tables.TableMCPLibrary + if err := baseQuery. + Order(orderClause). + Offset(offset). + Limit(limit). + Find(&entries).Error; err != nil { + return nil, 0, err + } + return entries, totalCount, nil +} + +// GetMCPLibraryFilterData returns the distinct facet values for the MCP library +// filter sidebar: categories, connection types, auth types, and tags. Empty +// values are skipped. Tags are stored as JSON arrays, so they are decoded and +// unioned in Go rather than via a DB-specific JSON operator. +func (s *RDBConfigStore) GetMCPLibraryFilterData(ctx context.Context) (*MCPLibraryFilterData, error) { + db := s.DB().WithContext(ctx) + result := &MCPLibraryFilterData{ + Categories: []string{}, + ConnectionTypes: []string{}, + AuthTypes: []string{}, + Tags: []string{}, + } + + distinct := func(column string, dst *[]string) error { + var values []string + if err := db.Model(&tables.TableMCPLibrary{}). + Distinct(column). + Where("deleted_at IS NULL"). + Where(column+" IS NOT NULL AND "+column+" != ?", ""). + Order(column+" ASC"). + Pluck(column, &values).Error; err != nil { + return err + } + *dst = append(*dst, values...) + return nil + } + + if err := distinct("category", &result.Categories); err != nil { + return nil, err + } + if err := distinct("connection_type", &result.ConnectionTypes); err != nil { + return nil, err + } + if err := distinct("auth_type", &result.AuthTypes); err != nil { + return nil, err + } + + // Tags: gather distinct JSON blobs, decode, and union the values. + var tagBlobs []string + if err := db.Model(&tables.TableMCPLibrary{}). + Distinct("tags"). + Where("deleted_at IS NULL"). + Where("tags IS NOT NULL AND tags != ?", ""). + Pluck("tags", &tagBlobs).Error; err != nil { + return nil, err + } + tagSet := make(map[string]struct{}) + for _, blob := range tagBlobs { + var tags []string + if err := json.Unmarshal([]byte(blob), &tags); err != nil { + continue // skip malformed blobs rather than failing the whole request + } + for _, tag := range tags { + if tag == "" { + continue + } + tagSet[tag] = struct{}{} + } + } + for tag := range tagSet { + result.Tags = append(result.Tags, tag) + } + sort.Strings(result.Tags) + + return result, nil +} + +// mcpLibrarySyncUpdateColumns enumerates the columns the MCP library sync may +// overwrite on conflict. The list is explicit (not UpdateAll) so id/created_at +// are preserved and any future editorial-only columns can be excluded. +var mcpLibrarySyncUpdateColumns = []string{ + "name", + "description", + "category", + "connection_type", + "connection_url", + "stdio_config", + "auth_type", + "required_header_keys", + "icon_url", + "docs_url", + "publisher", + "tags", + "metadata", + "updated_at", +} + +// UpsertMCPLibraryEntry creates or updates an MCP library catalog row, keyed by +// the unique slug. Mirrors UpsertModelPrices: a single atomic ON CONFLICT +// statement so concurrent syncs across nodes don't deadlock. +func (s *RDBConfigStore) UpsertMCPLibraryEntry(ctx context.Context, entry *tables.TableMCPLibrary, tx ...*gorm.DB) error { + var txDB *gorm.DB + if len(tx) > 0 { + txDB = tx[0] + } else { + txDB = s.DB() + } + db := txDB.WithContext(ctx) + + if err := db.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "slug"}}, + DoUpdates: clause.AssignmentColumns(mcpLibrarySyncUpdateColumns), + // Atomically protect custom/tombstoned rows: only overwrite an existing + // row if it is still a live remote row. This closes the TOCTOU race where + // a row turns custom or is soft-deleted between the snapshot taken by + // GetProtectedMCPLibrarySlugs and this upsert. INSERTs are unaffected. + Where: clause.Where{Exprs: []clause.Expression{ + clause.Expr{SQL: "mcp_library.source = 'remote' AND mcp_library.deleted_at IS NULL"}, + }}, + }).Create(entry).Error; err != nil { + return s.parseGormError(err) + } + return nil +} + +// CreateCustomMCPLibraryEntry inserts an org-internal ("custom") library row. +// Source is forced to "custom" regardless of what the caller passed. The unique +// slug index prevents duplicates; parseGormError maps that to ErrAlreadyExists. +func (s *RDBConfigStore) CreateCustomMCPLibraryEntry(ctx context.Context, entry *tables.TableMCPLibrary) error { + entry.Source = "custom" + if err := s.DB().WithContext(ctx).Create(entry).Error; err != nil { + return s.parseGormError(err) + } + return nil +} + +// SoftDeleteMCPLibraryEntry tombstones a library row by ID (sets deleted_at to +// now) so it no longer appears in listings and the remote sync respects the +// tombstone. Works on both "remote" and "custom" rows. +func (s *RDBConfigStore) SoftDeleteMCPLibraryEntry(ctx context.Context, id uint) error { + result := s.DB().WithContext(ctx). + Model(&tables.TableMCPLibrary{}). + Where("id = ? AND deleted_at IS NULL", id). + Update("deleted_at", time.Now()) + if result.Error != nil { + return result.Error + } + if result.RowsAffected == 0 { + return ErrNotFound + } + return nil +} + +// GetProtectedMCPLibrarySlugs returns the slugs the remote sync must skip: +// custom rows (any source != "remote") and soft-deleted rows (deleted_at set). +func (s *RDBConfigStore) GetProtectedMCPLibrarySlugs(ctx context.Context) ([]string, error) { + var slugs []string + if err := s.DB().WithContext(ctx). + Model(&tables.TableMCPLibrary{}). + Where("source != 'remote' OR deleted_at IS NOT NULL"). + Pluck("slug", &slugs).Error; err != nil { + return nil, err + } + return slugs, nil +} func (s *RDBConfigStore) GetMCPClientByID(ctx context.Context, id string) (*tables.TableMCPClient, error) { var mcpClient tables.TableMCPClient if err := s.DB().WithContext(ctx).Where("client_id = ?", id).First(&mcpClient).Error; err != nil { @@ -2092,6 +2332,8 @@ var pricingSyncUpdateColumns = []string{ "output_cost_per_token_priority", "input_cost_per_token_flex", "output_cost_per_token_flex", + "input_cost_per_token_fast", + "output_cost_per_token_fast", "input_cost_per_character", // Costs - 128k Tier "input_cost_per_token_above_128k_tokens", @@ -2490,11 +2732,22 @@ func (s *RDBConfigStore) UpdatePlugin(ctx context.Context, plugin *tables.TableP } else { plugin.IsCustom = false } - if err := txDB.WithContext(ctx).Delete(&tables.TablePlugin{}, "name = ?", plugin.Name).Error; err != nil { - if localTx { - txDB.Rollback() + var existing tables.TablePlugin + if err := txDB.WithContext(ctx).Where("name = ?", plugin.Name).First(&existing).Error; err != nil { + if !errors.Is(err, gorm.ErrRecordNotFound) { + if localTx { + txDB.Rollback() + } + return err + } + // not found — nothing to delete + } else { + if err := txDB.WithContext(ctx).Delete(&existing).Error; err != nil { + if localTx { + txDB.Rollback() + } + return err } - return err } if err := txDB.WithContext(ctx).Create(plugin).Error; err != nil { if localTx { @@ -2516,7 +2769,14 @@ func (s *RDBConfigStore) DeletePlugin(ctx context.Context, name string, tx ...*g } else { txDB = s.DB() } - return txDB.WithContext(ctx).Delete(&tables.TablePlugin{}, "name = ?", name).Error + var plugin tables.TablePlugin + if err := txDB.WithContext(ctx).Where("name = ?", name).First(&plugin).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return ErrNotFound + } + return err + } + return txDB.WithContext(ctx).Delete(&plugin).Error } // GOVERNANCE METHODS @@ -3038,8 +3298,8 @@ func (s *RDBConfigStore) DeleteVirtualKey(ctx context.Context, id string, tx ... return err } rateLimitID := virtualKey.RateLimitID - // Delete the virtual key - if err := txDB.WithContext(ctx).Delete(&tables.TableVirtualKey{}, "id = ?", id).Error; err != nil { + // Delete the virtual key (use hydrated struct so AfterDelete vault cleanup fires correctly) + if err := txDB.WithContext(ctx).Delete(&virtualKey).Error; err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { return ErrNotFound } @@ -4698,12 +4958,12 @@ func (s *RDBConfigStore) GetGovernanceConfig(ctx context.Context) (*GovernanceCo return nil, nil } var authConfig *AuthConfig + var complexityAnalyzerConfig *ComplexityAnalyzerConfig if len(governanceConfigs) > 0 { // Checking if username and password is present var username *string var password *string var isEnabled bool - var disableAuthOnInference bool for _, entry := range governanceConfigs { switch entry.Key { case tables.ConfigAdminUsernameKey: @@ -4712,39 +4972,85 @@ func (s *RDBConfigStore) GetGovernanceConfig(ctx context.Context) (*GovernanceCo password = bifrost.Ptr(entry.Value) case tables.ConfigIsAuthEnabledKey: isEnabled = entry.Value == "true" - case tables.ConfigDisableAuthOnInferenceKey: - disableAuthOnInference = entry.Value == "true" + case tables.ConfigComplexityAnalyzerConfigKey: + if strings.TrimSpace(entry.Value) == "" { + continue + } + decoded, err := DecodeComplexityAnalyzerConfig([]byte(entry.Value)) + if err != nil { + if s.logger != nil { + s.logger.Warn("failed to load complexity analyzer config from governance_config: %v", err) + } + continue + } + complexityAnalyzerConfig = decoded } } if username != nil && password != nil { authConfig = &AuthConfig{ - AdminUserName: schemas.NewEnvVar(*username), - AdminPassword: schemas.NewEnvVar(*password), - IsEnabled: isEnabled, - DisableAuthOnInference: disableAuthOnInference, + AdminUserName: schemas.NewEnvVar(*username), + AdminPassword: schemas.NewEnvVar(*password), + IsEnabled: isEnabled, } } } return &GovernanceConfig{ - VirtualKeys: virtualKeys, - Teams: teams, - Customers: customers, - Budgets: budgets, - RateLimits: rateLimits, - ModelConfigs: modelConfigs, - Providers: providers, - RoutingRules: routingRules, - PricingOverrides: pricingOverrides, - AuthConfig: authConfig, + VirtualKeys: virtualKeys, + Teams: teams, + Customers: customers, + Budgets: budgets, + RateLimits: rateLimits, + ModelConfigs: modelConfigs, + Providers: providers, + RoutingRules: routingRules, + PricingOverrides: pricingOverrides, + AuthConfig: authConfig, + ComplexityAnalyzerConfig: complexityAnalyzerConfig, }, nil } +// GetComplexityAnalyzerConfig retrieves the typed complexity analyzer config. +func (s *RDBConfigStore) GetComplexityAnalyzerConfig(ctx context.Context) (*ComplexityAnalyzerConfig, error) { + configEntry, err := s.GetConfig(ctx, tables.ConfigComplexityAnalyzerConfigKey) + if err != nil { + if errors.Is(err, ErrNotFound) { + return nil, nil + } + return nil, err + } + if configEntry == nil || strings.TrimSpace(configEntry.Value) == "" { + return nil, nil + } + return DecodeComplexityAnalyzerConfig([]byte(configEntry.Value)) +} + +// UpdateComplexityAnalyzerConfig normalizes, validates, and persists the typed analyzer config. +func (s *RDBConfigStore) UpdateComplexityAnalyzerConfig(ctx context.Context, config *ComplexityAnalyzerConfig, tx ...*gorm.DB) error { + if config == nil { + return fmt.Errorf("complexity analyzer config is nil") + } + + normalized := config.Normalized() + if err := normalized.Validate(); err != nil { + return err + } + + raw, err := json.Marshal(normalized) + if err != nil { + return fmt.Errorf("failed to marshal complexity analyzer config: %w", err) + } + + return s.UpdateConfig(ctx, &tables.TableGovernanceConfig{ + Key: tables.ConfigComplexityAnalyzerConfigKey, + Value: string(raw), + }, tx...) +} + // GetAuthConfig retrieves the auth configuration from the database. func (s *RDBConfigStore) GetAuthConfig(ctx context.Context) (*AuthConfig, error) { var username *string var password *string var isEnabled bool - var disableAuthOnInference bool if err := s.DB().WithContext(ctx).First(&tables.TableGovernanceConfig{}, "key = ?", tables.ConfigAdminUsernameKey).Select("value").Scan(&username).Error; err != nil { if !errors.Is(err, gorm.ErrRecordNotFound) { return nil, err @@ -4760,23 +5066,13 @@ func (s *RDBConfigStore) GetAuthConfig(ctx context.Context) (*AuthConfig, error) return nil, err } } - if err := s.DB().WithContext(ctx).First(&tables.TableGovernanceConfig{}, "key = ?", tables.ConfigDisableAuthOnInferenceKey).Select("value").Scan(&disableAuthOnInference).Error; err != nil { - if !errors.Is(err, gorm.ErrRecordNotFound) { - return nil, err - } - } if username == nil || password == nil { return nil, nil } - // We are no longer keeping this option in the database - if !isEnabled { - disableAuthOnInference = true - } return &AuthConfig{ - AdminUserName: schemas.NewEnvVar(*username), - AdminPassword: schemas.NewEnvVar(*password), - IsEnabled: isEnabled, - DisableAuthOnInference: disableAuthOnInference, + AdminUserName: schemas.NewEnvVar(*username), + AdminPassword: schemas.NewEnvVar(*password), + IsEnabled: isEnabled, }, nil } @@ -4801,12 +5097,6 @@ func (s *RDBConfigStore) UpdateAuthConfig(ctx context.Context, config *AuthConfi }).Error; err != nil { return err } - if err := tx.Save(&tables.TableGovernanceConfig{ - Key: tables.ConfigDisableAuthOnInferenceKey, - Value: fmt.Sprintf("%t", config.DisableAuthOnInference), - }).Error; err != nil { - return err - } return nil }) } @@ -4935,14 +5225,18 @@ func (s *RDBConfigStore) CreateSession(ctx context.Context, session *tables.Sess // DeleteSession deletes a session from the database. func (s *RDBConfigStore) DeleteSession(ctx context.Context, token string) error { tokenHash := encrypt.HashSHA256(token) - result := s.DB().WithContext(ctx).Delete(&tables.SessionsTable{}, "token_hash = ?", tokenHash) + var session tables.SessionsTable + if err := s.DB().WithContext(ctx).First(&session, "token_hash = ?", tokenHash).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + // Fall back to plaintext lookup for backward compatibility + return s.DB().WithContext(ctx).Delete(&tables.SessionsTable{}, "token = ?", token).Error // vault token is saved via tokenHash, so this case will not hit the vault scenario, but we keep it for backward compatibility with any existing plaintext tokens + } + return err + } + result := s.DB().WithContext(ctx).Delete(&session) if result.Error != nil { return result.Error } - if result.RowsAffected == 0 { - // Fall back to plaintext lookup for backward compatibility - return s.DB().WithContext(ctx).Delete(&tables.SessionsTable{}, "token = ?", token).Error - } return nil } @@ -5292,7 +5586,16 @@ func (s *RDBConfigStore) UpdateOauthToken(ctx context.Context, token *tables.Tab // DeleteOauthToken deletes an OAuth token by its ID func (s *RDBConfigStore) DeleteOauthToken(ctx context.Context, id string) error { - result := s.DB().WithContext(ctx).Where("id = ?", id).Delete(&tables.TableOauthToken{}) + var existing tables.TableOauthToken + // Check if the token exists before attempting to delete + err := s.DB().WithContext(ctx).Where("id = ?", id).First(&existing).Error + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil // Token doesn't exist, consider it deleted + } + return fmt.Errorf("failed to check existence of oauth token: %w", err) + } + result := s.DB().WithContext(ctx).Delete(&existing) if result.Error != nil { return fmt.Errorf("failed to delete oauth token: %w", result.Error) } diff --git a/framework/configstore/rdb_test.go b/framework/configstore/rdb_test.go index 7047f47166..d01b75600e 100644 --- a/framework/configstore/rdb_test.go +++ b/framework/configstore/rdb_test.go @@ -29,6 +29,10 @@ func setupRDBTestStore(t *testing.T) *RDBConfigStore { &tables.TableKey{}, &tables.TableBudget{}, &tables.TableRateLimit{}, + &tables.TableModelConfig{}, + &tables.TableRoutingRule{}, + &tables.TableRoutingTarget{}, + &tables.TablePricingOverride{}, &tables.TableVirtualKey{}, &tables.TableVirtualKeyProviderConfig{}, &tables.TableVirtualKeyProviderConfigKey{}, @@ -36,8 +40,10 @@ func setupRDBTestStore(t *testing.T) *RDBConfigStore { &tables.TableCustomer{}, &tables.TableTeam{}, &tables.TableClientConfig{}, + &tables.TableGovernanceConfig{}, &tables.TablePlugin{}, &tables.TableMCPClient{}, + &tables.TableMCPLibrary{}, &tables.TableVirtualKeyMCPConfig{}, &tables.TableFolder{}, &tables.TablePrompt{}, @@ -65,6 +71,145 @@ func setupRDBTestStore(t *testing.T) *RDBConfigStore { return s } +func testComplexityAnalyzerConfig() *ComplexityAnalyzerConfig { + return &ComplexityAnalyzerConfig{ + TierBoundaries: ComplexityTierBoundaries{ + SimpleMedium: 0.10, + MediumComplex: 0.30, + ComplexReasoning: 0.70, + }, + Keywords: ComplexityEditableKeywordConfig{ + CodeKeywords: []string{" Function ", "api", "API"}, + ReasoningKeywords: []string{"tradeoffs"}, + TechnicalKeywords: []string{"latency"}, + SimpleKeywords: []string{"hello"}, + }, + } +} + +func TestRDBConfigStore_ComplexityAnalyzerConfigRoundTrip(t *testing.T) { + store := setupRDBTestStore(t) + ctx := context.Background() + + require.NoError(t, store.UpdateComplexityAnalyzerConfig(ctx, testComplexityAnalyzerConfig())) + + got, err := store.GetComplexityAnalyzerConfig(ctx) + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, ComplexityTierBoundaries{ + SimpleMedium: 0.10, + MediumComplex: 0.30, + ComplexReasoning: 0.70, + }, got.TierBoundaries) + assert.Equal(t, []string{"api", "function"}, got.Keywords.CodeKeywords) +} + +func TestRDBConfigStore_GetGovernanceConfigIncludesComplexityAnalyzerConfig(t *testing.T) { + store := setupRDBTestStore(t) + ctx := context.Background() + + require.NoError(t, store.UpdateComplexityAnalyzerConfig(ctx, testComplexityAnalyzerConfig())) + + governanceConfig, err := store.GetGovernanceConfig(ctx) + require.NoError(t, err) + require.NotNil(t, governanceConfig) + require.NotNil(t, governanceConfig.ComplexityAnalyzerConfig) + assert.Equal(t, 0.70, governanceConfig.ComplexityAnalyzerConfig.TierBoundaries.ComplexReasoning) +} + +func TestRDBConfigStore_UpdateComplexityAnalyzerConfigRejectsInvalidConfig(t *testing.T) { + store := setupRDBTestStore(t) + ctx := context.Background() + + tests := []struct { + name string + mutate func(*ComplexityAnalyzerConfig) + }{ + { + name: "simple medium below minimum", + mutate: func(cfg *ComplexityAnalyzerConfig) { + cfg.TierBoundaries.SimpleMedium = -0.1 + }, + }, + { + name: "medium complex at minimum", + mutate: func(cfg *ComplexityAnalyzerConfig) { + cfg.TierBoundaries.MediumComplex = 0 + }, + }, + { + name: "complex reasoning at maximum", + mutate: func(cfg *ComplexityAnalyzerConfig) { + cfg.TierBoundaries.ComplexReasoning = 1.0 + }, + }, + { + name: "boundaries out of order", + mutate: func(cfg *ComplexityAnalyzerConfig) { + cfg.TierBoundaries.ComplexReasoning = cfg.TierBoundaries.MediumComplex - 0.1 + }, + }, + { + name: "empty code keywords", + mutate: func(cfg *ComplexityAnalyzerConfig) { + cfg.Keywords.CodeKeywords = nil + }, + }, + { + name: "empty reasoning keywords", + mutate: func(cfg *ComplexityAnalyzerConfig) { + cfg.Keywords.ReasoningKeywords = nil + }, + }, + { + name: "empty technical keywords", + mutate: func(cfg *ComplexityAnalyzerConfig) { + cfg.Keywords.TechnicalKeywords = nil + }, + }, + { + name: "empty simple keywords", + mutate: func(cfg *ComplexityAnalyzerConfig) { + cfg.Keywords.SimpleKeywords = nil + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + invalid := testComplexityAnalyzerConfig() + tt.mutate(invalid) + + err := store.UpdateComplexityAnalyzerConfig(ctx, invalid) + require.Error(t, err) + }) + } +} + +func TestUpsertMCPLibraryEntry(t *testing.T) { + store := setupRDBTestStore(t) + ctx := context.Background() + + entry := &tables.TableMCPLibrary{ + Slug: "filesystem", + Name: "Filesystem", + Description: "original", + ConnectionType: schemas.MCPConnectionTypeSTDIO, + AuthType: schemas.MCPAuthTypeNone, + Source: "remote", + } + require.NoError(t, store.UpsertMCPLibraryEntry(ctx, entry)) + + entry.Description = "updated" + require.NoError(t, store.UpsertMCPLibraryEntry(ctx, entry)) + + entries, totalCount, err := store.GetMCPLibraryPaginated(ctx, MCPLibraryQueryParams{Limit: 1}) + require.NoError(t, err) + require.Equal(t, int64(1), totalCount) + require.Len(t, entries, 1) + require.Equal(t, "updated", entries[0].Description) +} + // ============================================================================= // Provider and Key Tests // ============================================================================= diff --git a/framework/configstore/store.go b/framework/configstore/store.go index 28b07c411e..444c3bda71 100644 --- a/framework/configstore/store.go +++ b/framework/configstore/store.go @@ -51,6 +51,30 @@ type MCPClientsQueryParams struct { Search string } +// MCPLibraryQueryParams holds pagination, filtering, search, and sort +// parameters for MCP library catalog queries. All fields are optional — an +// empty struct returns the first default-sized page ordered by name. +type MCPLibraryQueryParams struct { + Limit int + Offset int + Search string // matches name/description/publisher (case-insensitive) + Categories []string // exact category filter(s), OR semantics + ConnectionTypes []string // exact connection_type filter(s) (http | stdio | sse) + AuthTypes []string // exact auth_type filter(s) + Tags []string // match rows carrying any of these tags + SortBy string // name, category, publisher, created_at, updated_at (default: name) + Order string // asc, desc (default: asc) +} + +// MCPLibraryFilterData holds the distinct facet values surfaced by the filter +// sidebar on the MCP library page. Populated via GetMCPLibraryFilterData. +type MCPLibraryFilterData struct { + Categories []string `json:"categories"` + ConnectionTypes []string `json:"connection_types"` + AuthTypes []string `json:"auth_types"` + Tags []string `json:"tags"` +} + // TeamsQueryParams holds pagination, filtering, and search parameters for team queries. type TeamsQueryParams struct { Limit int @@ -166,6 +190,20 @@ type ConfigStore interface { UpdateMCPClientConfig(ctx context.Context, id string, clientConfig *tables.TableMCPClient) error DeleteMCPClientConfig(ctx context.Context, id string) error + // MCP library catalog (synced + org-custom) + GetMCPLibraryPaginated(ctx context.Context, params MCPLibraryQueryParams) ([]tables.TableMCPLibrary, int64, error) + GetMCPLibraryFilterData(ctx context.Context) (*MCPLibraryFilterData, error) + UpsertMCPLibraryEntry(ctx context.Context, entry *tables.TableMCPLibrary, tx ...*gorm.DB) error + // CreateCustomMCPLibraryEntry inserts an org-internal ("custom") library row. + // Returns ErrAlreadyExists when the slug collides with an existing entry. + CreateCustomMCPLibraryEntry(ctx context.Context, entry *tables.TableMCPLibrary) error + // SoftDeleteMCPLibraryEntry tombstones a library row by ID (sets deleted_at) + // so it is hidden from listings and never resurrected by the remote sync. + SoftDeleteMCPLibraryEntry(ctx context.Context, id uint) error + // GetProtectedMCPLibrarySlugs returns the slugs the remote sync must not + // overwrite or recreate: custom rows and soft-deleted (tombstoned) rows. + GetProtectedMCPLibrarySlugs(ctx context.Context) ([]string, error) + // Vector store config CRUD UpdateVectorStoreConfig(ctx context.Context, config *vectorstore.Config) error GetVectorStoreConfig(ctx context.Context) (*vectorstore.Config, error) @@ -177,6 +215,10 @@ type ConfigStore interface { // Config CRUD GetConfig(ctx context.Context, key string) (*tables.TableGovernanceConfig, error) UpdateConfig(ctx context.Context, config *tables.TableGovernanceConfig, tx ...*gorm.DB) error + // GetComplexityAnalyzerConfig retrieves the persisted analyzer config, if configured. + GetComplexityAnalyzerConfig(ctx context.Context) (*ComplexityAnalyzerConfig, error) + // UpdateComplexityAnalyzerConfig persists the normalized analyzer config. + UpdateComplexityAnalyzerConfig(ctx context.Context, config *ComplexityAnalyzerConfig, tx ...*gorm.DB) error // Plugins CRUD GetPlugins(ctx context.Context) ([]*tables.TablePlugin, error) diff --git a/framework/configstore/tables/config.go b/framework/configstore/tables/config.go index bb4776d418..bb89ef42ab 100644 --- a/framework/configstore/tables/config.go +++ b/framework/configstore/tables/config.go @@ -3,13 +3,14 @@ package tables import "github.com/maximhq/bifrost/core/network" const ( - ConfigAdminUsernameKey = "admin_username" - ConfigAdminPasswordKey = "admin_password" - ConfigIsAuthEnabledKey = "is_auth_enabled" - ConfigDisableAuthOnInferenceKey = "disable_auth_on_inference" - ConfigProxyKey = "proxy_config" - ConfigRestartRequiredKey = "restart_required" - ConfigHeaderFilterKey = "header_filter_config" + ConfigAdminUsernameKey = "admin_username" + ConfigAdminPasswordKey = "admin_password" + ConfigIsAuthEnabledKey = "is_auth_enabled" + ConfigProxyKey = "proxy_config" + // ConfigComplexityAnalyzerConfigKey stores the persisted analyzer config JSON. + ConfigComplexityAnalyzerConfigKey = "complexity_analyzer_config" + ConfigRestartRequiredKey = "restart_required" + ConfigHeaderFilterKey = "header_filter_config" ) // Keys for the ClientConfig.MetadataJSON blob. diff --git a/framework/configstore/tables/customer.go b/framework/configstore/tables/customer.go index f70574f59a..bb8a3a2e95 100644 --- a/framework/configstore/tables/customer.go +++ b/framework/configstore/tables/customer.go @@ -9,14 +9,14 @@ import ( // TableCustomer represents a customer entity with budgets, rate limit and team/VK association type TableCustomer struct { ID string `gorm:"primaryKey;type:varchar(255)" json:"id"` - Name string `gorm:"type:varchar(255);not null" json:"name"` + Name string `gorm:"type:varchar(255);not null;uniqueIndex:idx_governance_customers_name" json:"name"` RateLimitID *string `gorm:"type:varchar(255);index" json:"rate_limit_id,omitempty"` // BudgetID is a config-file-only field referencing a pre-declared budget (from governance.budgets) to link to this customer. Not persisted; used by the config sync path to set customer_id on the referenced budget row. BudgetID *string `gorm:"-" json:"budget_id,omitempty"` // Relationships - Budgets []TableBudget `gorm:"foreignKey:CustomerID;constraint:OnDelete:CASCADE" json:"budgets,omitempty"` + Budgets []TableBudget `gorm:"foreignKey:CustomerID;constraint:OnDelete:CASCADE" json:"budgets,omitempty"` RateLimit *TableRateLimit `gorm:"foreignKey:RateLimitID" json:"rate_limit,omitempty"` Teams []TableTeam `gorm:"foreignKey:CustomerID" json:"teams"` VirtualKeys []TableVirtualKey `gorm:"foreignKey:CustomerID" json:"virtual_keys"` diff --git a/framework/configstore/tables/encryption.go b/framework/configstore/tables/encryption.go index 158d6d09f8..76d495f61a 100644 --- a/framework/configstore/tables/encryption.go +++ b/framework/configstore/tables/encryption.go @@ -1,6 +1,9 @@ package tables import ( + "context" + "fmt" + "github.com/maximhq/bifrost/core/schemas" "github.com/maximhq/bifrost/framework/encrypt" ) @@ -10,8 +13,88 @@ const ( EncryptionStatusPlainText = "plain_text" // EncryptionStatusEncrypted indicates the row's sensitive fields have been encrypted. EncryptionStatusEncrypted = "encrypted" + // EncryptionStatusVault indicates the row's sensitive fields are stored as vault references. + EncryptionStatusVault = "vault" + + // defaultVaultPrefix is the path prefix used when VaultHooks.Prefix is not set. + defaultVaultPrefix = "bifrost" ) +// VaultHooks is populated at startup when a vault backend is configured. +// OSS table hooks check these function pointers before falling through to AES encryption. +var VaultHooks struct { + // IsEnabled reports whether vault is active. + IsEnabled func() bool + // Prefix returns the configured vault path prefix (e.g. "bifrost"). + Prefix func() string + // StoreString vaults *value at path, then replaces *value with the vault reference. + StoreString func(ctx context.Context, path string, value *string) error + // ResolveString resolves a vault reference, replacing *value with the secret. + ResolveString func(ctx context.Context, value *string) error + // Remove deletes the secret at path (best-effort; errors are ignored by callers). + Remove func(ctx context.Context, path string) error +} + +func VaultIsEnabled() bool { + return VaultHooks.IsEnabled != nil && VaultHooks.IsEnabled() && + VaultHooks.StoreString != nil && VaultHooks.ResolveString != nil +} + +func VaultPrefix() string { + if VaultHooks.Prefix != nil { + return VaultHooks.Prefix() + } + return defaultVaultPrefix +} + +// vaultEnvVar vaults the Val field of an EnvVar at path, replacing it with a vault reference. +// No-op if nil, references an env var, or empty. Returns an error if the hook is not configured. +func vaultEnvVar(ctx context.Context, path string, field *schemas.EnvVar) error { + if field == nil || field.IsFromEnv() || field.GetValue() == "" { + return nil + } + if VaultHooks.StoreString == nil { + return fmt.Errorf("vault store hook is not configured") + } + return VaultHooks.StoreString(ctx, path, &field.Val) +} + +// resolveVaultEnvVar resolves a vault reference stored in EnvVar.Val. +// No-op if nil, references an env var, or empty. Returns an error if the hook is not configured. +func resolveVaultEnvVar(ctx context.Context, field *schemas.EnvVar) error { + if field == nil || field.IsFromEnv() || field.GetValue() == "" { + return nil + } + if VaultHooks.ResolveString == nil { + return fmt.Errorf("vault resolve hook is not configured") + } + return VaultHooks.ResolveString(ctx, &field.Val) +} + +// vaultString stores *value at path in vault, replacing it with a vault reference. +// No-op if nil or empty. Returns an error if the hook is not configured. +func vaultString(ctx context.Context, path string, value *string) error { + if value == nil || *value == "" { + return nil + } + if VaultHooks.StoreString == nil { + return fmt.Errorf("vault store hook is not configured") + } + return VaultHooks.StoreString(ctx, path, value) +} + +// resolveVaultString resolves a vault reference in *value, replacing it with the secret. +// No-op if nil or empty. Returns an error if the hook is not configured. +func resolveVaultString(ctx context.Context, value *string) error { + if value == nil || *value == "" { + return nil + } + if VaultHooks.ResolveString == nil { + return fmt.Errorf("vault resolve hook is not configured") + } + return VaultHooks.ResolveString(ctx, value) +} + // encryptEnvVar encrypts the Val field of an EnvVar in place using AES-256-GCM. // It is a no-op if the field is nil, references an environment variable, or has an empty value. func encryptEnvVar(field *schemas.EnvVar) error { @@ -85,3 +168,29 @@ func decryptString(value *string) error { *value = decrypted return nil } + +// removeVaultEnvVar best-effort removes a vault secret for the given EnvVar field. +// Called in BeforeSave when a field is nil, env-backed, or empty so stale vault +// entries are cleaned up when a field is cleared or switched away from a literal value. +func removeVaultEnvVar(ctx context.Context, path string, field *schemas.EnvVar) { + if VaultHooks.Remove == nil { + return + } + if field != nil && !field.IsFromEnv() && field.GetValue() != "" { + return // field has a real value; vaultEnvVar will overwrite it, no cleanup needed + } + _ = VaultHooks.Remove(ctx, path) +} + +// removeVaultString best-effort removes a vault secret for the given string field. +// Called in BeforeSave when a field is nil or empty so stale vault entries are +// cleaned up when a field is cleared. +func removeVaultString(ctx context.Context, path string, value *string) { + if VaultHooks.Remove == nil { + return + } + if value != nil && *value != "" { + return // field has a real value; vaultString will overwrite it, no cleanup needed + } + _ = VaultHooks.Remove(ctx, path) +} diff --git a/framework/configstore/tables/encryption_test.go b/framework/configstore/tables/encryption_test.go index 9b329fbe51..2454f5cfd1 100644 --- a/framework/configstore/tables/encryption_test.go +++ b/framework/configstore/tables/encryption_test.go @@ -178,7 +178,7 @@ func TestTableKey_BedrockFieldsEncryptDecrypt(t *testing.T) { Provider: "bedrock", KeyID: "bedrock-uuid-1", Value: *schemas.NewEnvVar("bedrock-val"), - Aliases: schemas.KeyAliases{"model-a": "profile-a"}, + Aliases: schemas.KeyAliases{"model-a": {ModelID: "profile-a"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAIOSFODNN7EXAMPLE"), SecretKey: *schemas.NewEnvVar("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"), @@ -224,7 +224,7 @@ func TestTableKey_BedrockFieldsEncryptDecrypt(t *testing.T) { assert.Equal(t, "us-west-2", found.BedrockKeyConfig.Region.GetValue()) require.NotNil(t, found.BedrockKeyConfig.ARN) assert.Equal(t, "arn:aws:iam::123456789:role/test", found.BedrockKeyConfig.ARN.GetValue()) - assert.Equal(t, "profile-a", found.Aliases["model-a"]) + assert.Equal(t, "profile-a", found.Aliases["model-a"].ModelID) require.NotNil(t, found.BedrockKeyConfig.BatchS3Config) require.Len(t, found.BedrockKeyConfig.BatchS3Config.Buckets, 1) assert.Equal(t, "my-batch-bucket", found.BedrockKeyConfig.BatchS3Config.Buckets[0].BucketName) @@ -1156,7 +1156,7 @@ func TestTableKey_AllProviderConfigs_EncryptDecrypt(t *testing.T) { Provider: "custom", KeyID: "multi-uuid", Value: *schemas.NewEnvVar("multi-api-key"), - Aliases: schemas.KeyAliases{"claude-3": "profile-claude"}, + Aliases: schemas.KeyAliases{"claude-3": {ModelID: "profile-claude"}}, AzureKeyConfig: &schemas.AzureKeyConfig{ Endpoint: *schemas.NewEnvVar("https://azure.endpoint.com"), ClientID: schemas.NewEnvVar("multi-azure-cid"), @@ -1230,7 +1230,7 @@ func TestTableKey_AllProviderConfigs_EncryptDecrypt(t *testing.T) { assert.Equal(t, "eu-west-1", found.BedrockKeyConfig.Region.GetValue()) require.NotNil(t, found.BedrockKeyConfig.ARN) assert.Equal(t, "arn:aws:bedrock:eu-west-1:123:role", found.BedrockKeyConfig.ARN.GetValue()) - assert.Equal(t, "profile-claude", found.Aliases["claude-3"]) + assert.Equal(t, "profile-claude", found.Aliases["claude-3"].ModelID) } // ============================================================================ @@ -1919,3 +1919,139 @@ func TestTableKey_VertexPlainValue_RoundTrip(t *testing.T) { assert.False(t, found.VertexKeyConfig.ProjectID.FromEnv) assert.Equal(t, "us-central1", found.VertexKeyConfig.Region.GetValue()) } + +// TestTableKey_AliasesJSON_LegacyWireShape verifies that a KeyAliases value +// containing only ModelID (the unenriched shape) is persisted to the DB as the +// legacy {"k":"v"} string-valued JSON, preserving byte-for-byte wire compat +// with pre-refactor consumers and keeping config_hash stable. +func TestTableKey_AliasesJSON_LegacyWireShape(t *testing.T) { + db := setupTestDB(t) + + key := &TableKey{ + Name: "openai-key", + ProviderID: 1, + Provider: "openai", + KeyID: "openai-uuid-aliases-legacy", + Value: *schemas.NewEnvVar("sk-test"), + Aliases: schemas.KeyAliases{ + "best-model": {ModelID: "gpt-4o-deployment"}, + "backup": {ModelID: "gpt-3.5-turbo"}, + }, + } + require.NoError(t, db.Create(key).Error) + + raw := rawRow(t, db, "config_keys", key.ID) + rawAliasesVal := raw["aliases_json"] + var rawAliasesStr string + switch v := rawAliasesVal.(type) { + case string: + rawAliasesStr = v + case []byte: + rawAliasesStr = string(v) + } + require.NotEmpty(t, rawAliasesStr) + + plaintext, err := encrypt.Decrypt(rawAliasesStr) + require.NoError(t, err, "aliases_json should be decryptable") + + // Both expected shapes are valid JSON encodings (map iteration order is not stable). + candidates := []string{ + `{"best-model":"gpt-4o-deployment","backup":"gpt-3.5-turbo"}`, + `{"backup":"gpt-3.5-turbo","best-model":"gpt-4o-deployment"}`, + } + assert.Contains(t, candidates, plaintext, "legacy ModelID-only aliases should marshal to the string-valued legacy wire shape") +} + +// TestTableKey_AliasesJSON_RichRoundTrip verifies that an enriched AliasConfig +// (with ModelName/ModelFamily/sub-config populated) survives the full DB +// encrypt → save → load → decrypt round-trip with no loss of information. +func TestTableKey_AliasesJSON_RichRoundTrip(t *testing.T) { + db := setupTestDB(t) + + apiVersion := "2024-08-01-preview" + canonical := "claude-3-5-sonnet" + family := schemas.ModelFamilyAnthropic + + key := &TableKey{ + Name: "azure-rich", + ProviderID: 1, + Provider: "azure", + KeyID: "azure-uuid-aliases-rich", + Value: *schemas.NewEnvVar("sk-test"), + Aliases: schemas.KeyAliases{ + "best-model": { + ModelID: "azure-deployment-xyz", + ModelName: &canonical, + ModelFamily: &family, + Description: "prod summarizer", + AzureAliasCfg: &schemas.AzureAliasCfg{ + APIVersion: &apiVersion, + }, + }, + "plain": {ModelID: "gpt-4o-fallback"}, + }, + AzureKeyConfig: &schemas.AzureKeyConfig{ + Endpoint: *schemas.NewEnvVar("https://example.openai.azure.com"), + }, + } + require.NoError(t, db.Create(key).Error) + + var found TableKey + require.NoError(t, db.First(&found, key.ID).Error) + require.NotNil(t, found.Aliases) + require.Len(t, found.Aliases, 2) + + rich := found.Aliases["best-model"] + assert.Equal(t, "azure-deployment-xyz", rich.ModelID) + require.NotNil(t, rich.ModelName) + assert.Equal(t, canonical, *rich.ModelName) + require.NotNil(t, rich.ModelFamily) + assert.Equal(t, schemas.ModelFamilyAnthropic, *rich.ModelFamily) + assert.Equal(t, "prod summarizer", rich.Description) + require.NotNil(t, rich.AzureAliasCfg) + require.NotNil(t, rich.AzureAliasCfg.APIVersion) + assert.Equal(t, apiVersion, *rich.AzureAliasCfg.APIVersion) + + // The unenriched sibling stays a legacy-shape entry — proves marshaling + // only escalates to the rich object form for entries that need it. + plain := found.Aliases["plain"] + assert.Equal(t, "gpt-4o-fallback", plain.ModelID) + assert.Nil(t, plain.ModelName) + assert.Nil(t, plain.ModelFamily) + assert.Nil(t, plain.AzureAliasCfg) +} + +// TestTableKey_AliasesJSON_LegacyInputRoundTrip simulates a row written before +// the refactor — raw legacy {"k":"v"} JSON in the aliases_json column — and +// verifies AfterFind promotes it to AliasConfig{ModelID: v} transparently. +func TestTableKey_AliasesJSON_LegacyInputRoundTrip(t *testing.T) { + db := setupTestDB(t) + + // First create a key without aliases so the row exists. + key := &TableKey{ + Name: "openai-key", + ProviderID: 1, + Provider: "openai", + KeyID: "openai-uuid-aliases-legacy-input", + Value: *schemas.NewEnvVar("sk-test"), + } + require.NoError(t, db.Create(key).Error) + + // Then write the legacy-shaped JSON directly into the aliases_json column, + // bypassing BeforeSave — this is what a pre-refactor row looks like. + legacy := `{"best-model":"gpt-4o-deployment"}` + encrypted, err := encrypt.Encrypt(legacy) + require.NoError(t, err) + require.NoError(t, db.Exec("UPDATE config_keys SET aliases_json = ? WHERE id = ?", encrypted, key.ID).Error) + + // Read back through GORM — AfterFind should decrypt + UnmarshalJSON should + // promote the legacy string value into AliasConfig{ModelID: ...}. + var found TableKey + require.NoError(t, db.First(&found, key.ID).Error) + require.NotNil(t, found.Aliases) + require.Len(t, found.Aliases, 1) + got := found.Aliases["best-model"] + assert.Equal(t, "gpt-4o-deployment", got.ModelID) + assert.Nil(t, got.ModelName) + assert.Nil(t, got.ModelFamily) +} diff --git a/framework/configstore/tables/framework.go b/framework/configstore/tables/framework.go index 33f270ba6f..cfb7be33ba 100644 --- a/framework/configstore/tables/framework.go +++ b/framework/configstore/tables/framework.go @@ -7,7 +7,12 @@ type TableFrameworkConfig struct { PricingURL *string `gorm:"type:text" json:"pricing_url"` PricingSyncInterval *int64 `gorm:"" json:"pricing_sync_interval"` ModelParametersURL *string `gorm:"type:text" json:"model_parameters_url"` - ConfigHash string `gorm:"type:text" json:"config_hash"` + // MCPLibraryURL is the endpoint the MCP server library catalog is synced + // from. Empty/nil falls back to modelcatalog.DefaultMCPLibraryURL. Mirrors + // PricingURL: the default ships out of the box and the user can override it. + MCPLibraryURL *string `gorm:"type:text" json:"mcp_library_url"` + MCPLibrarySyncInterval *int64 `gorm:"" json:"mcp_library_sync_interval"` + ConfigHash string `gorm:"type:text" json:"config_hash"` } // TableName sets the table name for each model diff --git a/framework/configstore/tables/key.go b/framework/configstore/tables/key.go index 51df1def45..db509f7a19 100644 --- a/framework/configstore/tables/key.go +++ b/framework/configstore/tables/key.go @@ -1,6 +1,7 @@ package tables import ( + "context" "encoding/json" "fmt" "time" @@ -276,9 +277,6 @@ func (k *TableKey) BeforeSave(tx *gorm.DB) error { } if k.Aliases != nil { - if err := k.Aliases.Validate(); err != nil { - return err - } data, err := sonic.Marshal(k.Aliases) if err != nil { return err @@ -329,7 +327,102 @@ func (k *TableKey) BeforeSave(tx *gorm.DB) error { } // Encrypt sensitive fields after serialization - if encrypt.IsEnabled() { + if VaultIsEnabled() { + base := fmt.Sprintf("%s/%s/%s", VaultPrefix(), k.TableName(), k.KeyID) + namer := tx.Statement.DB.NamingStrategy + col := func(field string) string { return base + "/" + namer.ColumnName("", field) } + // For each field: best-effort remove stale vault entry when the field is being cleared + // or switched to an env-var, then store the new value (no-op if nil/empty/env-backed). + removeVaultEnvVar(tx.Statement.Context, col("Value"), &k.Value) + if err := vaultEnvVar(tx.Statement.Context, col("Value"), &k.Value); err != nil { + return fmt.Errorf("failed to vault key value: %w", err) + } + removeVaultEnvVar(tx.Statement.Context, col("AzureEndpoint"), k.AzureEndpoint) + if err := vaultEnvVar(tx.Statement.Context, col("AzureEndpoint"), k.AzureEndpoint); err != nil { + return fmt.Errorf("failed to vault azure endpoint: %w", err) + } + removeVaultEnvVar(tx.Statement.Context, col("AzureClientID"), k.AzureClientID) + if err := vaultEnvVar(tx.Statement.Context, col("AzureClientID"), k.AzureClientID); err != nil { + return fmt.Errorf("failed to vault azure client id: %w", err) + } + removeVaultEnvVar(tx.Statement.Context, col("AzureClientSecret"), k.AzureClientSecret) + if err := vaultEnvVar(tx.Statement.Context, col("AzureClientSecret"), k.AzureClientSecret); err != nil { + return fmt.Errorf("failed to vault azure client secret: %w", err) + } + removeVaultEnvVar(tx.Statement.Context, col("AzureTenantID"), k.AzureTenantID) + if err := vaultEnvVar(tx.Statement.Context, col("AzureTenantID"), k.AzureTenantID); err != nil { + return fmt.Errorf("failed to vault azure tenant id: %w", err) + } + removeVaultEnvVar(tx.Statement.Context, col("VertexProjectID"), k.VertexProjectID) + if err := vaultEnvVar(tx.Statement.Context, col("VertexProjectID"), k.VertexProjectID); err != nil { + return fmt.Errorf("failed to vault vertex project id: %w", err) + } + removeVaultEnvVar(tx.Statement.Context, col("VertexProjectNumber"), k.VertexProjectNumber) + if err := vaultEnvVar(tx.Statement.Context, col("VertexProjectNumber"), k.VertexProjectNumber); err != nil { + return fmt.Errorf("failed to vault vertex project number: %w", err) + } + removeVaultEnvVar(tx.Statement.Context, col("VertexRegion"), k.VertexRegion) + if err := vaultEnvVar(tx.Statement.Context, col("VertexRegion"), k.VertexRegion); err != nil { + return fmt.Errorf("failed to vault vertex region: %w", err) + } + removeVaultEnvVar(tx.Statement.Context, col("VertexAuthCredentials"), k.VertexAuthCredentials) + if err := vaultEnvVar(tx.Statement.Context, col("VertexAuthCredentials"), k.VertexAuthCredentials); err != nil { + return fmt.Errorf("failed to vault vertex auth credentials: %w", err) + } + removeVaultEnvVar(tx.Statement.Context, col("BedrockAccessKey"), k.BedrockAccessKey) + if err := vaultEnvVar(tx.Statement.Context, col("BedrockAccessKey"), k.BedrockAccessKey); err != nil { + return fmt.Errorf("failed to vault bedrock access key: %w", err) + } + removeVaultEnvVar(tx.Statement.Context, col("BedrockSecretKey"), k.BedrockSecretKey) + if err := vaultEnvVar(tx.Statement.Context, col("BedrockSecretKey"), k.BedrockSecretKey); err != nil { + return fmt.Errorf("failed to vault bedrock secret key: %w", err) + } + removeVaultEnvVar(tx.Statement.Context, col("BedrockSessionToken"), k.BedrockSessionToken) + if err := vaultEnvVar(tx.Statement.Context, col("BedrockSessionToken"), k.BedrockSessionToken); err != nil { + return fmt.Errorf("failed to vault bedrock session token: %w", err) + } + removeVaultEnvVar(tx.Statement.Context, col("BedrockRegion"), k.BedrockRegion) + if err := vaultEnvVar(tx.Statement.Context, col("BedrockRegion"), k.BedrockRegion); err != nil { + return fmt.Errorf("failed to vault bedrock region: %w", err) + } + removeVaultEnvVar(tx.Statement.Context, col("BedrockARN"), k.BedrockARN) + if err := vaultEnvVar(tx.Statement.Context, col("BedrockARN"), k.BedrockARN); err != nil { + return fmt.Errorf("failed to vault bedrock arn: %w", err) + } + removeVaultEnvVar(tx.Statement.Context, col("BedrockRoleARN"), k.BedrockRoleARN) + if err := vaultEnvVar(tx.Statement.Context, col("BedrockRoleARN"), k.BedrockRoleARN); err != nil { + return fmt.Errorf("failed to vault bedrock role arn: %w", err) + } + removeVaultEnvVar(tx.Statement.Context, col("BedrockExternalID"), k.BedrockExternalID) + if err := vaultEnvVar(tx.Statement.Context, col("BedrockExternalID"), k.BedrockExternalID); err != nil { + return fmt.Errorf("failed to vault bedrock external id: %w", err) + } + removeVaultEnvVar(tx.Statement.Context, col("BedrockRoleSessionName"), k.BedrockRoleSessionName) + if err := vaultEnvVar(tx.Statement.Context, col("BedrockRoleSessionName"), k.BedrockRoleSessionName); err != nil { + return fmt.Errorf("failed to vault bedrock role session name: %w", err) + } + removeVaultString(tx.Statement.Context, col("BedrockBatchS3ConfigJSON"), k.BedrockBatchS3ConfigJSON) + if err := vaultString(tx.Statement.Context, col("BedrockBatchS3ConfigJSON"), k.BedrockBatchS3ConfigJSON); err != nil { + return fmt.Errorf("failed to vault bedrock batch s3 config: %w", err) + } + removeVaultString(tx.Statement.Context, col("AliasesJSON"), k.AliasesJSON) + if err := vaultString(tx.Statement.Context, col("AliasesJSON"), k.AliasesJSON); err != nil { + return fmt.Errorf("failed to vault aliases: %w", err) + } + removeVaultEnvVar(tx.Statement.Context, col("VLLMUrl"), k.VLLMUrl) + if err := vaultEnvVar(tx.Statement.Context, col("VLLMUrl"), k.VLLMUrl); err != nil { + return fmt.Errorf("failed to vault vllm url: %w", err) + } + removeVaultEnvVar(tx.Statement.Context, col("OllamaUrl"), k.OllamaUrl) + if err := vaultEnvVar(tx.Statement.Context, col("OllamaUrl"), k.OllamaUrl); err != nil { + return fmt.Errorf("failed to vault ollama url: %w", err) + } + removeVaultEnvVar(tx.Statement.Context, col("SGLUrl"), k.SGLUrl) + if err := vaultEnvVar(tx.Statement.Context, col("SGLUrl"), k.SGLUrl); err != nil { + return fmt.Errorf("failed to vault sgl url: %w", err) + } + k.EncryptionStatus = EncryptionStatusVault + } else if encrypt.IsEnabled() { if err := encryptEnvVar(&k.Value); err != nil { return fmt.Errorf("failed to encrypt key value: %w", err) } @@ -413,7 +506,79 @@ func (k *TableKey) BeforeSave(tx *gorm.DB) error { // AzureKeyConfig, VertexKeyConfig, etc. receive plaintext data. func (k *TableKey) AfterFind(tx *gorm.DB) error { // Decrypt sensitive fields before deserialization/reconstruction - if k.EncryptionStatus == EncryptionStatusEncrypted { + switch k.EncryptionStatus { + case EncryptionStatusVault: + ctx := context.Background() + if tx != nil && tx.Statement != nil && tx.Statement.Context != nil { + ctx = tx.Statement.Context + } + if err := resolveVaultEnvVar(ctx, &k.Value); err != nil { + return fmt.Errorf("failed to resolve vault key value: %w", err) + } + if err := resolveVaultEnvVar(ctx, k.AzureEndpoint); err != nil { + return fmt.Errorf("failed to resolve vault azure endpoint: %w", err) + } + if err := resolveVaultEnvVar(ctx, k.AzureClientID); err != nil { + return fmt.Errorf("failed to resolve vault azure client id: %w", err) + } + if err := resolveVaultEnvVar(ctx, k.AzureClientSecret); err != nil { + return fmt.Errorf("failed to resolve vault azure client secret: %w", err) + } + if err := resolveVaultEnvVar(ctx, k.AzureTenantID); err != nil { + return fmt.Errorf("failed to resolve vault azure tenant id: %w", err) + } + if err := resolveVaultEnvVar(ctx, k.VertexProjectID); err != nil { + return fmt.Errorf("failed to resolve vault vertex project id: %w", err) + } + if err := resolveVaultEnvVar(ctx, k.VertexProjectNumber); err != nil { + return fmt.Errorf("failed to resolve vault vertex project number: %w", err) + } + if err := resolveVaultEnvVar(ctx, k.VertexRegion); err != nil { + return fmt.Errorf("failed to resolve vault vertex region: %w", err) + } + if err := resolveVaultEnvVar(ctx, k.VertexAuthCredentials); err != nil { + return fmt.Errorf("failed to resolve vault vertex auth credentials: %w", err) + } + if err := resolveVaultEnvVar(ctx, k.BedrockAccessKey); err != nil { + return fmt.Errorf("failed to resolve vault bedrock access key: %w", err) + } + if err := resolveVaultEnvVar(ctx, k.BedrockSecretKey); err != nil { + return fmt.Errorf("failed to resolve vault bedrock secret key: %w", err) + } + if err := resolveVaultEnvVar(ctx, k.BedrockSessionToken); err != nil { + return fmt.Errorf("failed to resolve vault bedrock session token: %w", err) + } + if err := resolveVaultEnvVar(ctx, k.BedrockRegion); err != nil { + return fmt.Errorf("failed to resolve vault bedrock region: %w", err) + } + if err := resolveVaultEnvVar(ctx, k.BedrockARN); err != nil { + return fmt.Errorf("failed to resolve vault bedrock arn: %w", err) + } + if err := resolveVaultEnvVar(ctx, k.BedrockRoleARN); err != nil { + return fmt.Errorf("failed to resolve vault bedrock role arn: %w", err) + } + if err := resolveVaultEnvVar(ctx, k.BedrockExternalID); err != nil { + return fmt.Errorf("failed to resolve vault bedrock external id: %w", err) + } + if err := resolveVaultEnvVar(ctx, k.BedrockRoleSessionName); err != nil { + return fmt.Errorf("failed to resolve vault bedrock role session name: %w", err) + } + if err := resolveVaultString(ctx, k.BedrockBatchS3ConfigJSON); err != nil { + return fmt.Errorf("failed to resolve vault bedrock batch s3 config: %w", err) + } + if err := resolveVaultString(ctx, k.AliasesJSON); err != nil { + return fmt.Errorf("failed to resolve vault aliases: %w", err) + } + if err := resolveVaultEnvVar(ctx, k.VLLMUrl); err != nil { + return fmt.Errorf("failed to resolve vault vllm url: %w", err) + } + if err := resolveVaultEnvVar(ctx, k.OllamaUrl); err != nil { + return fmt.Errorf("failed to resolve vault ollama url: %w", err) + } + if err := resolveVaultEnvVar(ctx, k.SGLUrl); err != nil { + return fmt.Errorf("failed to resolve vault sgl url: %w", err) + } + case EncryptionStatusEncrypted: if err := decryptEnvVar(&k.Value); err != nil { return fmt.Errorf("failed to decrypt key value: %w", err) } @@ -627,3 +792,26 @@ func (k *TableKey) AfterFind(tx *gorm.DB) error { } return nil } + +// AfterDelete hook for best-effort vault cleanup on row deletion. +func (k *TableKey) AfterDelete(tx *gorm.DB) error { + if k.EncryptionStatus != EncryptionStatusVault || VaultHooks.Remove == nil { + return nil + } + colName := func(field string) string { + return tx.Statement.DB.NamingStrategy.ColumnName("", field) + } + base := fmt.Sprintf("%s/%s/%s", VaultPrefix(), k.TableName(), k.KeyID) + for _, field := range []string{ + "Value", + "AzureEndpoint", "AzureClientID", "AzureClientSecret", "AzureTenantID", + "VertexProjectID", "VertexProjectNumber", "VertexRegion", "VertexAuthCredentials", + "BedrockAccessKey", "BedrockSecretKey", "BedrockSessionToken", "BedrockRegion", + "BedrockARN", "BedrockRoleARN", "BedrockExternalID", "BedrockRoleSessionName", + "BedrockBatchS3ConfigJSON", "AliasesJSON", + "VLLMUrl", "OllamaUrl", "SGLUrl", + } { + _ = VaultHooks.Remove(tx.Statement.Context, fmt.Sprintf("%s/%s", base, colName(field))) + } + return nil +} diff --git a/framework/configstore/tables/mcp.go b/framework/configstore/tables/mcp.go index 17cd0a7458..386019ce86 100644 --- a/framework/configstore/tables/mcp.go +++ b/framework/configstore/tables/mcp.go @@ -30,8 +30,8 @@ type TableMCPClient struct { ToolSyncInterval int `gorm:"default:0" json:"tool_sync_interval"` // Per-client tool sync interval in seconds (0 = use global, negative = disabled) // Per-user OAuth: discovered tools persisted so they survive restart - DiscoveredToolsJSON string `gorm:"type:text" json:"-"` // JSON serialized map[string]schemas.ChatTool - ToolNameMappingJSON string `gorm:"type:text" json:"-"` // JSON serialized map[string]string + DiscoveredToolsJSON string `gorm:"type:text" json:"-"` // JSON serialized map[string]schemas.ChatTool + ToolNameMappingJSON string `gorm:"type:text" json:"-"` // JSON serialized map[string]string // OAuth authentication fields AuthType string `gorm:"type:varchar(20);default:'headers'" json:"auth_type"` // "none", "headers", "oauth", "per_user_oauth", "per_user_headers" @@ -57,13 +57,13 @@ type TableMCPClient struct { UpdatedAt time.Time `gorm:"index;not null" json:"updated_at"` // Virtual fields for runtime use (not stored in DB) - StdioConfig *schemas.MCPStdioConfig `gorm:"-" json:"stdio_config,omitempty"` - TLSConfig *schemas.MCPTLSConfig `gorm:"-" json:"tls_config,omitempty"` - ToolsToExecute schemas.WhiteList `gorm:"-" json:"tools_to_execute"` - ToolsToAutoExecute schemas.WhiteList `gorm:"-" json:"tools_to_auto_execute"` - Headers map[string]schemas.EnvVar `gorm:"-" json:"headers"` - AllowedExtraHeaders schemas.WhiteList `gorm:"-" json:"allowed_extra_headers"` - ToolPricing map[string]float64 `gorm:"-" json:"tool_pricing"` + StdioConfig *schemas.MCPStdioConfig `gorm:"-" json:"stdio_config,omitempty"` + TLSConfig *schemas.MCPTLSConfig `gorm:"-" json:"tls_config,omitempty"` + ToolsToExecute schemas.WhiteList `gorm:"-" json:"tools_to_execute"` + ToolsToAutoExecute schemas.WhiteList `gorm:"-" json:"tools_to_auto_execute"` + Headers map[string]schemas.EnvVar `gorm:"-" json:"headers"` + AllowedExtraHeaders schemas.WhiteList `gorm:"-" json:"allowed_extra_headers"` + ToolPricing map[string]float64 `gorm:"-" json:"tool_pricing"` DiscoveredTools map[string]schemas.ChatTool `gorm:"-" json:"-"` DiscoveredToolNameMapping map[string]string `gorm:"-" json:"-"` PerUserHeaderKeys []string `gorm:"-" json:"per_user_header_keys"` @@ -221,7 +221,7 @@ func (c *TableMCPClient) BeforeSave(tx *gorm.DB) error { // AfterFind is a GORM hook that decrypts the connection string and headers (if encrypted) // and deserializes JSON columns back into runtime structs after reading from the database. func (c *TableMCPClient) AfterFind(tx *gorm.DB) error { - if c.EncryptionStatus == "encrypted" { + if c.EncryptionStatus == EncryptionStatusEncrypted { if c.HeadersJSON != "" && c.HeadersJSON != "{}" { decrypted, err := encrypt.Decrypt(c.HeadersJSON) if err != nil { diff --git a/framework/configstore/tables/mcp_library.go b/framework/configstore/tables/mcp_library.go new file mode 100644 index 0000000000..69f32bf0ca --- /dev/null +++ b/framework/configstore/tables/mcp_library.go @@ -0,0 +1,81 @@ +package tables + +import ( + "time" + + "github.com/maximhq/bifrost/core/schemas" +) + +// TableMCPLibrary represents a single discoverable MCP server in the MCP +// library catalog. Most rows are synced from the external MCP library datasheet +// (see modelcatalog.DefaultMCPLibraryURL) on a configurable interval, mirroring +// the governance_model_pricing / governance_model_parameters tables. Orgs may +// also publish their own internal servers as "custom" rows (see Source), which +// are protected from being overwritten or resurrected by the remote sync. +// +// A row is a *template* for an schemas.MCPClientConfig: it carries the +// connection details a user needs to install the server, shaped the same way +// the live config is. The connection fields are mutually exclusive by +// ConnectionType — ConnectionURL for http/sse, StdioConfig for stdio — matching +// MCPClientConfig.ConnectionString / MCPClientConfig.StdioConfig. +// +// Each row is keyed by a stable slug derived from the display name so the sync +// upsert is idempotent. +type TableMCPLibrary struct { + ID uint `gorm:"primaryKey;autoIncrement" json:"id"` + Slug string `gorm:"type:varchar(255);not null;uniqueIndex:idx_mcp_library_slug" json:"slug"` + Name string `gorm:"type:varchar(255);not null" json:"name"` + Description string `gorm:"type:text" json:"description,omitempty"` + Category string `gorm:"type:varchar(100);index:idx_mcp_library_category" json:"category,omitempty"` + + // ConnectionType is one of schemas.MCPConnectionType ("http" | "stdio" | + // "sse") and selects which connection field below is populated. + ConnectionType schemas.MCPConnectionType `gorm:"type:varchar(20);not null" json:"connection_type"` + + // ConnectionURL is the server endpoint for http/sse entries (parallel to + // MCPClientConfig.ConnectionString). Empty for stdio entries. Stored as a + // plain template string — the catalog publishes no secrets, so callers + // supply auth at install time. + ConnectionURL string `gorm:"type:text" json:"connection_url,omitempty"` + + // StdioConfig holds the command/args/env names for stdio entries (parallel + // to MCPClientConfig.StdioConfig). Nil for http/sse entries. Envs lists the + // environment variable *names* the user must provide locally; no values are + // ever published in the catalog. + StdioConfig *schemas.MCPStdioConfig `gorm:"type:text;serializer:json;default:null" json:"stdio_config,omitempty"` + + // AuthType declares what authentication the server expects (none, headers, + // oauth, ...) so the install UI can prompt accordingly. RequiredHeaderKeys + // lists the header names a headers/per-user-headers server needs — values + // are supplied by the user at install time, never stored in the catalog. + AuthType schemas.MCPAuthType `gorm:"type:varchar(20);default:'none'" json:"auth_type,omitempty"` + RequiredHeaderKeys []string `gorm:"type:text;serializer:json;default:null" json:"required_header_keys,omitempty"` + + // Presentation / discovery metadata. + IconURL string `gorm:"type:text" json:"icon_url,omitempty"` + DocsURL string `gorm:"type:text" json:"docs_url,omitempty"` + Publisher string `gorm:"type:varchar(255)" json:"publisher,omitempty"` + Tags []string `gorm:"type:text;serializer:json;default:null" json:"tags,omitempty"` + Metadata map[string]any `gorm:"type:text;serializer:json;default:null" json:"metadata,omitempty"` + + // Source distinguishes remote-synced rows ("remote") from org-internal rows + // a user published through the API ("custom"). Custom rows are protected from + // the remote sync: a slug clash in the remote payload is skipped, never + // overwritten. Defaults to "remote" so existing rows and the sync upsert keep + // their old behavior. + Source string `gorm:"type:varchar(20);not null;default:'remote';index:idx_mcp_library_source" json:"source"` + + // DeletedAt is a soft-delete tombstone (nil = visible). A user may hide any + // entry — including a remote-seeded one — and the tombstone must survive the + // next sync so the row is never resurrected. This is a plain nullable + // timestamp rather than gorm.DeletedAt on purpose: the sync upsert keys off + // slug and must still see tombstoned rows by slug to skip them; gorm's + // soft-delete would hide them from that lookup and let duplicates reinsert. + DeletedAt *time.Time `gorm:"index:idx_mcp_library_deleted_at;default:null" json:"-"` + + CreatedAt time.Time `gorm:"index;not null" json:"created_at"` + UpdatedAt time.Time `gorm:"index;not null" json:"updated_at"` +} + +// TableName sets the table name for the MCP library catalog. +func (TableMCPLibrary) TableName() string { return "mcp_library" } diff --git a/framework/configstore/tables/mcp_per_user_headers.go b/framework/configstore/tables/mcp_per_user_headers.go index ba863a1a6d..9a16a67d31 100644 --- a/framework/configstore/tables/mcp_per_user_headers.go +++ b/framework/configstore/tables/mcp_per_user_headers.go @@ -1,6 +1,7 @@ package tables import ( + "context" "encoding/json" "fmt" "time" @@ -66,17 +67,17 @@ func (f *TableMCPPerUserHeaderFlow) BeforeSave(tx *gorm.DB) error { // oauth_user_tokens). Schema (i.e. the set of allowed header names) lives on // TableMCPClient.PerUserHeaderKeysJSON; this table holds the values only. type TableMCPPerUserHeaderCredential struct { - ID string `gorm:"type:varchar(255);primaryKey" json:"id"` // UUID - SessionID string `gorm:"type:varchar(255);index" json:"session_id,omitempty"` // Session-mode identity: client-asserted x-bf-mcp-session-id. Empty for vk/user mode rows. - VirtualKeyID *string `gorm:"type:varchar(255);index" json:"virtual_key_id"` // VK identity (vk-mode rows) - UserID *string `gorm:"type:varchar(255);index" json:"user_id"` // User identity (user-mode rows) - MCPClientID string `gorm:"type:varchar(255);not null;index" json:"mcp_client_id"` // Which MCP server - AuthMode string `gorm:"type:varchar(20);not null" json:"auth_mode"` // 'user' | 'vk' | 'session' — which identity column keys this row - Status string `gorm:"type:varchar(20);not null;default:'active'" json:"status"` // 'active' | 'orphaned' | 'needs_update' - HeadersJSON string `gorm:"type:text;not null" json:"-"` // Encrypted JSON map[string]string of user-supplied header values - EncryptionStatus string `gorm:"type:varchar(20);default:'plain_text'" json:"-"` - CreatedAt time.Time `gorm:"index;not null" json:"created_at"` - UpdatedAt time.Time `gorm:"index;not null" json:"updated_at"` + ID string `gorm:"type:varchar(255);primaryKey" json:"id"` // UUID + SessionID string `gorm:"type:varchar(255);index" json:"session_id,omitempty"` // Session-mode identity: client-asserted x-bf-mcp-session-id. Empty for vk/user mode rows. + VirtualKeyID *string `gorm:"type:varchar(255);index" json:"virtual_key_id"` // VK identity (vk-mode rows) + UserID *string `gorm:"type:varchar(255);index" json:"user_id"` // User identity (user-mode rows) + MCPClientID string `gorm:"type:varchar(255);not null;index" json:"mcp_client_id"` // Which MCP server + AuthMode string `gorm:"type:varchar(20);not null" json:"auth_mode"` // 'user' | 'vk' | 'session' — which identity column keys this row + Status string `gorm:"type:varchar(20);not null;default:'active'" json:"status"` // 'active' | 'orphaned' | 'needs_update' + HeadersJSON string `gorm:"type:text;not null" json:"-"` // Encrypted JSON map[string]string of user-supplied header values + EncryptionStatus string `gorm:"type:varchar(20);default:'plain_text'" json:"-"` + CreatedAt time.Time `gorm:"index;not null" json:"created_at"` + UpdatedAt time.Time `gorm:"index;not null" json:"updated_at"` // Display-only relations (no DB-level FK constraint; preloaded for sessions UI). MCPClient *TableMCPClient `gorm:"foreignKey:MCPClientID;references:ClientID" json:"-"` @@ -122,6 +123,30 @@ func (c *TableMCPPerUserHeaderCredential) AfterFind(tx *gorm.DB) error { return nil } +// AfterDelete hook for best-effort vault cleanup on row deletion. +func (c *TableMCPPerUserHeaderCredential) AfterDelete(tx *gorm.DB) error { + if c.EncryptionStatus != EncryptionStatusVault || VaultHooks.Remove == nil { + return nil + } + headersField := tx.Statement.DB.NamingStrategy.ColumnName("", "HeadersJSON") + path := fmt.Sprintf("%s/%s/%s/%s", VaultPrefix(), c.TableName(), c.ID, headersField) + _ = VaultHooks.Remove(tx.Statement.Context, path) + return nil +} + +// DeleteVaultSecrets removes vault entries for the given credential IDs. +// Called after a batch delete so vault cleanup runs even when AfterDelete can't fire. +func (TableMCPPerUserHeaderCredential) DeleteVaultSecrets(ctx context.Context, ids []string) { + if VaultHooks.Remove == nil { + return + } + tableName := TableMCPPerUserHeaderCredential{}.TableName() + for _, id := range ids { + path := fmt.Sprintf("%s/%s/%s/headers_json", VaultPrefix(), tableName, id) + _ = VaultHooks.Remove(ctx, path) + } +} + // SetHeaders serializes the caller-supplied header map into HeadersJSON. // Callers must use this rather than assigning HeadersJSON directly so the // JSON shape stays consistent. diff --git a/framework/configstore/tables/modelpricing.go b/framework/configstore/tables/modelpricing.go index 7c99b6fba2..2b015daf90 100644 --- a/framework/configstore/tables/modelpricing.go +++ b/framework/configstore/tables/modelpricing.go @@ -28,7 +28,11 @@ type TableModelPricing struct { OutputCostPerTokenPriority *float64 `gorm:"default:null;column:output_cost_per_token_priority" json:"output_cost_per_token_priority,omitempty"` InputCostPerTokenFlex *float64 `gorm:"default:null;column:input_cost_per_token_flex" json:"input_cost_per_token_flex,omitempty"` OutputCostPerTokenFlex *float64 `gorm:"default:null;column:output_cost_per_token_flex" json:"output_cost_per_token_flex,omitempty"` - InputCostPerCharacter *float64 `gorm:"default:null;column:input_cost_per_character" json:"input_cost_per_character,omitempty"` + // Fast mode (Anthropic research preview, speed:"fast" on Opus 4.6/4.7/4.8). + // Flat rate across the full context window; cache tokens bill at standard cache rates. + InputCostPerTokenFast *float64 `gorm:"default:null;column:input_cost_per_token_fast" json:"input_cost_per_token_fast,omitempty"` + OutputCostPerTokenFast *float64 `gorm:"default:null;column:output_cost_per_token_fast" json:"output_cost_per_token_fast,omitempty"` + InputCostPerCharacter *float64 `gorm:"default:null;column:input_cost_per_character" json:"input_cost_per_character,omitempty"` // Costs - 128k Tier InputCostPerTokenAbove128kTokens *float64 `gorm:"default:null;column:input_cost_per_token_above_128k_tokens" json:"input_cost_per_token_above_128k_tokens,omitempty"` InputCostPerImageAbove128kTokens *float64 `gorm:"default:null;column:input_cost_per_image_above_128k_tokens" json:"input_cost_per_image_above_128k_tokens,omitempty"` diff --git a/framework/configstore/tables/oauth.go b/framework/configstore/tables/oauth.go index 980479c24c..0ee01c4576 100644 --- a/framework/configstore/tables/oauth.go +++ b/framework/configstore/tables/oauth.go @@ -46,8 +46,28 @@ func (c *TableOauthConfig) BeforeSave(tx *gorm.DB) error { c.Status = "pending" } - // Encrypt sensitive fields (skip if value is an env var reference — the reference itself is not sensitive) - if encrypt.IsEnabled() { + if VaultIsEnabled() { + vaulted := false + if c.ClientSecret != nil && !c.ClientSecret.FromEnv && c.ClientSecret.Val != "" { + path := fmt.Sprintf("%s/%s/%s/%s", VaultPrefix(), c.TableName(), c.ID, + tx.Statement.DB.NamingStrategy.ColumnName("", "ClientSecret")) + if err := vaultEnvVar(tx.Statement.Context, path, c.ClientSecret); err != nil { + return fmt.Errorf("failed to vault oauth client secret: %w", err) + } + vaulted = true + } + if c.CodeVerifier != "" { + path := fmt.Sprintf("%s/%s/%s/%s", VaultPrefix(), c.TableName(), c.ID, + tx.Statement.DB.NamingStrategy.ColumnName("", "CodeVerifier")) + if err := vaultString(tx.Statement.Context, path, &c.CodeVerifier); err != nil { + return fmt.Errorf("failed to vault oauth code verifier: %w", err) + } + vaulted = true + } + if vaulted { + c.EncryptionStatus = EncryptionStatusVault + } + } else if encrypt.IsEnabled() { encrypted := false if c.ClientSecret != nil && !c.ClientSecret.FromEnv && c.ClientSecret.Val != "" { if err := encryptString(&c.ClientSecret.Val); err != nil { @@ -70,7 +90,15 @@ func (c *TableOauthConfig) BeforeSave(tx *gorm.DB) error { // AfterFind hook to decrypt sensitive fields func (c *TableOauthConfig) AfterFind(tx *gorm.DB) error { - if c.EncryptionStatus == EncryptionStatusEncrypted { + switch c.EncryptionStatus { + case EncryptionStatusVault: + if err := resolveVaultEnvVar(tx.Statement.Context, c.ClientSecret); err != nil { + return fmt.Errorf("failed to resolve vault oauth client secret: %w", err) + } + if err := resolveVaultString(tx.Statement.Context, &c.CodeVerifier); err != nil { + return fmt.Errorf("failed to resolve vault oauth code verifier: %w", err) + } + case EncryptionStatusEncrypted: if c.ClientSecret != nil && !c.ClientSecret.FromEnv && c.ClientSecret.Val != "" { if err := decryptString(&c.ClientSecret.Val); err != nil { return fmt.Errorf("failed to decrypt oauth client secret: %w", err) @@ -83,6 +111,18 @@ func (c *TableOauthConfig) AfterFind(tx *gorm.DB) error { return nil } +// AfterDelete hook for best-effort vault cleanup on row deletion. +func (c *TableOauthConfig) AfterDelete(tx *gorm.DB) error { + if c.EncryptionStatus != EncryptionStatusVault || VaultHooks.Remove == nil { + return nil + } + secretField := tx.Statement.DB.NamingStrategy.ColumnName("", "ClientSecret") + verifierField := tx.Statement.DB.NamingStrategy.ColumnName("", "CodeVerifier") + _ = VaultHooks.Remove(tx.Statement.Context, fmt.Sprintf("%s/%s/%s/%s", VaultPrefix(), c.TableName(), c.ID, secretField)) + _ = VaultHooks.Remove(tx.Statement.Context, fmt.Sprintf("%s/%s/%s/%s", VaultPrefix(), c.TableName(), c.ID, verifierField)) + return nil +} + // GetResolvedClientID returns the resolved ClientID value, expanding env var references at runtime. func (c *TableOauthConfig) GetResolvedClientID() string { return c.ClientID.GetValue() @@ -124,8 +164,6 @@ func (t *TableOauthToken) BeforeSave(tx *gorm.DB) error { if t.TokenType == "" { t.TokenType = "Bearer" } - - // Encrypt sensitive fields if encrypt.IsEnabled() { if err := encryptString(&t.AccessToken); err != nil { return fmt.Errorf("failed to encrypt oauth access token: %w", err) diff --git a/framework/configstore/tables/plugin.go b/framework/configstore/tables/plugin.go index 2b61b9b1b4..9fae984988 100644 --- a/framework/configstore/tables/plugin.go +++ b/framework/configstore/tables/plugin.go @@ -24,7 +24,7 @@ type TablePlugin struct { IsCustom bool `gorm:"not null;default:false" json:"isCustom"` Placement *schemas.PluginPlacement `gorm:"column:placement;type:varchar(20);null" json:"placement,omitempty"` - Order *int `gorm:"column:exec_order;type:int;null" json:"order,omitempty"` + Order *int `gorm:"column:exec_order;type:int;null" json:"order,omitempty"` // Config hash is used to detect the changes synced from config.json file // Every time we sync the config.json file, we will update the config hash @@ -53,7 +53,14 @@ func (p *TablePlugin) BeforeSave(tx *gorm.DB) error { } // Encrypt config after serialization - if encrypt.IsEnabled() && p.ConfigJSON != "" && p.ConfigJSON != "{}" { + if VaultIsEnabled() && p.ConfigJSON != "" && p.ConfigJSON != "{}" { + fieldName := tx.Statement.DB.NamingStrategy.ColumnName("", "ConfigJSON") + path := fmt.Sprintf("%s/%s/%s/%s", VaultPrefix(), p.TableName(), p.Name, fieldName) + if err := vaultString(tx.Statement.Context, path, &p.ConfigJSON); err != nil { + return fmt.Errorf("failed to vault plugin config: %w", err) + } + p.EncryptionStatus = EncryptionStatusVault + } else if encrypt.IsEnabled() && p.ConfigJSON != "" && p.ConfigJSON != "{}" { encrypted, err := encrypt.Encrypt(p.ConfigJSON) if err != nil { return fmt.Errorf("failed to encrypt plugin config: %w", err) @@ -68,12 +75,21 @@ func (p *TablePlugin) BeforeSave(tx *gorm.DB) error { // AfterFind is a GORM hook that decrypts the plugin config JSON (if encrypted) and // deserializes it back into the runtime Config field after reading from the database. func (p *TablePlugin) AfterFind(tx *gorm.DB) error { - if p.EncryptionStatus == "encrypted" && p.ConfigJSON != "" { - decrypted, err := encrypt.Decrypt(p.ConfigJSON) - if err != nil { - return fmt.Errorf("failed to decrypt plugin config: %w", err) + switch p.EncryptionStatus { + case EncryptionStatusVault: + if p.ConfigJSON != "" { + if err := resolveVaultString(tx.Statement.Context, &p.ConfigJSON); err != nil { + return fmt.Errorf("failed to resolve vault plugin config: %w", err) + } + } + case EncryptionStatusEncrypted: + if p.ConfigJSON != "" { + decrypted, err := encrypt.Decrypt(p.ConfigJSON) + if err != nil { + return fmt.Errorf("failed to decrypt plugin config: %w", err) + } + p.ConfigJSON = decrypted } - p.ConfigJSON = decrypted } if p.ConfigJSON != "" { if err := json.Unmarshal([]byte(p.ConfigJSON), &p.Config); err != nil { @@ -85,3 +101,14 @@ func (p *TablePlugin) AfterFind(tx *gorm.DB) error { return nil } + +// AfterDelete hook for best-effort vault cleanup on row deletion. +func (p *TablePlugin) AfterDelete(tx *gorm.DB) error { + if p.EncryptionStatus != EncryptionStatusVault || VaultHooks.Remove == nil { + return nil + } + fieldName := tx.Statement.DB.NamingStrategy.ColumnName("", "ConfigJSON") + path := fmt.Sprintf("%s/%s/%s/%s", VaultPrefix(), p.TableName(), p.Name, fieldName) + _ = VaultHooks.Remove(tx.Statement.Context, path) + return nil +} diff --git a/framework/configstore/tables/provider.go b/framework/configstore/tables/provider.go index dcc253cec6..5fe538d0d0 100644 --- a/framework/configstore/tables/provider.go +++ b/framework/configstore/tables/provider.go @@ -118,7 +118,14 @@ func (p *TableProvider) BeforeSave(tx *gorm.DB) error { } // Encrypt proxy config after serialization (only if there's data to encrypt) - if encrypt.IsEnabled() && p.ProxyConfigJSON != "" { + if VaultIsEnabled() && p.ProxyConfigJSON != "" && p.ProxyConfigJSON != "{}" { + fieldName := tx.Statement.DB.NamingStrategy.ColumnName("", "ProxyConfigJSON") + path := fmt.Sprintf("%s/%s/%s/%s", VaultPrefix(), p.TableName(), p.Name, fieldName) + if err := vaultString(tx.Statement.Context, path, &p.ProxyConfigJSON); err != nil { + return fmt.Errorf("failed to vault proxy config: %w", err) + } + p.EncryptionStatus = EncryptionStatusVault + } else if encrypt.IsEnabled() && p.ProxyConfigJSON != "" { encrypted, err := encrypt.Encrypt(p.ProxyConfigJSON) if err != nil { return fmt.Errorf("failed to encrypt proxy config: %w", err) @@ -156,6 +163,11 @@ func (p *TableProvider) AfterFind(tx *gorm.DB) error { } p.ProxyConfigJSON = decrypted } + if p.EncryptionStatus == EncryptionStatusVault && p.ProxyConfigJSON != "" { + if err := resolveVaultString(tx.Statement.Context, &p.ProxyConfigJSON); err != nil { + return fmt.Errorf("failed to resolve vault proxy config: %w", err) + } + } if p.ProxyConfigJSON != "" { var proxyConfig schemas.ProxyConfig if err := json.Unmarshal([]byte(p.ProxyConfigJSON), &proxyConfig); err != nil { @@ -182,3 +194,14 @@ func (p *TableProvider) AfterFind(tx *gorm.DB) error { return nil } + +// AfterDelete hook for best-effort vault cleanup on row deletion. +func (p *TableProvider) AfterDelete(tx *gorm.DB) error { + if p.EncryptionStatus != EncryptionStatusVault || VaultHooks.Remove == nil { + return nil + } + fieldName := tx.Statement.DB.NamingStrategy.ColumnName("", "ProxyConfigJSON") + path := fmt.Sprintf("%s/%s/%s/%s", VaultPrefix(), p.TableName(), p.Name, fieldName) + _ = VaultHooks.Remove(tx.Statement.Context, path) + return nil +} diff --git a/framework/configstore/tables/vectorstore.go b/framework/configstore/tables/vectorstore.go index e9563f5bc0..fe4e3165ec 100644 --- a/framework/configstore/tables/vectorstore.go +++ b/framework/configstore/tables/vectorstore.go @@ -10,13 +10,13 @@ import ( // TableVectorStoreConfig represents Cache plugin configuration in the database type TableVectorStoreConfig struct { - ID uint `gorm:"primaryKey;autoIncrement" json:"id"` - Enabled bool `json:"enabled"` // Enable vector store - Type string `gorm:"type:varchar(50);not null" json:"type"` // "weaviate, redis, qdrant." - TTLSeconds int `gorm:"default:300" json:"ttl_seconds"` // TTL in seconds (default: 5 minutes) - CacheByModel bool `gorm:"" json:"cache_by_model"` // Include model in cache key - CacheByProvider bool `gorm:"" json:"cache_by_provider"` // Include provider in cache key - Config *string `gorm:"type:text" json:"config"` // JSON serialized schemas.RedisVectorStoreConfig + ID uint `gorm:"primaryKey;autoIncrement" json:"id"` + Enabled bool `json:"enabled"` // Enable vector store + Type string `gorm:"type:varchar(50);not null" json:"type"` // "weaviate, redis, qdrant." + TTLSeconds int `gorm:"default:300" json:"ttl_seconds"` // TTL in seconds (default: 5 minutes) + CacheByModel bool `gorm:"" json:"cache_by_model"` // Include model in cache key + CacheByProvider bool `gorm:"" json:"cache_by_provider"` // Include provider in cache key + Config *string `gorm:"type:text" json:"config"` // JSON serialized schemas.RedisVectorStoreConfig EncryptionStatus string `gorm:"type:varchar(20);default:'plain_text'" json:"-"` CreatedAt time.Time `gorm:"index;not null" json:"created_at"` UpdatedAt time.Time `gorm:"index;not null" json:"updated_at"` diff --git a/framework/configstore/tables/virtualkey.go b/framework/configstore/tables/virtualkey.go index 9343984231..12765f4d6f 100644 --- a/framework/configstore/tables/virtualkey.go +++ b/framework/configstore/tables/virtualkey.go @@ -24,14 +24,14 @@ func (TableVirtualKeyProviderConfigKey) TableName() string { // TableVirtualKeyProviderConfig represents a provider configuration for a virtual key type TableVirtualKeyProviderConfig struct { - ID uint `gorm:"primaryKey;autoIncrement" json:"id"` - VirtualKeyID string `gorm:"type:varchar(255);not null" json:"virtual_key_id"` - Provider string `gorm:"type:varchar(50);not null" json:"provider"` - Weight *float64 `json:"weight"` - AllowedModels schemas.WhiteList `gorm:"type:text;serializer:json" json:"allowed_models"` // ["*"] allows all models; empty denies all (deny-by-default) - BlacklistedModels schemas.BlackList `gorm:"type:text;serializer:json" json:"blacklisted_models"` // ["*"] blocks all models; empty blocks none - AllowAllKeys bool `gorm:"default:false" json:"allow_all_keys"` // True means all keys allowed; false with empty Keys means no keys allowed (deny-by-default) - RateLimitID *string `gorm:"type:varchar(255);index" json:"rate_limit_id,omitempty"` + ID uint `gorm:"primaryKey;autoIncrement" json:"id"` + VirtualKeyID string `gorm:"type:varchar(255);not null" json:"virtual_key_id"` + Provider string `gorm:"type:varchar(50);not null" json:"provider"` + Weight *float64 `json:"weight"` + AllowedModels schemas.WhiteList `gorm:"type:text;serializer:json" json:"allowed_models"` // ["*"] allows all models; empty denies all (deny-by-default) + BlacklistedModels schemas.BlackList `gorm:"type:text;serializer:json" json:"blacklisted_models"` // ["*"] blocks all models; empty blocks none + AllowAllKeys bool `gorm:"default:false" json:"allow_all_keys"` // True means all keys allowed; false with empty Keys means no keys allowed (deny-by-default) + RateLimitID *string `gorm:"type:varchar(255);index" json:"rate_limit_id,omitempty"` // Relationships RateLimit *TableRateLimit `gorm:"foreignKey:RateLimitID;onDelete:CASCADE" json:"rate_limit,omitempty"` @@ -267,7 +267,14 @@ func (vk *TableVirtualKey) BeforeSave(tx *gorm.DB) error { if vk.Value != "" { vk.ValueHash = encrypt.HashSHA256(vk.Value) } - if encrypt.IsEnabled() && vk.Value != "" { + if VaultIsEnabled() && vk.Value != "" { + fieldName := tx.Statement.DB.NamingStrategy.ColumnName("", "Value") + path := fmt.Sprintf("%s/%s/%s/%s", VaultPrefix(), vk.TableName(), vk.ID, fieldName) + if err := vaultString(tx.Statement.Context, path, &vk.Value); err != nil { + return fmt.Errorf("failed to vault virtual key value: %w", err) + } + vk.EncryptionStatus = EncryptionStatusVault + } else if encrypt.IsEnabled() && vk.Value != "" { if err := encryptString(&vk.Value); err != nil { return fmt.Errorf("failed to encrypt virtual key value: %w", err) } @@ -282,7 +289,12 @@ func (vk *TableVirtualKey) BeforeSave(tx *gorm.DB) error { // The reset path reads the stamped value; Update*InMemory paths re-stamp on // every VK update. func (vk *TableVirtualKey) AfterFind(tx *gorm.DB) error { - if vk.EncryptionStatus == EncryptionStatusEncrypted { + switch vk.EncryptionStatus { + case EncryptionStatusVault: + if err := resolveVaultString(tx.Statement.Context, &vk.Value); err != nil { + return fmt.Errorf("failed to resolve vault virtual key value: %w", err) + } + case EncryptionStatusEncrypted: if err := decryptString(&vk.Value); err != nil { return fmt.Errorf("failed to decrypt virtual key value: %w", err) } @@ -304,3 +316,14 @@ func (vk *TableVirtualKey) AfterFind(tx *gorm.DB) error { } return nil } + +// AfterDelete hook for best-effort vault cleanup on row deletion. +func (vk *TableVirtualKey) AfterDelete(tx *gorm.DB) error { + if vk.EncryptionStatus != EncryptionStatusVault || VaultHooks.Remove == nil { + return nil + } + fieldName := tx.Statement.DB.NamingStrategy.ColumnName("", "Value") + path := fmt.Sprintf("%s/%s/%s/%s", VaultPrefix(), vk.TableName(), vk.ID, fieldName) + _ = VaultHooks.Remove(tx.Statement.Context, path) + return nil +} diff --git a/framework/go.mod b/framework/go.mod index 8bb4ae0edc..a899a9db6e 100644 --- a/framework/go.mod +++ b/framework/go.mod @@ -26,7 +26,7 @@ require ( cloud.google.com/go v0.123.0 // indirect cloud.google.com/go/auth v0.20.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect - cloud.google.com/go/iam v1.5.3 // indirect + cloud.google.com/go/iam v1.7.0 // indirect cloud.google.com/go/monitoring v1.24.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect @@ -98,19 +98,19 @@ require ( require ( cloud.google.com/go/compute/metadata v0.9.0 // indirect github.com/andybalholm/brotli v1.2.1 // indirect - github.com/aws/aws-sdk-go-v2 v1.41.7 + github.com/aws/aws-sdk-go-v2 v1.41.12 github.com/aws/aws-sdk-go-v2/config v1.32.11 github.com/aws/aws-sdk-go-v2/credentials v1.19.14 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 - github.com/aws/smithy-go v1.25.1 // indirect + github.com/aws/smithy-go v1.27.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.2 // indirect github.com/bytedance/sonic v1.15.1 diff --git a/framework/go.sum b/framework/go.sum index 806a0be8d4..e73a765676 100644 --- a/framework/go.sum +++ b/framework/go.sum @@ -8,8 +8,8 @@ cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIi cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= -cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= +cloud.google.com/go/iam v1.7.0 h1:JD3zh0C6LHl16aCn5Akff0+GELdp1+4hmh6ndoFLl8U= +cloud.google.com/go/iam v1.7.0/go.mod h1:tetWZW1PD/m6vcuY2Zj/aU0eCHNPuxedbnbRTyKXvdY= cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHAjA= cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZCn6hSBtKak= cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8= @@ -45,8 +45,8 @@ github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eT github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= -github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= -github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2 v1.41.12 h1:DIKX2c31ekm9RA2D9FBj1EWXx++9AdAqRw+e78Tq2Ck= +github.com/aws/aws-sdk-go-v2 v1.41.12/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= github.com/aws/aws-sdk-go-v2/config v1.32.11 h1:ftxI5sgz8jZkckuUHXfC/wMUc8u3fG1vQS0plr2F2Zs= @@ -55,10 +55,10 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8TH github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 h1:Xf2j7NdVcUKomlZ4iihOP4AZ3Fzlr8h4yKpXeP+OFPg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28/go.mod h1:O8cDo1dW63jU7ki//kRe1z+tLGcpnD1jrouitsQddDw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 h1:KqIfN9kpkKkcBqBbNpNGTIrXO6ExTUvFKvXkC+YAzVo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28/go.mod h1:uxtQiKvLtNS4iXVsH2McVD/ls8FKN/uUhe1hGxPjrw0= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 h1:clHU5fm//kWS1C2HgtgWxfQbFbx4b6rx+5jzhgX9HrI= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= @@ -81,8 +81,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6f github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= -github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= -github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8= +github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= diff --git a/framework/logstore/hybrid_test.go b/framework/logstore/hybrid_test.go index a927962bf7..31ff9da88f 100644 --- a/framework/logstore/hybrid_test.go +++ b/framework/logstore/hybrid_test.go @@ -594,7 +594,7 @@ func TestHybrid_Tags(t *testing.T) { assert.Equal(t, "2026-04-03", tags["date"]) } -func TestHybrid_MetadataIsRetainedInDBAndExcludedFromObjectPayload(t *testing.T) { +func TestHybrid_MetadataIsRetainedInDBAndWrittenToObjectPayload(t *testing.T) { hybrid, inner, objStore := newTestHybrid(t) defer hybrid.Close(context.Background()) ctx := context.Background() @@ -625,14 +625,16 @@ func TestHybrid_MetadataIsRetainedInDBAndExcludedFromObjectPayload(t *testing.T) require.NotNil(t, dbLog.Metadata) assert.Contains(t, *dbLog.Metadata, "cortex-user-id") - // Metadata is DB-authoritative and must never be written to the object - // store snapshot. + // Metadata is DB-authoritative but a copy is written to the object store + // snapshot so consumers reading objects directly see custom attributes. key := ObjectKey("test", ts, "metadata-1") rawPayload, err := objStore.Get(ctx, key) require.NoError(t, err) var payload map[string]string require.NoError(t, sonic.Unmarshal(rawPayload, &payload)) - assert.NotContains(t, payload, "metadata", "metadata must not be written to the object store snapshot") + require.Contains(t, payload, "metadata", "metadata must be written to the object store snapshot") + assert.Contains(t, payload["metadata"], "cortex-user-id") + assert.Contains(t, payload["metadata"], "payments") // Hydration still returns metadata, sourced from the DB row. found, err := hybrid.FindByID(ctx, "metadata-1") diff --git a/framework/logstore/metadata_filter_fix_test.go b/framework/logstore/metadata_filter_fix_test.go new file mode 100644 index 0000000000..86e7f32562 --- /dev/null +++ b/framework/logstore/metadata_filter_fix_test.go @@ -0,0 +1,195 @@ +package logstore + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// insertLogWithMetadata inserts a log row with the given JSON metadata string via raw SQL, +// bypassing GORM serialization so the exact byte sequence reaches the database. +func insertLogWithMetadata(t *testing.T, store *RDBLogStore, id string, metadataJSON string, ts time.Time) { + t.Helper() + err := store.db.Exec(` + INSERT INTO logs (id, timestamp, object_type, provider, model, status, metadata, created_at) + VALUES (?, ?, 'chat.completion', 'openai', 'gpt-4o', 'success', ?, ?) + `, id, ts, metadataJSON, ts).Error + require.NoError(t, err, "failed to insert log %q", id) +} + +// runMetadataStringMatchingSuite verifies that metadata filter values that look like numbers +// are matched as JSON strings. Both SQLite and Postgres store all metadata values as JSON +// strings (from HTTP headers), so both dialects must match them correctly. +// +// Boolean values ("true"/"false") are excluded: SQLite intentionally uses json_type to match +// JSON booleans for those values, while Postgres always matches as a JSON string. +func runMetadataStringMatchingSuite(t *testing.T, store *RDBLogStore) { + t.Helper() + ctx := context.Background() + now := time.Now().UTC() + + insertLogWithMetadata(t, store, "meta-num", `{"chat-id": "4000126002"}`, now.Add(-1*time.Second)) + insertLogWithMetadata(t, store, "meta-float", `{"score": "3.14"}`, now.Add(-2*time.Second)) + insertLogWithMetadata(t, store, "meta-str", `{"env": "production"}`, now.Add(-3*time.Second)) + // Row with different value — must NOT appear in filtered results. + insertLogWithMetadata(t, store, "meta-other", `{"chat-id": "9999"}`, now.Add(-4*time.Second)) + + tests := []struct { + name string + filters map[string]string + wantIDs []string + wantMiss []string + }{ + { + name: "numeric_string_value_matches", + filters: map[string]string{"chat-id": "4000126002"}, + wantIDs: []string{"meta-num"}, + wantMiss: []string{"meta-other"}, + }, + { + name: "float_string_matches", + filters: map[string]string{"score": "3.14"}, + wantIDs: []string{"meta-float"}, + }, + { + name: "plain_string_matches", + filters: map[string]string{"env": "production"}, + wantIDs: []string{"meta-str"}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result, err := store.SearchLogs(ctx, SearchFilters{MetadataFilters: tc.filters}, PaginationOptions{Limit: 100}) + require.NoError(t, err) + require.NotNil(t, result) + + gotIDs := make(map[string]bool, len(result.Logs)) + for _, l := range result.Logs { + gotIDs[l.ID] = true + } + for _, wantID := range tc.wantIDs { + assert.True(t, gotIDs[wantID], "expected log %q to be returned by filter %v", wantID, tc.filters) + } + for _, missID := range tc.wantMiss { + assert.False(t, gotIDs[missID], "log %q should NOT be returned by filter %v", missID, tc.filters) + } + }) + } +} + +// runPostgresMetadataBooleanStringMatchingSuite verifies that values like "true" and "false" +// that come from HTTP headers are stored as JSON strings and match as strings via JSONB @> +// containment. The old code emitted a JSON boolean fragment {"active": true} which never +// matched the stored string "true". +func runPostgresMetadataBooleanStringMatchingSuite(t *testing.T, store *RDBLogStore) { + t.Helper() + ctx := context.Background() + now := time.Now().UTC() + + insertLogWithMetadata(t, store, "pg-bool-true", `{"active": "true"}`, now.Add(-1*time.Second)) + insertLogWithMetadata(t, store, "pg-bool-false", `{"active": "false"}`, now.Add(-2*time.Second)) + + t.Run("boolean_true_string_matches", func(t *testing.T) { + result, err := store.SearchLogs(ctx, SearchFilters{MetadataFilters: map[string]string{"active": "true"}}, PaginationOptions{Limit: 100}) + require.NoError(t, err) + require.NotNil(t, result) + gotIDs := make(map[string]bool) + for _, l := range result.Logs { + gotIDs[l.ID] = true + } + assert.True(t, gotIDs["pg-bool-true"], "stored string 'true' must match filter active=true on Postgres") + assert.False(t, gotIDs["pg-bool-false"], "log with active='false' must not match filter active=true") + }) + + t.Run("boolean_false_string_matches", func(t *testing.T) { + result, err := store.SearchLogs(ctx, SearchFilters{MetadataFilters: map[string]string{"active": "false"}}, PaginationOptions{Limit: 100}) + require.NoError(t, err) + require.NotNil(t, result) + gotIDs := make(map[string]bool) + for _, l := range result.Logs { + gotIDs[l.ID] = true + } + assert.True(t, gotIDs["pg-bool-false"], "stored string 'false' must match filter active=false on Postgres") + assert.False(t, gotIDs["pg-bool-true"], "log with active='true' must not match filter active=false") + }) +} + +// runPaginationTotalCountSuite verifies that SearchLogs sets pagination.TotalCount correctly. +// Previously, totalCount was computed but only stored in Stats.TotalRequests — never +// assigned to Pagination.TotalCount — so every response returned total_count: 0. +func runPaginationTotalCountSuite(t *testing.T, store *RDBLogStore) { + t.Helper() + ctx := context.Background() + now := time.Now().UTC() + + const total = 5 + const pageLimit = 2 + + for i := 0; i < total; i++ { + err := store.Create(ctx, &Log{ + ID: fmt.Sprintf("%s-log-%d", t.Name(), i), + Timestamp: now.Add(-time.Duration(i) * time.Second), + Object: "chat.completion", + Provider: "openai", + Model: "gpt-4o", + Status: "success", + CreatedAt: now, + }) + require.NoError(t, err, "failed to insert log %d", i) + } + + // Page 1: limit < total — TotalCount must equal total, not just the page size. + result, err := store.SearchLogs(ctx, SearchFilters{}, PaginationOptions{Limit: pageLimit}) + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, int64(total), result.Pagination.TotalCount, + "TotalCount should equal total rows (%d), not page size (%d)", total, pageLimit) + assert.Len(t, result.Logs, pageLimit, "result page should have %d rows", pageLimit) + + // Page 2: with offset — TotalCount must still equal total. + result2, err := store.SearchLogs(ctx, SearchFilters{}, PaginationOptions{Limit: pageLimit, Offset: pageLimit}) + require.NoError(t, err) + require.NotNil(t, result2) + assert.Equal(t, int64(total), result2.Pagination.TotalCount, + "TotalCount should be stable across pages") + + // No results: TotalCount should be 0. + result3, err := store.SearchLogs(ctx, SearchFilters{Models: []string{"nonexistent-model-xyz"}}, PaginationOptions{Limit: 100}) + require.NoError(t, err) + require.NotNil(t, result3) + assert.Equal(t, int64(0), result3.Pagination.TotalCount, + "TotalCount should be 0 when no rows match") +} + +// TestSearchLogs_MetadataFilter_StringMatching_SQLite exercises the metadata string matching fix on SQLite. +func TestSearchLogs_MetadataFilter_StringMatching_SQLite(t *testing.T) { + store := newTestSQLiteStore(t) + defer store.Close(context.Background()) + runMetadataStringMatchingSuite(t, store) +} + +// TestSearchLogs_MetadataFilter_StringMatching_Postgres exercises the metadata string matching fix on Postgres, +// where the old code generated JSONB number/boolean fragments that never matched stored string values. +func TestSearchLogs_MetadataFilter_StringMatching_Postgres(t *testing.T) { + store, _ := setupPerfTestDB(t) + runMetadataStringMatchingSuite(t, store) + runPostgresMetadataBooleanStringMatchingSuite(t, store) +} + +// TestSearchLogs_PaginationTotalCount_SQLite verifies pagination.TotalCount on SQLite. +func TestSearchLogs_PaginationTotalCount_SQLite(t *testing.T) { + store := newTestSQLiteStore(t) + defer store.Close(context.Background()) + runPaginationTotalCountSuite(t, store) +} + +// TestSearchLogs_PaginationTotalCount_Postgres verifies pagination.TotalCount on Postgres. +func TestSearchLogs_PaginationTotalCount_Postgres(t *testing.T) { + store, _ := setupPerfTestDB(t) + runPaginationTotalCountSuite(t, store) +} diff --git a/framework/logstore/payload.go b/framework/logstore/payload.go index 32d1dd1dea..853c67bf58 100644 --- a/framework/logstore/payload.go +++ b/framework/logstore/payload.go @@ -55,7 +55,7 @@ var payloadFields = []string{ // ExtractPayload reads the serialized TEXT payload fields from a Log into a map. // The map keys are the DB column names. func ExtractPayload(l *Log) map[string]string { - m := make(map[string]string, len(payloadFields)) + m := make(map[string]string, len(payloadFields)+1) m["input_history"] = l.InputHistory m["responses_input_history"] = l.ResponsesInputHistory m["output_message"] = l.OutputMessage @@ -90,10 +90,15 @@ func ExtractPayload(l *Log) map[string]string { m["passthrough_request_body"] = l.PassthroughRequestBody m["passthrough_response_body"] = l.PassthroughResponseBody m["routing_engine_logs"] = l.RoutingEngineLogs - // Metadata is deliberately NOT included in the snapshot (nor in - // payloadFields): it must always stay DB-resident (filters, rankings, - // log-list display). The DB row is authoritative, so metadata is neither - // written to nor restored from the object store. + // Metadata is written to the snapshot so consumers reading objects + // directly see custom attributes, but it is deliberately NOT part of + // payloadFields: it must always stay DB-resident as well (filters, + // rankings, log-list display), so ClearPayload never strips it from the + // row. NOTE: the snapshot carries the metadata value as of upload time; + // subsequent DB updates are NOT reflected in the object store. + if l.Metadata != nil && *l.Metadata != "" { + m["metadata"] = *l.Metadata + } return m } @@ -281,8 +286,9 @@ func MergePayloadFromJSON(l *Log, data []byte) error { if v, ok := m["routing_engine_logs"]; ok && v != "" { l.RoutingEngineLogs = v } - // Metadata is intentionally NOT restored from the snapshot: it is never - // written there (see ExtractPayload), and the DB row is authoritative. + // Metadata is intentionally NOT restored from the snapshot: the copy + // written there (see ExtractPayload) is for external object consumers + // only, and the DB row stays authoritative. return l.DeserializeFields() } diff --git a/framework/logstore/payload_test.go b/framework/logstore/payload_test.go index d0dd56ab39..b73e6dd71a 100644 --- a/framework/logstore/payload_test.go +++ b/framework/logstore/payload_test.go @@ -50,11 +50,11 @@ func TestExtractPayload_RoundTrip(t *testing.T) { } payload := ExtractPayload(log) - assert.Equal(t, len(payloadFields), len(payload), "payload map should have all payload fields") + assert.Equal(t, len(payloadFields)+1, len(payload), "payload map should have all payload fields plus metadata") assert.Equal(t, `[{"role":"user","content":"hello"}]`, payload["input_history"]) assert.Equal(t, `{"role":"assistant","content":"world"}`, payload["output_message"]) assert.Equal(t, `routing log`, payload["routing_engine_logs"]) - assert.NotContains(t, payload, "metadata", "metadata is DB-resident and must not be written to the snapshot") + assert.Equal(t, metadata, payload["metadata"], "metadata must be written to the snapshot for object consumers") // Clear and verify. ClearPayload(log) @@ -86,6 +86,15 @@ func TestExtractPayload_RoundTrip(t *testing.T) { assert.Equal(t, "user-456", log.MetadataParsed["cortex-user-id"]) } +func TestExtractPayload_NilOrEmptyMetadataOmittedFromSnapshot(t *testing.T) { + payload := ExtractPayload(&Log{ID: "x"}) + assert.NotContains(t, payload, "metadata", "nil Metadata must not appear in snapshot") + + empty := "" + payload = ExtractPayload(&Log{ID: "y", Metadata: &empty}) + assert.NotContains(t, payload, "metadata", "empty Metadata must not appear in snapshot") +} + func TestClearPayload_DoesNotTouchIndexFields(t *testing.T) { log := &Log{ ID: "test-1", diff --git a/framework/logstore/rdb.go b/framework/logstore/rdb.go index ba5b3074ab..f180e9c041 100644 --- a/framework/logstore/rdb.go +++ b/framework/logstore/rdb.go @@ -327,17 +327,10 @@ func (s *RDBLogStore) applyFilters(baseQuery *gorm.DB, filters SearchFilters) *g } switch dialect { case "postgres": - // Use @> containment operator to leverage GIN index on metadata::jsonb - // Preserve value type (number/boolean) for JSON containment - var jsonFragment string - if value == "true" || value == "false" { - jsonFragment = fmt.Sprintf(`{%q: %s}`, key, value) - } else if f, err := strconv.ParseFloat(value, 64); err == nil && !math.IsNaN(f) && !math.IsInf(f, 0) { - // Reject NaN/Inf which would produce invalid JSON; normalize the number - jsonFragment = fmt.Sprintf(`{%q: %s}`, key, strconv.FormatFloat(f, 'f', -1, 64)) - } else { - jsonFragment = fmt.Sprintf(`{%q: %q}`, key, value) - } + // Use @> containment operator to leverage GIN index on metadata::jsonb. + // Metadata values always originate from HTTP headers and are stored as JSON + // strings — always match as a string to avoid type mismatch with jsonb. + jsonFragment := fmt.Sprintf(`{%q: %q}`, key, value) baseQuery = baseQuery.Where("metadata::jsonb @> ?::jsonb", jsonFragment) default: // SQLite: quote the member name so dots/hyphens stay part of the key @@ -691,6 +684,7 @@ func (s *RDBLogStore) SearchLogs(ctx context.Context, filters SearchFilters, pag } } + pagination.TotalCount = totalCount return &SearchResult{ Logs: logs, Pagination: pagination, diff --git a/framework/modelcatalog/config.go b/framework/modelcatalog/config.go index 80563eb1e0..7a51c513d9 100644 --- a/framework/modelcatalog/config.go +++ b/framework/modelcatalog/config.go @@ -2,25 +2,19 @@ package modelcatalog import ( "time" + + "github.com/maximhq/bifrost/core/schemas" + "github.com/maximhq/bifrost/framework/modelcatalog/datasheet" + "github.com/maximhq/bifrost/framework/modelcatalog/keyconfig" ) const ( - DefaultSyncInterval = 24 * time.Hour + DefaultSyncInterval = datasheet.DefaultSyncInterval MinimumPricingSyncIntervalSec = int64(3600) - // syncWorkerTickerPeriod is the fixed interval at which the background sync worker - // wakes up to check whether a sync is due. This is independent of pricingSyncInterval — - // the ticker defines the check granularity, not the sync frequency. - // Kept well below MinimumPricingSyncIntervalSec so the threshold check is not - // defeated by ticker drift when pricingSyncInterval is set near the minimum. - syncWorkerTickerPeriod = 5 * time.Minute - - ConfigLastPricingSyncKey = "LastModelPricingSync" - ConfigLastParamsSyncKey = "LastModelParametersSync" - DefaultPricingURL = "https://getbifrost.ai/datasheet" - DefaultModelParametersURL = "https://getbifrost.ai/datasheet/model-parameters" - DefaultPricingTimeout = 45 * time.Second - DefaultModelParametersTimeout = 45 * time.Second + ConfigLastPricingSyncKey = "LastModelPricingSync" + ConfigLastParamsSyncKey = "LastModelParametersSync" + ConfigLastMCPLibrarySyncKey = "LastMCPLibrarySync" ) // Config is the model pricing configuration. @@ -28,4 +22,63 @@ type Config struct { PricingURL *string `json:"pricing_url,omitempty"` PricingSyncInterval *int64 `json:"pricing_sync_interval,omitempty"` // seconds ModelParametersURL *string `json:"model_parameters_url,omitempty"` + + // MCPLibraryURL overrides the endpoint the MCP server library catalog is + // synced from. Empty/nil uses DefaultMCPLibraryURL. Mirrors PricingURL: the + // default ships out of the box and the user can point it at a custom source. + MCPLibraryURL *string `json:"mcp_library_url,omitempty"` + MCPLibrarySyncInterval *int64 `json:"mcp_library_sync_interval,omitempty"` // seconds +} + +// Type re-exports so external callers can continue importing the legacy +// names (PricingEntry, PricingOptions, etc.) without changing imports. +// Internally these live in the datasheet / keyconfig subpackages. +type ( + PricingEntry = datasheet.Entry + PricingOptions = datasheet.Options + PricingOverride = datasheet.Override + PricingLookupScopes = datasheet.LookupScopes + ScopeKind = datasheet.ScopeKind + MatchType = datasheet.MatchType + + KeyConfigEntry = keyconfig.KeyEntry + AliasOwner = keyconfig.AliasOwner +) + +// Scope kind constants re-exported for callers that compare by value. +const ( + ScopeKindGlobal = datasheet.ScopeKindGlobal + ScopeKindProvider = datasheet.ScopeKindProvider + ScopeKindProviderKey = datasheet.ScopeKindProviderKey + ScopeKindVirtualKey = datasheet.ScopeKindVirtualKey + ScopeKindVirtualKeyProvider = datasheet.ScopeKindVirtualKeyProvider + ScopeKindVirtualKeyProviderKey = datasheet.ScopeKindVirtualKeyProviderKey + + MatchTypeExact = datasheet.MatchTypeExact + MatchTypeWildcard = datasheet.MatchTypeWildcard +) + +// PricingLookupScopesFromContext is re-exported so callers don't have to +// change their imports. +func PricingLookupScopesFromContext(ctx *schemas.BifrostContext, provider string) *PricingLookupScopes { + return datasheet.LookupScopesFromContext(ctx, provider) } + +// Sync timing defaults re-exported from datasheet for consumers of the +// historical constants. +const ( + DefaultPricingURL = datasheet.DefaultURL + DefaultModelParametersURL = datasheet.DefaultModelParametersURL + DefaultPricingTimeout = datasheet.DefaultPricingTimeout + DefaultModelParametersTimeout = datasheet.DefaultModelParametersTimeout + + DefaultMCPLibraryURL = "https://getbifrost.ai/mcp-library" + DefaultMCPLibraryTimeout = 45 * time.Second +) + +// syncWorkerTickerPeriod is the fixed interval at which the background sync worker +// wakes up to check whether a sync is due. This is independent of pricingSyncInterval — +// the ticker defines the check granularity, not the sync frequency. +// Kept well below MinimumPricingSyncIntervalSec so the threshold check is not +// defeated by ticker drift when pricingSyncInterval is set near the minimum. +const syncWorkerTickerPeriod = 5 * time.Minute diff --git a/framework/modelcatalog/capabilities_test.go b/framework/modelcatalog/datasheet/capabilities_test.go similarity index 83% rename from framework/modelcatalog/capabilities_test.go rename to framework/modelcatalog/datasheet/capabilities_test.go index 3d188f775d..7ec61f28d4 100644 --- a/framework/modelcatalog/capabilities_test.go +++ b/framework/modelcatalog/datasheet/capabilities_test.go @@ -1,4 +1,4 @@ -package modelcatalog +package datasheet import ( "testing" @@ -7,13 +7,13 @@ import ( configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" ) -func TestGetModelCapabilityEntryForModel_PrefersChatThenResponsesThenCompletion(t *testing.T) { +func TestGetCapabilityEntry_PrefersChatThenResponsesThenCompletion(t *testing.T) { contextLengthChat := 128000 maxInputTokensChat := 64000 maxOutputTokensChat := 16000 modality := "text" - mc := &ModelCatalog{ + s := &Store{ pricingData: map[string]configstoreTables.TableModelPricing{ makeKey("gpt-4o", "openai", "responses"): { Model: "gpt-4o", @@ -37,7 +37,7 @@ func TestGetModelCapabilityEntryForModel_PrefersChatThenResponsesThenCompletion( }, } - entry := mc.GetModelCapabilityEntryForModel("gpt-4o", schemas.OpenAI) + entry := s.GetCapabilityEntry("gpt-4o", schemas.OpenAI) if entry == nil { t.Fatal("expected capability entry") } @@ -58,8 +58,8 @@ func TestGetModelCapabilityEntryForModel_PrefersChatThenResponsesThenCompletion( } } -func TestGetModelCapabilityEntryForModel_FallsBackToAnyModeDeterministically(t *testing.T) { - mc := &ModelCatalog{ +func TestGetCapabilityEntry_FallsBackToAnyModeDeterministically(t *testing.T) { + s := &Store{ pricingData: map[string]configstoreTables.TableModelPricing{ makeKey("imagen", "vertex", "image_generation"): { Model: "imagen", @@ -71,7 +71,7 @@ func TestGetModelCapabilityEntryForModel_FallsBackToAnyModeDeterministically(t * }, } - entry := mc.GetModelCapabilityEntryForModel("imagen", schemas.Vertex) + entry := s.GetCapabilityEntry("imagen", schemas.Vertex) if entry == nil { t.Fatal("expected capability entry") } @@ -80,10 +80,10 @@ func TestGetModelCapabilityEntryForModel_FallsBackToAnyModeDeterministically(t * } } -func TestGetModelCapabilityEntryForModel_ResolvesAliasFamilyViaBaseModel(t *testing.T) { +func TestGetCapabilityEntry_ResolvesAliasFamilyViaBaseModel(t *testing.T) { contextLengthChat := 128000 - mc := &ModelCatalog{ + s := &Store{ pricingData: map[string]configstoreTables.TableModelPricing{ makeKey("gpt-4o-2024-08-06", "openai", "responses"): { Model: "gpt-4o-2024-08-06", @@ -107,7 +107,7 @@ func TestGetModelCapabilityEntryForModel_ResolvesAliasFamilyViaBaseModel(t *test }, } - entry := mc.GetModelCapabilityEntryForModel("gpt-4o", schemas.OpenAI) + entry := s.GetCapabilityEntry("gpt-4o", schemas.OpenAI) if entry == nil { t.Fatal("expected capability entry for base-model alias") } @@ -119,8 +119,8 @@ func TestGetModelCapabilityEntryForModel_ResolvesAliasFamilyViaBaseModel(t *test } } -func TestGetModelCapabilityEntryForModel_ResolvesProviderPrefixedAlias(t *testing.T) { - mc := &ModelCatalog{ +func TestGetCapabilityEntry_ResolvesProviderPrefixedAlias(t *testing.T) { + s := &Store{ pricingData: map[string]configstoreTables.TableModelPricing{ makeKey("gpt-4o-2024-08-06", "openai", "chat"): { Model: "gpt-4o-2024-08-06", @@ -136,7 +136,7 @@ func TestGetModelCapabilityEntryForModel_ResolvesProviderPrefixedAlias(t *testin }, } - entry := mc.GetModelCapabilityEntryForModel("openai/gpt-4o", schemas.OpenAI) + entry := s.GetCapabilityEntry("openai/gpt-4o", schemas.OpenAI) if entry == nil { t.Fatal("expected capability entry for provider-prefixed alias") } @@ -145,11 +145,11 @@ func TestGetModelCapabilityEntryForModel_ResolvesProviderPrefixedAlias(t *testin } } -func TestGetModelCapabilityEntryForModel_PrefersLiteralMatchOverAliasFamily(t *testing.T) { +func TestGetCapabilityEntry_PrefersLiteralMatchOverAliasFamily(t *testing.T) { literalContextLength := 32000 aliasContextLength := 128000 - mc := &ModelCatalog{ + s := &Store{ pricingData: map[string]configstoreTables.TableModelPricing{ makeKey("gpt-4o", "openai", "chat"): { Model: "gpt-4o", @@ -174,7 +174,7 @@ func TestGetModelCapabilityEntryForModel_PrefersLiteralMatchOverAliasFamily(t *t }, } - entry := mc.GetModelCapabilityEntryForModel("gpt-4o", schemas.OpenAI) + entry := s.GetCapabilityEntry("gpt-4o", schemas.OpenAI) if entry == nil { t.Fatal("expected literal capability entry") } @@ -187,24 +187,24 @@ func TestCapabilityFieldsRoundTripThroughPricingConversions(t *testing.T) { modality := "text" inputCost := float64(1) outputCost := float64(2) - entry := PricingEntry{ + entry := Entry{ BaseModel: "gpt-4o", Provider: "openai", Mode: "chat", - PricingOptions: PricingOptions{ + Options: Options{ InputCostPerToken: &inputCost, OutputCostPerToken: &outputCost, }, - ContextLength: capabilityIntPtr(128000), - MaxInputTokens: capabilityIntPtr(64000), - MaxOutputTokens: capabilityIntPtr(16000), + ContextLength: capabilityIntPtr(128000), + MaxInputTokens: capabilityIntPtr(64000), + MaxOutputTokens: capabilityIntPtr(16000), Architecture: &schemas.Architecture{ Modality: &modality, }, } - table := convertPricingDataToTableModelPricing("gpt-4o", entry) - roundTrip := convertTableModelPricingToPricingData(&table) + table := convertEntryToTablePricing("gpt-4o", entry) + roundTrip := convertTablePricingToEntry(&table) if roundTrip.ContextLength == nil || *roundTrip.ContextLength != 128000 { t.Fatalf("expected context_length to round-trip, got %#v", roundTrip.ContextLength) diff --git a/framework/modelcatalog/datasheet/cost.go b/framework/modelcatalog/datasheet/cost.go new file mode 100644 index 0000000000..f3e65c692c --- /dev/null +++ b/framework/modelcatalog/datasheet/cost.go @@ -0,0 +1,1337 @@ +package datasheet + +import ( + "context" + "fmt" + "strconv" + "strings" + + "github.com/maximhq/bifrost/core/schemas" + configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" +) + +// CalculateCost calculates the cost of a Bifrost response. +// It handles all request types, cache debug billing, and tiered pricing. +// If scopes is nil, an empty LookupScopes is used; global and provider-scoped +// overrides may still apply since the provider is derived from the response. +func (s *Store) CalculateCost(result *schemas.BifrostResponse, scopes *LookupScopes) float64 { + if result == nil { + return 0 + } + + var lookupScopes LookupScopes + if scopes != nil { + lookupScopes = *scopes + } + + // Handle semantic cache billing + cacheDebug := result.GetExtraFields().CacheDebug + if cacheDebug != nil { + return s.calculateCostWithCache(result, cacheDebug, lookupScopes) + } + + return s.calculateBaseCost(result, lookupScopes) +} + +// calculateCostWithCache handles cost calculation when semantic cache debug info is present. +func (s *Store) calculateCostWithCache(result *schemas.BifrostResponse, cacheDebug *schemas.BifrostCacheDebug, scopes LookupScopes) float64 { + if cacheDebug.CacheHit { + // Direct cache hit — no LLM call, no cost + if cacheDebug.HitType != nil && *cacheDebug.HitType == "direct" { + return 0 + } + // Semantic cache hit — only the embedding lookup cost + if cacheDebug.ProviderUsed != nil && cacheDebug.ModelUsed != nil && cacheDebug.InputTokens != nil { + return s.computeCacheEmbeddingCost(cacheDebug, scopes) + } + return 0 + } + + // Cache miss — full LLM cost + embedding lookup cost + baseCost := s.calculateBaseCost(result, scopes) + embeddingCost := s.computeCacheEmbeddingCost(cacheDebug, scopes) + return baseCost + embeddingCost +} + +// computeCacheEmbeddingCost calculates the embedding cost for a semantic cache lookup. +func (s *Store) computeCacheEmbeddingCost(cacheDebug *schemas.BifrostCacheDebug, scopes LookupScopes) float64 { + if cacheDebug == nil || cacheDebug.ProviderUsed == nil || cacheDebug.ModelUsed == nil || cacheDebug.InputTokens == nil { + return 0 + } + if scopes.Provider == "" { + scopes.Provider = *cacheDebug.ProviderUsed + } + // Cache-debug pricing has only a single model identifier (whatever the + // cache recorded). Maps to RoutingInfo.Model — no alias resolution + // context exists for the cache-replayed request. + pricing := s.resolvePricing(schemas.RoutingInfo{ + Provider: schemas.ModelProvider(*cacheDebug.ProviderUsed), + Model: *cacheDebug.ModelUsed, + }, schemas.EmbeddingRequest, scopes) + if pricing == nil { + return 0 + } + return float64(*cacheDebug.InputTokens) * tieredInputRate(pricing, *cacheDebug.InputTokens, serviceTier{}) +} + +// computeContainerCreationCost returns the cost for creating a container from an already-resolved pricing entry. +func computeContainerCreationCost(pricing *configstoreTables.TableModelPricing) float64 { + if pricing == nil || pricing.CodeInterpreterCostPerSession == nil { + return 0 + } + return *pricing.CodeInterpreterCostPerSession +} + +// calculateBaseCost extracts usage from the response and routes to the appropriate compute function. +func (s *Store) calculateBaseCost(result *schemas.BifrostResponse, scopes LookupScopes) float64 { + extraFields := result.GetExtraFields() + if extraFields == nil { + return 0 + } + + // Read routing info populated by core.bifrost at request time. + // + // Backward-compat fallback: when the caller (e.g. LoggerPlugin's + // RecalculateCosts replaying logs written before RoutingInfo existed, + // or third-party plugins still on the legacy ExtraFields shape) leaves + // RoutingInfo empty, synthesise one from the deprecated triplet so + // pricing keeps working. Triggered only when RoutingInfo is fully + // unset — partial population is trusted as-is. + routingInfo := extraFields.RoutingInfo + if routingInfo.Provider == "" && routingInfo.Model == "" && routingInfo.ResolvedKeyAlias == nil { + routingInfo.Provider = extraFields.Provider + routingInfo.Model = extraFields.OriginalModelRequested + if r := extraFields.ResolvedModelUsed; r != "" && r != extraFields.OriginalModelRequested { + routingInfo.ResolvedKeyAlias = &schemas.ResolvedKeyAlias{ModelID: r} + } + } + requestType := extraFields.RequestType + + // Extract usage data from the response (passthrough and native paths unified) + input := extractCostInput(result) + + // If provider already computed cost, use it + if input.usage != nil && input.usage.Cost != nil && input.usage.Cost.TotalCost > 0 { + return input.usage.Cost.TotalCost + } + + // If no usage data at all, nothing to price + if input.usage == nil && input.audioSeconds == nil && input.audioTokenDetails == nil && input.imageUsage == nil && input.videoSeconds == nil && input.audioTextInputChars == 0 && input.ocrProcessedPages == nil && input.containerIdentifierString == "" { + return 0 + } + + if result.PassthroughResponse != nil { + // Infer request type from usage fields + path; passthrough bypasses stream normalization. + requestType = inferPassthroughRequestType(routingInfo.Provider, extraFields.PassthroughPath, result.PassthroughResponse.PassthroughUsage) + } else { + // Normalize stream request types to their base type for pricing lookup + requestType = normalizeStreamRequestType(requestType) + } + + // When a pricing model override is set (e.g. container creates always look + // up "container"), it replaces the lookup hierarchy entirely. Build a + // synthetic RoutingInfo that reuses Provider but pins the model fields to + // the container identifier — the lookup tries it as ModelName, the + // override key is the container identifier so per-container overrides + // stay addressable. + if input.containerIdentifierString != "" { + routingInfo = schemas.RoutingInfo{ + Provider: routingInfo.Provider, + Model: input.containerIdentifierString, + } + } + + pricing := s.resolvePricing(routingInfo, requestType, scopes) + if pricing == nil { + return 0 + } + + // Route to the appropriate compute function + switch requestType { + case schemas.ChatCompletionRequest, schemas.TextCompletionRequest, schemas.ResponsesRequest, schemas.RealtimeRequest, schemas.CompactionRequest: + return computeTextCost(pricing, input.usage, input.tier) + case schemas.EmbeddingRequest: + return computeEmbeddingCost(pricing, input.usage, input.tier) + case schemas.RerankRequest: + return computeRerankCost(pricing, input.usage, input.tier) + case schemas.SpeechRequest: + return computeSpeechCost(pricing, input.usage, input.audioSeconds, input.audioTextInputChars, input.tier) + case schemas.TranscriptionRequest: + return computeTranscriptionCost(pricing, input.usage, input.audioSeconds, input.audioTokenDetails, input.tier) + case schemas.ImageGenerationRequest, schemas.ImageEditRequest, schemas.ImageVariationRequest: + return computeImageCost(pricing, input.imageUsage, input.imageSize, input.imageQuality, input.tier) + case schemas.VideoGenerationRequest, schemas.VideoRemixRequest: + return computeVideoCost(pricing, input.usage, input.videoSeconds, input.tier) + case schemas.OCRRequest: + return computeOCRCost(pricing, input.ocrProcessedPages, input.ocrIsAnnotated) + case schemas.ContainerCreateRequest: + return computeContainerCreationCost(pricing) + default: + return 0 + } +} + +// --------------------------------------------------------------------------- +// Usage extraction +// --------------------------------------------------------------------------- + +func extractCostInput(result *schemas.BifrostResponse) costInput { + var input costInput + + switch { + case result.PassthroughResponse != nil && result.PassthroughResponse.PassthroughUsage != nil: + return passthroughUsageToCostInput(result.PassthroughResponse.PassthroughUsage) + + case result.TextCompletionResponse != nil && result.TextCompletionResponse.Usage != nil: + input.usage = result.TextCompletionResponse.Usage + + case result.ChatResponse != nil && result.ChatResponse.Usage != nil: + input.usage = result.ChatResponse.Usage + input.tier = tierFromResponse(result.ChatResponse.ServiceTier, result.ChatResponse.Speed) + + case result.ResponsesResponse != nil && result.ResponsesResponse.Usage != nil: + input.usage = responsesUsageToBifrostUsage(result.ResponsesResponse.Usage) + input.tier = tierFromResponse(result.ResponsesResponse.ServiceTier, result.ResponsesResponse.Speed) + + case result.CompactionResponse != nil && result.CompactionResponse.Usage != nil: + input.usage = responsesUsageToBifrostUsage(result.CompactionResponse.Usage) + + case result.ResponsesStreamResponse != nil && result.ResponsesStreamResponse.Response != nil && result.ResponsesStreamResponse.Response.Usage != nil: + input.usage = responsesUsageToBifrostUsage(result.ResponsesStreamResponse.Response.Usage) + input.tier = tierFromResponse(result.ResponsesStreamResponse.Response.ServiceTier, result.ResponsesStreamResponse.Response.Speed) + + case result.EmbeddingResponse != nil && result.EmbeddingResponse.Usage != nil: + input.usage = result.EmbeddingResponse.Usage + + case result.RerankResponse != nil && result.RerankResponse.Usage != nil: + input.usage = result.RerankResponse.Usage + + case result.SpeechResponse != nil && result.SpeechResponse.Usage != nil: + input.usage = speechUsageToBifrostUsage(result.SpeechResponse.Usage) + input.audioTextInputChars = result.SpeechResponse.Usage.InputChars + + case result.SpeechStreamResponse != nil && result.SpeechStreamResponse.Usage != nil: + input.usage = speechUsageToBifrostUsage(result.SpeechStreamResponse.Usage) + input.audioTextInputChars = result.SpeechStreamResponse.Usage.InputChars + + case result.TranscriptionResponse != nil && result.TranscriptionResponse.Usage != nil: + input.usage, input.audioSeconds, input.audioTokenDetails = extractTranscriptionUsage(result.TranscriptionResponse.Usage) + + case result.TranscriptionStreamResponse != nil && result.TranscriptionStreamResponse.Usage != nil: + input.usage, input.audioSeconds, input.audioTokenDetails = extractTranscriptionUsage(result.TranscriptionStreamResponse.Usage) + + case result.ImageGenerationResponse != nil: + // Defensive copy: populateOutputImageCount writes into imageUsage, + // and we must not mutate the caller's BifrostResponse during what is + // otherwise a pure read path. + if result.ImageGenerationResponse.Usage != nil { + input.imageUsage = result.ImageGenerationResponse.Usage.DeepCopy() + } else { + // No usage data but response exists — default to empty so per-image pricing can apply + input.imageUsage = &schemas.ImageUsage{} + } + populateOutputImageCount(input.imageUsage, len(result.ImageGenerationResponse.Data)) + if result.ImageGenerationResponse.ImageGenerationResponseParameters != nil { + input.imageSize = result.ImageGenerationResponse.ImageGenerationResponseParameters.Size + input.imageQuality = result.ImageGenerationResponse.ImageGenerationResponseParameters.Quality + } + + case result.ImageGenerationStreamResponse != nil: + // Defensive copy mirrors the non-stream path so CalculateCost never + // aliases the caller's response — keeps the read-only invariant + // uniform and prevents accidental mutation if image-count derivation + // is later added on this branch. + if result.ImageGenerationStreamResponse.Usage != nil { + input.imageUsage = result.ImageGenerationStreamResponse.Usage.DeepCopy() + } else { + input.imageUsage = &schemas.ImageUsage{} + } + input.imageSize = result.ImageGenerationStreamResponse.Size + input.imageQuality = result.ImageGenerationStreamResponse.Quality + + case result.VideoGenerationResponse != nil && result.VideoGenerationResponse.Seconds != nil: + seconds, err := strconv.Atoi(*result.VideoGenerationResponse.Seconds) + if err == nil { + input.videoSeconds = &seconds + } + + case result.OCRResponse != nil: + pages := len(result.OCRResponse.Pages) + if result.OCRResponse.UsageInfo != nil && result.OCRResponse.UsageInfo.PagesProcessed > 0 { + pages = result.OCRResponse.UsageInfo.PagesProcessed + } + input.ocrProcessedPages = &pages + isAnnotated := result.OCRResponse.DocumentAnnotation != nil && *result.OCRResponse.DocumentAnnotation != "" + input.ocrIsAnnotated = &isAnnotated + + case result.ContainerCreateResponse != nil: + if memLimit := result.ContainerCreateResponse.MemoryLimit; memLimit != "" { + input.containerIdentifierString = "container-" + memLimit + } else { + input.containerIdentifierString = "container" + } + } + + return input +} + +func responsesUsageToBifrostUsage(u *schemas.ResponsesResponseUsage) *schemas.BifrostLLMUsage { + usage := &schemas.BifrostLLMUsage{ + PromptTokens: u.InputTokens, + CompletionTokens: u.OutputTokens, + TotalTokens: u.TotalTokens, + Cost: u.Cost, + } + // Map token details for cache and search query pricing + if u.InputTokensDetails != nil { + usage.PromptTokensDetails = &schemas.ChatPromptTokensDetails{ + TextTokens: u.InputTokensDetails.TextTokens, + AudioTokens: u.InputTokensDetails.AudioTokens, + ImageTokens: u.InputTokensDetails.ImageTokens, + CachedReadTokens: u.InputTokensDetails.CachedReadTokens, + CachedWriteTokens: u.InputTokensDetails.CachedWriteTokens, + CachedWriteTokenDetails: u.InputTokensDetails.CachedWriteTokenDetails, + } + } + if u.OutputTokensDetails != nil { + usage.CompletionTokensDetails = &schemas.ChatCompletionTokensDetails{ + ReasoningTokens: u.OutputTokensDetails.ReasoningTokens, + AudioTokens: u.OutputTokensDetails.AudioTokens, + } + if u.OutputTokensDetails.NumSearchQueries != nil { + usage.CompletionTokensDetails.NumSearchQueries = u.OutputTokensDetails.NumSearchQueries + } + } + return usage +} + +func speechUsageToBifrostUsage(u *schemas.SpeechUsage) *schemas.BifrostLLMUsage { + return &schemas.BifrostLLMUsage{ + PromptTokens: u.InputTokens, + CompletionTokens: u.OutputTokens, + TotalTokens: u.TotalTokens, + } +} + +func extractTranscriptionUsage(u *schemas.TranscriptionUsage) (*schemas.BifrostLLMUsage, *int, *schemas.TranscriptionUsageInputTokenDetails) { + usage := &schemas.BifrostLLMUsage{} + if u.InputTokens != nil { + usage.PromptTokens = *u.InputTokens + } + if u.OutputTokens != nil { + usage.CompletionTokens = *u.OutputTokens + } + if u.TotalTokens != nil { + usage.TotalTokens = *u.TotalTokens + } else { + usage.TotalTokens = usage.PromptTokens + usage.CompletionTokens + } + + var audioTokenDetails *schemas.TranscriptionUsageInputTokenDetails + if u.InputTokenDetails != nil { + audioTokenDetails = &schemas.TranscriptionUsageInputTokenDetails{ + AudioTokens: u.InputTokenDetails.AudioTokens, + TextTokens: u.InputTokenDetails.TextTokens, + } + } + + return usage, u.Seconds, audioTokenDetails +} + +// --------------------------------------------------------------------------- +// Per-request-type cost computation +// --------------------------------------------------------------------------- + +// computeTextCost handles chat, text completion, and responses requests. +func computeTextCost(pricing *configstoreTables.TableModelPricing, usage *schemas.BifrostLLMUsage, tier serviceTier) float64 { + if usage == nil { + return 0 + } + + totalTokens := usage.TotalTokens + promptTokens := usage.PromptTokens + completionTokens := usage.CompletionTokens + + // Extract cached token counts + cachedReadTokens := 0 + cachedWriteTokens := 0 + cachedWriteTokensAbove1hr := 0 + if usage.PromptTokensDetails != nil { + cachedReadTokens = usage.PromptTokensDetails.CachedReadTokens + cachedWriteTokens = usage.PromptTokensDetails.CachedWriteTokens + if usage.PromptTokensDetails.CachedWriteTokenDetails != nil { + cachedWriteTokensAbove1hr = usage.PromptTokensDetails.CachedWriteTokenDetails.CachedWriteTokens1h + } + } + + inputRate := tieredInputRate(pricing, totalTokens, tier) + outputRate := tieredOutputRate(pricing, totalTokens, tier) + cacheReadInputRate := tieredCacheReadInputTokenRate(pricing, totalTokens, tier) + cacheCreationInputRate := tieredCacheCreationInputTokenRate(pricing, totalTokens, tier) + cacheCreationInputAbove1hrInputRate := tieredCacheCreationInputAbove1hrTokenRate(pricing, totalTokens, tier) + + // Clamp cached token counts to avoid negative billing on malformed provider payloads + if cachedReadTokens > promptTokens { + cachedReadTokens = promptTokens + } + if cachedWriteTokens > promptTokens-cachedReadTokens { + cachedWriteTokens = promptTokens - cachedReadTokens + } + // Should not happen, but just in case + if cachedWriteTokensAbove1hr > cachedWriteTokens { + cachedWriteTokensAbove1hr = cachedWriteTokens + } + + // Input cost: non-cached tokens at regular rate + nonCachedPrompt := promptTokens - cachedReadTokens - cachedWriteTokens + inputCost := float64(nonCachedPrompt) * inputRate + + // Add cached prompt tokens at cache read rate + if cachedReadTokens > 0 { + inputCost += float64(cachedReadTokens) * cacheReadInputRate + } + + // Add cached write tokens at cache creation rate + if cachedWriteTokens > 0 { + if cachedWriteTokensAbove1hr > 0 { + inputCost += float64(cachedWriteTokensAbove1hr) * cacheCreationInputAbove1hrInputRate + } + inputCost += float64(cachedWriteTokens-cachedWriteTokensAbove1hr) * cacheCreationInputRate + } + + outputCost := float64(completionTokens) * outputRate + + // Audio token cost: when token details include audio tokens, price them + // at the dedicated audio rate and subtract from the text token costs above. + // Realtime and audio-enabled chat models report audio tokens in details. + audioCost := 0.0 + inputAudioTokens := 0 + outputAudioTokens := 0 + if usage.PromptTokensDetails != nil { + inputAudioTokens = usage.PromptTokensDetails.AudioTokens + } + if usage.CompletionTokensDetails != nil { + outputAudioTokens = usage.CompletionTokensDetails.AudioTokens + } + if inputAudioTokens < 0 { + inputAudioTokens = 0 + } else if inputAudioTokens > promptTokens { + inputAudioTokens = promptTokens + } + if outputAudioTokens < 0 { + outputAudioTokens = 0 + } else if outputAudioTokens > completionTokens { + outputAudioTokens = completionTokens + } + if inputAudioTokens > 0 && pricing.InputCostPerAudioToken != nil { + // Subtract audio tokens charged at text rate, add at audio rate. + audioCost += float64(inputAudioTokens) * (*pricing.InputCostPerAudioToken - inputRate) + } + if outputAudioTokens > 0 && pricing.OutputCostPerAudioToken != nil { + audioCost += float64(outputAudioTokens) * (*pricing.OutputCostPerAudioToken - outputRate) + } + + // Search query cost + searchCost := 0.0 + if pricing.SearchContextCostPerQuery != nil && usage.CompletionTokensDetails != nil && usage.CompletionTokensDetails.NumSearchQueries != nil { + searchCost = float64(*usage.CompletionTokensDetails.NumSearchQueries) * *pricing.SearchContextCostPerQuery + } + + return inputCost + outputCost + audioCost + searchCost +} + +// computeEmbeddingCost handles embedding requests (input-only). +func computeEmbeddingCost(pricing *configstoreTables.TableModelPricing, usage *schemas.BifrostLLMUsage, tier serviceTier) float64 { + if usage == nil { + return 0 + } + return float64(usage.PromptTokens) * tieredInputRate(pricing, usage.TotalTokens, tier) +} + +// computeRerankCost handles rerank requests. +func computeRerankCost(pricing *configstoreTables.TableModelPricing, usage *schemas.BifrostLLMUsage, tier serviceTier) float64 { + if usage == nil { + return 0 + } + inputCost := float64(usage.PromptTokens) * tieredInputRate(pricing, usage.TotalTokens, tier) + outputCost := float64(usage.CompletionTokens) * tieredOutputRate(pricing, usage.TotalTokens, tier) + + searchCost := 0.0 + if pricing.SearchContextCostPerQuery != nil && usage.CompletionTokensDetails != nil && usage.CompletionTokensDetails.NumSearchQueries != nil { + searchCost = float64(*usage.CompletionTokensDetails.NumSearchQueries) * *pricing.SearchContextCostPerQuery + } + + return inputCost + outputCost + searchCost +} + +// computeSpeechCost handles speech (TTS) requests. +// Input is text (PromptTokens), output is audio (CompletionTokens). +// +// Per-character pricing (InputCostPerCharacter) is used as first-class support for TTS/audio +// models — providers such as OpenAI TTS, ElevenLabs, and AWS Polly bill per character of +// input text rather than per token. PromptTokens from usage is treated as the character count +// since TTS providers report their billable unit in that field. +// Output falls back to per-second duration when no audio token rate is configured. +func computeSpeechCost(pricing *configstoreTables.TableModelPricing, usage *schemas.BifrostLLMUsage, audioSeconds *int, audioTextInputChars int, tier serviceTier) float64 { + totalTokens := safeTotalTokens(usage) + + // Input: per-character rate takes precedence for TTS/audio models + inputCost := 0.0 + if audioTextInputChars > 0 { + if pricing.InputCostPerCharacter != nil { + inputCost = float64(audioTextInputChars) * *pricing.InputCostPerCharacter + } else { + inputCost = float64(audioTextInputChars) * tieredInputRate(pricing, totalTokens, tier) + } + } else if usage != nil && usage.PromptTokens > 0 { + inputCost = float64(usage.PromptTokens) * tieredInputRate(pricing, totalTokens, tier) + } + + // Output: audio tokens first, then per-second fallback + outputCost := computeAudioOutputCost(pricing, usage, audioSeconds, totalTokens, tier) + + return inputCost + outputCost +} + +// computeTranscriptionCost handles transcription (STT) requests. +// Input is audio, output is text (CompletionTokens). +// Input and output are calculated independently — tokens first, then per-second fallback. +func computeTranscriptionCost(pricing *configstoreTables.TableModelPricing, usage *schemas.BifrostLLMUsage, audioSeconds *int, audioTokenDetails *schemas.TranscriptionUsageInputTokenDetails, tier serviceTier) float64 { + totalTokens := safeTotalTokens(usage) + + // Input: audio tokens/details first, then per-second fallback + inputCost := computeAudioInputCost(pricing, usage, audioSeconds, audioTokenDetails, totalTokens, tier) + + // Output: text tokens + outputCost := 0.0 + if usage != nil && usage.CompletionTokens > 0 { + outputCost = float64(usage.CompletionTokens) * tieredOutputRate(pricing, totalTokens, tier) + } + + return inputCost + outputCost +} + +// computeAudioInputCost calculates input cost for audio: audio token details first, +// then generic input tokens, then per-second duration fallback. +func computeAudioInputCost(pricing *configstoreTables.TableModelPricing, usage *schemas.BifrostLLMUsage, audioSeconds *int, audioTokenDetails *schemas.TranscriptionUsageInputTokenDetails, totalTokens int, tier serviceTier) float64 { + // Audio token detail pricing (audio + text token breakdown) + if audioTokenDetails != nil && (audioTokenDetails.AudioTokens > 0 || audioTokenDetails.TextTokens > 0) { + return float64(audioTokenDetails.AudioTokens)*tieredAudioTokenInputRate(pricing, totalTokens, tier) + + float64(audioTokenDetails.TextTokens)*tieredInputRate(pricing, totalTokens, tier) + } + + // Generic input tokens + if usage != nil && usage.PromptTokens > 0 { + return float64(usage.PromptTokens) * tieredInputRate(pricing, totalTokens, tier) + } + + // Per-second duration fallback + if audioSeconds != nil && *audioSeconds > 0 { + if rate := tieredAudioInputPerSecondRate(pricing, totalTokens); rate > 0 { + return float64(*audioSeconds) * rate + } + } + + return 0 +} + +// computeAudioOutputCost calculates output cost for audio: audio tokens first, +// then generic output tokens, then per-second duration fallback. +func computeAudioOutputCost(pricing *configstoreTables.TableModelPricing, usage *schemas.BifrostLLMUsage, audioSeconds *int, totalTokens int, tier serviceTier) float64 { + // Audio-specific output tokens + if usage != nil && usage.CompletionTokens > 0 { + return float64(usage.CompletionTokens) * tieredAudioTokenOutputRate(pricing, totalTokens, tier) + } + + // Per-second duration fallback + if audioSeconds != nil && *audioSeconds > 0 { + if pricing.OutputCostPerSecond != nil { + return float64(*audioSeconds) * *pricing.OutputCostPerSecond + } + } + + return 0 +} + +// computeImageCost handles image generation requests. +// Input and output are calculated independently — each tries token-based pricing first, +// then per-pixel pricing, falling back to per-image count pricing. +// imageQuality must be one of "low", "medium", "high", "auto" to use quality-specific rates; other values use base rates. +func computeImageCost(pricing *configstoreTables.TableModelPricing, imageUsage *schemas.ImageUsage, imageSize string, imageQuality string, tier serviceTier) float64 { + if imageUsage == nil { + return 0 + } + + totalTokens := imageUsage.TotalTokens + pixels := parseImagePixels(imageSize) + inputCost := computeImageInputCost(pricing, imageUsage, totalTokens, pixels, tier) + outputCost := computeImageOutputCost(pricing, imageUsage, totalTokens, pixels, imageQuality, tier) + + return inputCost + outputCost +} + +// computeImageInputCost calculates input cost: tokens first, then per-pixel, then per-image count fallback. +func computeImageInputCost(pricing *configstoreTables.TableModelPricing, imageUsage *schemas.ImageUsage, totalTokens int, pixels int, tier serviceTier) float64 { + // Try token-based pricing first + var inputTextTokens, inputImageTokens int + if imageUsage.InputTokensDetails != nil { + inputImageTokens = imageUsage.InputTokensDetails.ImageTokens + inputTextTokens = imageUsage.InputTokensDetails.TextTokens + } else { + inputTextTokens = imageUsage.InputTokens + } + + if inputTextTokens > 0 || inputImageTokens > 0 { + return float64(inputTextTokens)*tieredInputRate(pricing, totalTokens, tier) + + float64(inputImageTokens)*tieredImageInputRate(pricing, totalTokens, tier) + } + + // Per-pixel pricing fallback + if pricing.InputCostPerPixel != nil && pixels > 0 && imageUsage.NumInputImages > 0 { + return float64(pixels*imageUsage.NumInputImages) * *pricing.InputCostPerPixel + } + + // Fall back to per-image count pricing + if pricing.InputCostPerImage != nil && imageUsage.NumInputImages > 0 { + return float64(imageUsage.NumInputImages) * *pricing.InputCostPerImage + } + + return 0 +} + +// computeImageOutputCost calculates output cost: tokens first, then per-pixel, then per-image count fallback. +// imageQuality: "low", "medium", "high", "auto" use quality-specific rates when available; other values use base/size-tier rates. +func computeImageOutputCost(pricing *configstoreTables.TableModelPricing, imageUsage *schemas.ImageUsage, totalTokens int, pixels int, imageQuality string, tier serviceTier) float64 { + // Try token-based pricing first + var outputTextTokens, outputImageTokens int + if imageUsage.OutputTokensDetails != nil { + outputImageTokens = imageUsage.OutputTokensDetails.ImageTokens + outputTextTokens = imageUsage.OutputTokensDetails.TextTokens + } else { + outputImageTokens = imageUsage.OutputTokens + } + + if outputTextTokens > 0 || outputImageTokens > 0 { + return float64(outputTextTokens)*tieredOutputRate(pricing, totalTokens, tier) + + float64(outputImageTokens)*tieredImageOutputRate(pricing, totalTokens, tier) + } + + // Per-pixel pricing fallback + if pricing.OutputCostPerPixel != nil && pixels > 0 { + numOutputImages := 1 + if imageUsage.OutputTokensDetails != nil && imageUsage.OutputTokensDetails.NImages > 0 { + numOutputImages = imageUsage.OutputTokensDetails.NImages + } + return float64(pixels*numOutputImages) * *pricing.OutputCostPerPixel + } + + // Fall back to per-image count pricing with size-tier selection + // TODO: handle premium image flag when it becomes available in imageUsage + numOutputImages := 1 + if imageUsage.OutputTokensDetails != nil && imageUsage.OutputTokensDetails.NImages > 0 { + numOutputImages = imageUsage.OutputTokensDetails.NImages + } + var perImageRate *float64 + q := imageQuality + if q == "" { + q = "auto" + } + switch q { + case "low": + if pricing.OutputCostPerImageLowQuality != nil { + perImageRate = pricing.OutputCostPerImageLowQuality + } + case "medium": + if pricing.OutputCostPerImageMediumQuality != nil { + perImageRate = pricing.OutputCostPerImageMediumQuality + } + case "high": + if pricing.OutputCostPerImageHighQuality != nil { + perImageRate = pricing.OutputCostPerImageHighQuality + } + case "auto": + if pricing.OutputCostPerImageAutoQuality != nil { + perImageRate = pricing.OutputCostPerImageAutoQuality + } + } + if perImageRate == nil { + const pixels512x512 = 512 * 512 + const pixels1024x1024 = 1024 * 1024 + const pixels2048x2048 = 2048 * 2048 + const pixels4096x4096 = 4096 * 4096 + switch { + case pixels >= pixels4096x4096 && pricing.OutputCostPerImageAbove4096x4096Pixels != nil: + perImageRate = pricing.OutputCostPerImageAbove4096x4096Pixels + case pixels >= pixels2048x2048 && pricing.OutputCostPerImageAbove2048x2048Pixels != nil: + perImageRate = pricing.OutputCostPerImageAbove2048x2048Pixels + case pixels >= pixels1024x1024 && pricing.OutputCostPerImageAbove1024x1024Pixels != nil: + perImageRate = pricing.OutputCostPerImageAbove1024x1024Pixels + case pixels >= pixels512x512 && pricing.OutputCostPerImageAbove512x512Pixels != nil: + perImageRate = pricing.OutputCostPerImageAbove512x512Pixels + default: + perImageRate = pricing.OutputCostPerImage + } + } + if perImageRate != nil { + return float64(numOutputImages) * *perImageRate + } + + return 0 +} + +// computeVideoCost handles video generation requests. +// Input and output are calculated independently — tokens first, then per-second fallback. +func computeVideoCost(pricing *configstoreTables.TableModelPricing, usage *schemas.BifrostLLMUsage, videoSeconds *int, tier serviceTier) float64 { + totalTokens := safeTotalTokens(usage) + + // Input: text prompt tokens first, then per-second fallback + inputCost := 0.0 + if usage != nil && usage.PromptTokens > 0 { + inputCost = float64(usage.PromptTokens) * tieredInputRate(pricing, totalTokens, tier) + } else if videoSeconds != nil && *videoSeconds > 0 { + if rate := tieredVideoInputPerSecondRate(pricing, totalTokens); rate > 0 { + inputCost = float64(*videoSeconds) * rate + } + } + + // Output: completion tokens first, then per-second fallback + outputCost := 0.0 + if usage != nil && usage.CompletionTokens > 0 { + outputCost = float64(usage.CompletionTokens) * tieredOutputRate(pricing, totalTokens, tier) + } else if videoSeconds != nil && *videoSeconds > 0 { + if pricing.OutputCostPerVideoPerSecond != nil { + outputCost = float64(*videoSeconds) * *pricing.OutputCostPerVideoPerSecond + } else if pricing.OutputCostPerSecond != nil { + outputCost = float64(*videoSeconds) * *pricing.OutputCostPerSecond + } + } + + return inputCost + outputCost +} + +// computeOCRCost handles OCR requests, billing per page processed. +// ocr_cost_per_page covers base processing; annotation_cost_per_page is added when set. +func computeOCRCost(pricing *configstoreTables.TableModelPricing, ocrProcessedPages *int, ocrIsAnnotated *bool) float64 { + if ocrProcessedPages == nil { + return 0 + } + pages := float64(*ocrProcessedPages) + cost := 0.0 + if pricing.OCRCostPerPage != nil { + cost += pages * *pricing.OCRCostPerPage + } + if ocrIsAnnotated != nil && *ocrIsAnnotated && pricing.AnnotationCostPerPage != nil { + cost += pages * *pricing.AnnotationCostPerPage + } + return cost +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +// tierFromResponse builds a serviceTier from a response's billing-relevant +// fields: the OpenAI service_tier (priority/flex) and the Anthropic speed +// (fast mode). speed == "fast" means fast mode was actually served — the +// provider echoes the served speed, so stripped/fell-back requests report +// "standard" and bill at standard rates. +func tierFromResponse(s *schemas.BifrostServiceTier, speed *string) serviceTier { + var tier serviceTier + if s != nil { + switch *s { + case schemas.BifrostServiceTierPriority: + tier.isPriority = true + case schemas.BifrostServiceTierFlex: + tier.isFlex = true + } + } + tier.isFast = speed != nil && *speed == "fast" + return tier +} + +// tieredInputRate returns the effective per-token input rate based on total token count. +// Flex applies a flat rate. Priority-specific tier rates are preferred where available. +func tieredInputRate(pricing *configstoreTables.TableModelPricing, totalTokens int, tier serviceTier) float64 { + // Fast mode (Anthropic) is a flat rate across the full context window — it + // takes precedence over the token-count tiers below. + if tier.isFast && pricing.InputCostPerTokenFast != nil { + return *pricing.InputCostPerTokenFast + } + if tier.isFlex && pricing.InputCostPerTokenFlex != nil { + return *pricing.InputCostPerTokenFlex + } + if totalTokens > TokenTierAbove272K { + if tier.isPriority && pricing.InputCostPerTokenAbove272kTokensPriority != nil { + return *pricing.InputCostPerTokenAbove272kTokensPriority + } + if pricing.InputCostPerTokenAbove272kTokens != nil { + return *pricing.InputCostPerTokenAbove272kTokens + } + } + if totalTokens > TokenTierAbove200K { + if tier.isPriority && pricing.InputCostPerTokenAbove200kTokensPriority != nil { + return *pricing.InputCostPerTokenAbove200kTokensPriority + } + if pricing.InputCostPerTokenAbove200kTokens != nil { + return *pricing.InputCostPerTokenAbove200kTokens + } + } + if totalTokens > TokenTierAbove128K && pricing.InputCostPerTokenAbove128kTokens != nil { + return *pricing.InputCostPerTokenAbove128kTokens + } + if tier.isPriority && pricing.InputCostPerTokenPriority != nil { + return *pricing.InputCostPerTokenPriority + } + if pricing.InputCostPerToken != nil { + return *pricing.InputCostPerToken + } + return 0 +} + +// tieredOutputRate returns the effective per-token output rate based on total token count. +// Flex applies a flat rate. Priority-specific tier rates are preferred where available. +func tieredOutputRate(pricing *configstoreTables.TableModelPricing, totalTokens int, tier serviceTier) float64 { + // Fast mode (Anthropic) is a flat rate across the full context window — it + // takes precedence over the token-count tiers below. + if tier.isFast && pricing.OutputCostPerTokenFast != nil { + return *pricing.OutputCostPerTokenFast + } + if tier.isFlex && pricing.OutputCostPerTokenFlex != nil { + return *pricing.OutputCostPerTokenFlex + } + if totalTokens > TokenTierAbove272K { + if tier.isPriority && pricing.OutputCostPerTokenAbove272kTokensPriority != nil { + return *pricing.OutputCostPerTokenAbove272kTokensPriority + } + if pricing.OutputCostPerTokenAbove272kTokens != nil { + return *pricing.OutputCostPerTokenAbove272kTokens + } + } + if totalTokens > TokenTierAbove200K { + if tier.isPriority && pricing.OutputCostPerTokenAbove200kTokensPriority != nil { + return *pricing.OutputCostPerTokenAbove200kTokensPriority + } + if pricing.OutputCostPerTokenAbove200kTokens != nil { + return *pricing.OutputCostPerTokenAbove200kTokens + } + } + if totalTokens > TokenTierAbove128K && pricing.OutputCostPerTokenAbove128kTokens != nil { + return *pricing.OutputCostPerTokenAbove128kTokens + } + + if tier.isPriority && pricing.OutputCostPerTokenPriority != nil { + return *pricing.OutputCostPerTokenPriority + } + + if pricing.OutputCostPerToken != nil { + return *pricing.OutputCostPerToken + } + + return 0 +} + +// tieredImageInputRate returns the effective rate for image tokens on the input side. +// Falls back to the general tieredInputRate when no image-specific rate is configured. +func tieredImageInputRate(pricing *configstoreTables.TableModelPricing, totalTokens int, tier serviceTier) float64 { + if totalTokens > TokenTierAbove128K && pricing.InputCostPerImageAbove128kTokens != nil { + return *pricing.InputCostPerImageAbove128kTokens + } + if pricing.InputCostPerImageToken != nil { + return *pricing.InputCostPerImageToken + } + return tieredInputRate(pricing, totalTokens, tier) +} + +// tieredImageOutputRate returns the effective rate for image tokens on the output side. +// Falls back to the general tieredOutputRate when no image-specific rate is configured. +func tieredImageOutputRate(pricing *configstoreTables.TableModelPricing, totalTokens int, tier serviceTier) float64 { + if pricing.OutputCostPerImageToken != nil { + return *pricing.OutputCostPerImageToken + } + return tieredOutputRate(pricing, totalTokens, tier) +} + +// tieredAudioInputPerSecondRate returns the effective per-second rate for audio input. +func tieredAudioInputPerSecondRate(pricing *configstoreTables.TableModelPricing, totalTokens int) float64 { + if totalTokens > TokenTierAbove128K && pricing.InputCostPerAudioPerSecondAbove128kTokens != nil { + return *pricing.InputCostPerAudioPerSecondAbove128kTokens + } + if pricing.InputCostPerAudioPerSecond != nil { + return *pricing.InputCostPerAudioPerSecond + } + if pricing.InputCostPerSecond != nil { + return *pricing.InputCostPerSecond + } + return 0 +} + +// tieredVideoInputPerSecondRate returns the effective per-second rate for video input. +func tieredVideoInputPerSecondRate(pricing *configstoreTables.TableModelPricing, totalTokens int) float64 { + if totalTokens > TokenTierAbove128K && pricing.InputCostPerVideoPerSecondAbove128kTokens != nil { + return *pricing.InputCostPerVideoPerSecondAbove128kTokens + } + if pricing.InputCostPerVideoPerSecond != nil { + return *pricing.InputCostPerVideoPerSecond + } + return 0 +} + +// tieredAudioTokenInputRate returns the effective per-token rate for audio input tokens. +// Falls back to the general tieredInputRate when no audio-specific rate is configured. +func tieredAudioTokenInputRate(pricing *configstoreTables.TableModelPricing, totalTokens int, tier serviceTier) float64 { + if pricing.InputCostPerAudioToken != nil { + return *pricing.InputCostPerAudioToken + } + return tieredInputRate(pricing, totalTokens, tier) +} + +// tieredAudioTokenOutputRate returns the effective per-token rate for audio output tokens. +// Falls back to the general tieredOutputRate when no audio-specific rate is configured. +func tieredAudioTokenOutputRate(pricing *configstoreTables.TableModelPricing, totalTokens int, tier serviceTier) float64 { + if pricing.OutputCostPerAudioToken != nil { + return *pricing.OutputCostPerAudioToken + } + return tieredOutputRate(pricing, totalTokens, tier) +} + +func tieredCacheReadInputTokenRate(pricing *configstoreTables.TableModelPricing, totalTokens int, tier serviceTier) float64 { + if tier.isFlex && pricing.CacheReadInputTokenCostFlex != nil { + return *pricing.CacheReadInputTokenCostFlex + } + if totalTokens > TokenTierAbove272K { + if tier.isPriority && pricing.CacheReadInputTokenCostAbove272kTokensPriority != nil { + return *pricing.CacheReadInputTokenCostAbove272kTokensPriority + } + if pricing.CacheReadInputTokenCostAbove272kTokens != nil { + return *pricing.CacheReadInputTokenCostAbove272kTokens + } + } + if totalTokens > TokenTierAbove200K { + if tier.isPriority && pricing.CacheReadInputTokenCostAbove200kTokensPriority != nil { + return *pricing.CacheReadInputTokenCostAbove200kTokensPriority + } + if pricing.CacheReadInputTokenCostAbove200kTokens != nil { + return *pricing.CacheReadInputTokenCostAbove200kTokens + } + } + if tier.isPriority && pricing.CacheReadInputTokenCostPriority != nil { + return *pricing.CacheReadInputTokenCostPriority + } + if pricing.CacheReadInputTokenCost != nil { + return *pricing.CacheReadInputTokenCost + } + return tieredInputRate(pricing, totalTokens, tier) +} + +// Note: flex tier is not checked here because cache creation is not a concept in +// OpenAI's pricing model (the only provider that uses flex tier). Only cache read +// has a flex-specific rate. +func tieredCacheCreationInputTokenRate(pricing *configstoreTables.TableModelPricing, totalTokens int, tier serviceTier) float64 { + if totalTokens > TokenTierAbove200K && pricing.CacheCreationInputTokenCostAbove200kTokens != nil { + return *pricing.CacheCreationInputTokenCostAbove200kTokens + } + if pricing.CacheCreationInputTokenCost != nil { + return *pricing.CacheCreationInputTokenCost + } + return tieredInputRate(pricing, totalTokens, tier) +} + +func tieredCacheCreationInputAbove1hrTokenRate(pricing *configstoreTables.TableModelPricing, totalTokens int, tier serviceTier) float64 { + if totalTokens > TokenTierAbove200K && pricing.CacheCreationInputTokenCostAbove1hrAbove200kTokens != nil { + return *pricing.CacheCreationInputTokenCostAbove1hrAbove200kTokens + } + if pricing.CacheCreationInputTokenCostAbove1hr != nil { + return *pricing.CacheCreationInputTokenCostAbove1hr + } + return tieredCacheCreationInputTokenRate(pricing, totalTokens, tier) +} + +func safeTotalTokens(usage *schemas.BifrostLLMUsage) int { + if usage == nil { + return 0 + } + return usage.TotalTokens +} + +// parseImagePixels parses a size string like "1024x1024" into total pixel count. +// Returns 0 if the size string is empty or malformed. +func parseImagePixels(size string) int { + if size == "" { + return 0 + } + parts := strings.SplitN(size, "x", 2) + if len(parts) != 2 { + return 0 + } + w, err := strconv.Atoi(parts[0]) + if err != nil || w <= 0 { + return 0 + } + h, err := strconv.Atoi(parts[1]) + if err != nil || h <= 0 { + return 0 + } + return w * h +} + +// populateOutputImageCount sets the output image count on ImageUsage from len(Data) +// when OutputTokensDetails.NImages is not already populated. +func populateOutputImageCount(imageUsage *schemas.ImageUsage, dataLen int) { + if imageUsage == nil || dataLen == 0 { + return + } + if imageUsage.OutputTokensDetails == nil { + imageUsage.OutputTokensDetails = &schemas.ImageTokenDetails{} + } + if imageUsage.OutputTokensDetails.NImages == 0 { + imageUsage.OutputTokensDetails.NImages = dataLen + } +} + +// --------------------------------------------------------------------------- +// Pricing resolution +// --------------------------------------------------------------------------- + +// resolvePricing resolves the pricing entry for a request directly from the +// RoutingInfo populated on the response/error by core.bifrost at request time. +// +// Lookup precedence — AliasModelName → AliasModelID → ModelName. Each +// non-empty candidate is tried against the base catalog in order; the first +// hit wins. +// +// - AliasModelName (RoutingInfo.ResolvedKeyAlias.ModelName) is the canonical +// model name the admin tagged on the matched alias. Catches the +// opaque-deployment-ID case where the wire model wouldn't hit the catalog +// on its own. +// - AliasModelID (RoutingInfo.ResolvedKeyAlias.ModelID) is the wire model +// when an alias matched. nil/empty otherwise. +// - ModelName (RoutingInfo.Model) is the model string the caller sent — the +// alias key when an alias matched, or the raw user input when none did. +// +// Overrides are applied keyed by the wire model (AliasModelID when an alias +// matched, otherwise ModelName) so per-deployment override pricing stays +// addressable in either flow. +func (s *Store) resolvePricing(routingInfo schemas.RoutingInfo, requestType schemas.RequestType, scopes LookupScopes) *configstoreTables.TableModelPricing { + provider := string(routingInfo.Provider) + var aliasModelID, aliasModelName string + if rka := routingInfo.ResolvedKeyAlias; rka != nil { + aliasModelID = rka.ModelID + if rka.ModelName != nil { + aliasModelName = *rka.ModelName + } + } + overrideKey := aliasModelID + if overrideKey == "" { + overrideKey = routingInfo.Model + } + s.logger.Debug("looking up pricing for wire model %s and provider %s of request type %s", overrideKey, provider, normalizeRequestType(requestType)) + + if scopes.Provider == "" { + scopes.Provider = provider + } + + for _, candidate := range []string{aliasModelName, aliasModelID, routingInfo.Model} { + if candidate == "" { + continue + } + base, exists := s.getBasePricing(candidate, provider, requestType) + if exists && base != nil { + result, _ := s.applyPricingOverrides(overrideKey, requestType, *base, scopes) + return &result + } + s.logger.Debug("pricing not found for %s, trying next candidate", candidate) + } + + // No base catalog entry found; still try overrides in case the user defined + // override-only pricing for a model not in the built-in catalog. + s.logger.Debug("pricing not found for any candidate (provider %s), trying override-only pricing keyed by %s", provider, overrideKey) + result, applied := s.applyPricingOverrides(overrideKey, requestType, configstoreTables.TableModelPricing{}, scopes) + if applied { + return &result + } + s.logger.Debug("no pricing found for wire model %s and provider %s, skipping cost calculation", overrideKey, provider) + return nil +} + +// getBasePricing looks up catalog pricing for the given model, provider, and request type. +// It applies a provider-specific fallback chain when an exact match is not found: +// +// - Gemini: retries under the "vertex" provider, then falls back to chat mode for Responses requests. +// - Vertex: strips the "provider/model" prefix and retries, then falls back to chat mode for Responses requests. +// - Bedrock: prepends the "anthropic." namespace for Claude models, then falls back to chat mode for Responses requests. +// - All providers: for Responses/ResponsesStream requests, retries the lookup in chat mode. +// - All providers: for ImageEdit/ImageVariation requests, retries the lookup in image-generation mode. +// +// The method acquires a read lock for the duration of the lookup. +// +// Input: model — exact model name to look up. +// +// provider — provider identifier (e.g. "openai", "anthropic"). +// requestType — the request type used to derive the pricing mode. +// +// Output: TableModelPricing — the matched pricing row (zero value when not found). +// +// bool — true when a pricing entry was found, false otherwise. +func (s *Store) getBasePricing(model, provider string, requestType schemas.RequestType) (*configstoreTables.TableModelPricing, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + + mode := normalizeRequestType(requestType) + + pricing, ok := s.pricingData[makeKey(model, provider, mode)] + if ok { + return &pricing, true + } + + // Lookup in vertex if gemini not found + if provider == string(schemas.Gemini) { + s.logger.Debug("primary lookup failed, trying vertex provider for the same model") + pricing, ok = s.pricingData[makeKey(model, "vertex", mode)] + if ok { + return &pricing, true + } + + // Lookup in chat if responses not found + if requestType == schemas.ResponsesRequest || requestType == schemas.ResponsesStreamRequest || requestType == schemas.WebSocketResponsesRequest || requestType == schemas.RealtimeRequest || requestType == schemas.CompactionRequest { + s.logger.Debug("secondary lookup failed, trying vertex provider for the same model in chat completion") + pricing, ok = s.pricingData[makeKey(model, "vertex", normalizeRequestType(schemas.ChatCompletionRequest))] + if ok { + return &pricing, true + } + } + } + + if provider == string(schemas.Vertex) { + // Vertex models can be of the form "provider/model", so try to lookup the model without the provider prefix and keep the original provider + if strings.Contains(model, "/") { + modelWithoutProvider := strings.SplitN(model, "/", 2)[1] + s.logger.Debug("primary lookup failed, trying vertex provider for the same model with provider/model format %s", modelWithoutProvider) + pricing, ok = s.pricingData[makeKey(modelWithoutProvider, "vertex", mode)] + if ok { + return &pricing, true + } + + // Lookup in chat if responses not found + if requestType == schemas.ResponsesRequest || requestType == schemas.ResponsesStreamRequest || requestType == schemas.WebSocketResponsesRequest || requestType == schemas.RealtimeRequest || requestType == schemas.CompactionRequest { + s.logger.Debug("secondary lookup failed, trying vertex provider for the same model in chat completion") + pricing, ok = s.pricingData[makeKey(modelWithoutProvider, "vertex", normalizeRequestType(schemas.ChatCompletionRequest))] + if ok { + return &pricing, true + } + } + } + } + + if provider == string(schemas.Bedrock) { + // If model is claude without "anthropic." prefix, try with "anthropic." prefix + if !strings.Contains(model, "anthropic.") && schemas.IsAnthropicModel(model) { + s.logger.Debug("primary lookup failed, trying with anthropic. prefix for the same model") + pricing, ok = s.pricingData[makeKey("anthropic."+model, provider, mode)] + if ok { + return &pricing, true + } + + // Lookup in chat if responses not found + if requestType == schemas.ResponsesRequest || requestType == schemas.ResponsesStreamRequest || requestType == schemas.WebSocketResponsesRequest || requestType == schemas.RealtimeRequest || requestType == schemas.CompactionRequest { + s.logger.Debug("secondary lookup failed, trying chat provider for the same model in chat completion") + pricing, ok = s.pricingData[makeKey("anthropic."+model, provider, normalizeRequestType(schemas.ChatCompletionRequest))] + if ok { + return &pricing, true + } + } + } + } + + // Lookup in chat if responses/compaction not found + if requestType == schemas.ResponsesRequest || requestType == schemas.ResponsesStreamRequest || requestType == schemas.WebSocketResponsesRequest || requestType == schemas.RealtimeRequest || requestType == schemas.CompactionRequest { + s.logger.Debug("primary lookup failed, trying chat provider for the same model in chat completion") + pricing, ok = s.pricingData[makeKey(model, provider, normalizeRequestType(schemas.ChatCompletionRequest))] + if ok { + return &pricing, true + } + } + + // Lookup in image generation if image edit not found + if requestType == schemas.ImageEditRequest || + requestType == schemas.ImageEditStreamRequest || + requestType == schemas.ImageVariationRequest { + s.logger.Debug("primary lookup failed, trying image generation provider for the same model") + pricing, ok = s.pricingData[makeKey(model, provider, normalizeRequestType(schemas.ImageGenerationRequest))] + if ok { + return &pricing, true + } + } + + // Lookup fallback chain for container_create: + // 1. Try chat mode for the same model (e.g. "container-1g" in chat mode) + // 2. Try the base "container" model in chat mode (default rate when no memory-specific entry exists) + if requestType == schemas.ContainerCreateRequest { + s.logger.Debug("primary lookup failed, trying chat mode for container create pricing") + pricing, ok = s.pricingData[makeKey(model, provider, normalizeRequestType(schemas.ChatCompletionRequest))] + if ok { + return &pricing, true + } + if model != "container" { + s.logger.Debug("memory-specific container pricing not found, falling back to base container entry") + pricing, ok = s.pricingData[makeKey("container", provider, normalizeRequestType(schemas.ChatCompletionRequest))] + if ok { + return &pricing, true + } + } + } + + return nil, false +} + +// UpsertModelPricingAttributes writes the additional_attributes column for +// every pricing row that matches (model, provider), then reloads the pricing +// cache so the new values are immediately visible to list-models. Returns +// the number of rows updated (0 = no such pricing row, which callers must +// surface as a validation error). An empty/nil attrs map clears the column. +func (s *Store) UpsertModelPricingAttributes(ctx context.Context, model string, provider schemas.ModelProvider, attrs map[string]string) (int64, error) { + if s.configStore == nil { + return 0, fmt.Errorf("model catalog requires a config store") + } + rows, err := s.configStore.UpsertModelPricingAttributes(ctx, model, string(provider), attrs) + if err != nil { + return 0, err + } + if rows == 0 { + return 0, nil + } + if err := s.LoadFromDB(ctx); err != nil { + return rows, fmt.Errorf("failed to reload pricing cache after attribute write: %w", err) + } + return rows, nil +} + +// --------------------------------------------------------------------------- +// Passthrough pricing helpers +// --------------------------------------------------------------------------- + +// detectPassthroughRequestType maps a provider + stripped path to a RequestType. +func detectPassthroughRequestType(provider schemas.ModelProvider, path string) schemas.RequestType { + if idx := strings.IndexByte(path, '?'); idx >= 0 { + path = path[:idx] + } + path = strings.TrimRight(path, "/") + switch provider { + case schemas.OpenAI, schemas.Azure: + switch { + case strings.HasSuffix(path, "/chat/completions"): + return schemas.ChatCompletionRequest + case strings.HasSuffix(path, "/completions"): + return schemas.TextCompletionRequest + case strings.HasSuffix(path, "/embeddings"): + return schemas.EmbeddingRequest + case strings.HasSuffix(path, "/responses/compact"): + return schemas.CompactionRequest + case strings.HasSuffix(path, "/responses"): + return schemas.ResponsesRequest + case strings.HasSuffix(path, "/images/generations"): + return schemas.ImageGenerationRequest + case strings.HasSuffix(path, "/images/edits"): + return schemas.ImageEditRequest + case strings.HasSuffix(path, "/images/variations"): + return schemas.ImageVariationRequest + case strings.HasSuffix(path, "/audio/speech"): + return schemas.SpeechRequest + case strings.HasSuffix(path, "/audio/transcriptions"), + strings.HasSuffix(path, "/audio/translations"): + return schemas.TranscriptionRequest + case strings.HasSuffix(path, "/containers"): + return schemas.ContainerCreateRequest + case strings.Contains(path, "/video"): + return schemas.VideoGenerationRequest + default: + return schemas.ChatCompletionRequest + } + case schemas.Gemini, schemas.Vertex: + // Interactions API paths carry no colon action suffix. + if strings.Contains(path, "/interactions") { + return schemas.ResponsesRequest + } + colonIdx := strings.LastIndexByte(path, ':') + if colonIdx < 0 { + return schemas.ChatCompletionRequest + } + switch path[colonIdx+1:] { + case "generateContent", "streamGenerateContent": + return schemas.ResponsesRequest + case "embedContent", "batchEmbedContents": + return schemas.EmbeddingRequest + case "generateImages": + return schemas.ImageGenerationRequest + case "predict": + return schemas.EmbeddingRequest + case "predictLongRunning": + return schemas.VideoGenerationRequest + default: + return schemas.ChatCompletionRequest + } + case schemas.Anthropic: + switch { + case strings.HasSuffix(path, "/messages"): + return schemas.ResponsesRequest + case strings.HasSuffix(path, "/complete"): + return schemas.TextCompletionRequest + default: + return schemas.ResponsesRequest + } + default: + return schemas.ChatCompletionRequest + } +} + +// inferPassthroughRequestType determines the request type from usage fields (primary) +// and falls back to path detection for text/embedding/responses where LLMUsage is ambiguous. +func inferPassthroughRequestType(provider schemas.ModelProvider, path string, su *schemas.BifrostPassthroughUsage) schemas.RequestType { + if su != nil { + if su.ContainerIdentifier != "" { + return schemas.ContainerCreateRequest + } + if su.ImageUsage != nil { + return schemas.ImageGenerationRequest + } + if su.AudioInputChars > 0 { + return schemas.SpeechRequest + } + if su.AudioTokenDetails != nil || su.AudioSeconds != nil { + return schemas.TranscriptionRequest + } + if su.VideoSeconds != nil { + return schemas.VideoGenerationRequest + } + } + return detectPassthroughRequestType(provider, path) +} + +// passthroughUsageToCostInput converts BifrostPassthroughUsage into costInput. +func passthroughUsageToCostInput(su *schemas.BifrostPassthroughUsage) costInput { + var input costInput + if su.LLMUsage != nil { + input.usage = su.LLMUsage + } + input.tier = tierFromResponse(su.ServiceTier, su.Speed) + if su.ImageUsage != nil { + input.imageUsage = su.ImageUsage + input.imageSize = su.ImageSize + input.imageQuality = su.ImageQuality + } + if su.AudioInputChars > 0 { + input.audioTextInputChars = su.AudioInputChars + } + if su.AudioSeconds != nil { + input.audioSeconds = su.AudioSeconds + } + if su.AudioTokenDetails != nil { + input.audioTokenDetails = su.AudioTokenDetails + } + if su.VideoSeconds != nil { + input.videoSeconds = su.VideoSeconds + } + if su.ContainerIdentifier != "" { + input.containerIdentifierString = su.ContainerIdentifier + } + return input +} diff --git a/framework/modelcatalog/pricing_test.go b/framework/modelcatalog/datasheet/cost_test.go similarity index 83% rename from framework/modelcatalog/pricing_test.go rename to framework/modelcatalog/datasheet/cost_test.go index 617c6ac86b..a407df145d 100644 --- a/framework/modelcatalog/pricing_test.go +++ b/framework/modelcatalog/datasheet/cost_test.go @@ -1,9 +1,6 @@ -package modelcatalog +package datasheet import ( - "context" - "encoding/json" - "os" "testing" bifrost "github.com/maximhq/bifrost/core" @@ -28,14 +25,20 @@ func chatPricing(input, output float64) configstoreTables.TableModelPricing { } } -// testCatalogWithPricing creates a catalog pre-loaded with the given pricing entries. -func testCatalogWithPricing(entries map[string]configstoreTables.TableModelPricing) *ModelCatalog { - mc := newTestCatalog(nil, nil) - mc.logger = noOpLogger{} +// testStoreWithPricing creates a catalog pre-loaded with the given pricing entries. +func testStoreWithPricing(entries map[string]configstoreTables.TableModelPricing) *Store { + s := newTestStore() + for k, v := range entries { - mc.pricingData[k] = v + s.pricingData[k] = v } - return mc + return s +} + +// routingInfoFor builds a minimal RoutingInfo populated by core.bifrost for a +// non-aliased request — the form pricing reads from. +func routingInfoFor(provider schemas.ModelProvider, model string) schemas.RoutingInfo { + return schemas.RoutingInfo{Provider: provider, Model: model} } // makeChatResponse builds a minimal BifrostResponse for a chat completion. @@ -44,9 +47,8 @@ func makeChatResponse(provider schemas.ModelProvider, model string, usage *schem ChatResponse: &schemas.BifrostChatResponse{ Usage: usage, ExtraFields: schemas.BifrostResponseExtraFields{ - RequestType: schemas.ChatCompletionRequest, - Provider: provider, - OriginalModelRequested: model, + RequestType: schemas.ChatCompletionRequest, + RoutingInfo: routingInfoFor(provider, model), }, }, } @@ -58,9 +60,8 @@ func makeEmbeddingResponse(provider schemas.ModelProvider, model string, usage * EmbeddingResponse: &schemas.BifrostEmbeddingResponse{ Usage: usage, ExtraFields: schemas.BifrostResponseExtraFields{ - RequestType: schemas.EmbeddingRequest, - Provider: provider, - OriginalModelRequested: model, + RequestType: schemas.EmbeddingRequest, + RoutingInfo: routingInfoFor(provider, model), }, }, } @@ -72,9 +73,8 @@ func makeRerankResponse(provider schemas.ModelProvider, model string, usage *sch RerankResponse: &schemas.BifrostRerankResponse{ Usage: usage, ExtraFields: schemas.BifrostResponseExtraFields{ - RequestType: schemas.RerankRequest, - Provider: provider, - OriginalModelRequested: model, + RequestType: schemas.RerankRequest, + RoutingInfo: routingInfoFor(provider, model), }, }, } @@ -86,9 +86,8 @@ func makeImageResponse(provider schemas.ModelProvider, model string, usage *sche ImageGenerationResponse: &schemas.BifrostImageGenerationResponse{ Usage: usage, ExtraFields: schemas.BifrostResponseExtraFields{ - RequestType: schemas.ImageGenerationRequest, - Provider: provider, - OriginalModelRequested: model, + RequestType: schemas.ImageGenerationRequest, + RoutingInfo: routingInfoFor(provider, model), }, }, } @@ -154,6 +153,90 @@ func TestComputeTextCost_WithCachedPromptTokens(t *testing.T) { assert.InDelta(t, 0.0096, cost, 1e-12) } +func TestComputeTextCost_FastMode(t *testing.T) { + // Opus 4.8: standard $5/$25, fast $10/$50 per MTok. + p := chatPricing(0.000005, 0.000025) + p.InputCostPerTokenFast = bifrost.Ptr(0.00001) + p.OutputCostPerTokenFast = bifrost.Ptr(0.00005) + + usage := &schemas.BifrostLLMUsage{ + PromptTokens: 1000, + CompletionTokens: 500, + TotalTokens: 1500, + } + + // Standard speed → standard rates. + standard := computeTextCost(&p, usage, serviceTier{}) + assert.InDelta(t, 1000*0.000005+500*0.000025, standard, 1e-12) + + // Fast speed → fast rates. + fast := computeTextCost(&p, usage, serviceTier{isFast: true}) + assert.InDelta(t, 1000*0.00001+500*0.00005, fast, 1e-12) +} + +func TestComputeTextCost_FastMode_FlatAcrossContextWindow(t *testing.T) { + // Fast mode is flat across the full window — it must ignore the 200k tier rate. + p := chatPricing(0.000005, 0.000025) + p.InputCostPerTokenFast = bifrost.Ptr(0.00001) + p.OutputCostPerTokenFast = bifrost.Ptr(0.00005) + p.InputCostPerTokenAbove200kTokens = bifrost.Ptr(0.0000075) + p.OutputCostPerTokenAbove200kTokens = bifrost.Ptr(0.0000375) + + usage := &schemas.BifrostLLMUsage{ + PromptTokens: 250000, + CompletionTokens: 1000, + TotalTokens: 251000, // above the 200k tier + } + + fast := computeTextCost(&p, usage, serviceTier{isFast: true}) + // Flat fast rate, not the above-200k rate. + assert.InDelta(t, 250000*0.00001+1000*0.00005, fast, 1e-9) +} + +func TestComputeTextCost_FastMode_FallsBackWhenUnconfigured(t *testing.T) { + // Model without fast columns (e.g. non-Opus) → fast flag is a no-op, standard rates apply. + p := chatPricing(0.000005, 0.000025) + usage := &schemas.BifrostLLMUsage{ + PromptTokens: 1000, + CompletionTokens: 500, + TotalTokens: 1500, + } + fast := computeTextCost(&p, usage, serviceTier{isFast: true}) + assert.InDelta(t, 1000*0.000005+500*0.000025, fast, 1e-12) +} + +func TestComputeTextCost_FastMode_CacheBillsAtStandardRates(t *testing.T) { + // Per design: cache tokens on a fast request bill at standard cache rates; + // only the non-cached input and the output use the fast rate. + p := chatPricing(0.000005, 0.000025) + p.InputCostPerTokenFast = bifrost.Ptr(0.00001) + p.OutputCostPerTokenFast = bifrost.Ptr(0.00005) + p.CacheReadInputTokenCost = bifrost.Ptr(0.0000005) // standard read + p.CacheCreationInputTokenCost = bifrost.Ptr(0.00000625) // standard 5m write + + usage := &schemas.BifrostLLMUsage{ + PromptTokens: 2000, + CompletionTokens: 500, + TotalTokens: 2500, + PromptTokensDetails: &schemas.ChatPromptTokensDetails{ + CachedReadTokens: 1500, + CachedWriteTokens: 200, + }, + } + + fast := computeTextCost(&p, usage, serviceTier{isFast: true}) + // Input: non-cached (2000-1500-200)*fast + read 1500*stdRead + write 200*stdWrite + // = 300*0.00001 + 1500*0.0000005 + 200*0.00000625 = 0.003 + 0.00075 + 0.00125 = 0.0019(? recompute) + expected := 300*0.00001 + 1500*0.0000005 + 200*0.00000625 + 500*0.00005 + assert.InDelta(t, expected, fast, 1e-12) +} + +func TestTierFromResponse_Speed(t *testing.T) { + assert.False(t, tierFromResponse(nil, nil).isFast) + assert.False(t, tierFromResponse(nil, bifrost.Ptr("standard")).isFast) + assert.True(t, tierFromResponse(nil, bifrost.Ptr("fast")).isFast) +} + func TestComputeTextCost_With1hrCacheCreationTokens(t *testing.T) { // claude-3-5-sonnet-20241022-v2:0 on Bedrock: // input=$3/M, output=$15/M, cache_creation=$3.75/M, cache_creation_1hr=$7.50/M, cache_read=$0.3/M @@ -1146,7 +1229,7 @@ func TestExtractCostInput_VideoResponseInvalidSeconds(t *testing.T) { // ========================================================================= func TestCalculateCost_SemanticCacheDirectHit(t *testing.T) { - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("gpt-4o", "openai", "chat"): { Model: "gpt-4o", Provider: "openai", Mode: "chat", InputCostPerToken: bifrost.Ptr(0.000005), OutputCostPerToken: bifrost.Ptr(0.000015), @@ -1158,9 +1241,8 @@ func TestCalculateCost_SemanticCacheDirectHit(t *testing.T) { ChatResponse: &schemas.BifrostChatResponse{ Usage: &schemas.BifrostLLMUsage{PromptTokens: 100, CompletionTokens: 50, TotalTokens: 150}, ExtraFields: schemas.BifrostResponseExtraFields{ - RequestType: schemas.ChatCompletionRequest, - Provider: schemas.OpenAI, - OriginalModelRequested: "gpt-4o", + RequestType: schemas.ChatCompletionRequest, + RoutingInfo: routingInfoFor(schemas.OpenAI, "gpt-4o"), CacheDebug: &schemas.BifrostCacheDebug{ CacheHit: true, HitType: &hitType, @@ -1169,7 +1251,7 @@ func TestCalculateCost_SemanticCacheDirectHit(t *testing.T) { }, } - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) assert.Equal(t, 0.0, cost) } @@ -1178,7 +1260,7 @@ func TestCalculateCost_SemanticCacheSemanticHit(t *testing.T) { embModel := "text-embedding-3-small" embTokens := 500 - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("gpt-4o", "openai", "chat"): { Model: "gpt-4o", Provider: "openai", Mode: "chat", InputCostPerToken: bifrost.Ptr(0.000005), OutputCostPerToken: bifrost.Ptr(0.000015), @@ -1194,9 +1276,8 @@ func TestCalculateCost_SemanticCacheSemanticHit(t *testing.T) { ChatResponse: &schemas.BifrostChatResponse{ Usage: &schemas.BifrostLLMUsage{PromptTokens: 100, CompletionTokens: 50, TotalTokens: 150}, ExtraFields: schemas.BifrostResponseExtraFields{ - RequestType: schemas.ChatCompletionRequest, - Provider: schemas.OpenAI, - OriginalModelRequested: "gpt-4o", + RequestType: schemas.ChatCompletionRequest, + RoutingInfo: routingInfoFor(schemas.OpenAI, "gpt-4o"), CacheDebug: &schemas.BifrostCacheDebug{ CacheHit: true, HitType: &hitType, @@ -1208,7 +1289,7 @@ func TestCalculateCost_SemanticCacheSemanticHit(t *testing.T) { }, } - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) // Only embedding cost: 500 * 0.00000002 = 0.00001 assert.InDelta(t, 0.00001, cost, 1e-12) } @@ -1218,7 +1299,7 @@ func TestCalculateCost_SemanticCacheMiss(t *testing.T) { embModel := "text-embedding-3-small" embTokens := 500 - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("gpt-4o", "openai", "chat"): { Model: "gpt-4o", Provider: "openai", Mode: "chat", InputCostPerToken: bifrost.Ptr(0.000005), OutputCostPerToken: bifrost.Ptr(0.000015), @@ -1233,9 +1314,8 @@ func TestCalculateCost_SemanticCacheMiss(t *testing.T) { ChatResponse: &schemas.BifrostChatResponse{ Usage: &schemas.BifrostLLMUsage{PromptTokens: 1000, CompletionTokens: 500, TotalTokens: 1500}, ExtraFields: schemas.BifrostResponseExtraFields{ - RequestType: schemas.ChatCompletionRequest, - Provider: schemas.OpenAI, - OriginalModelRequested: "gpt-4o", + RequestType: schemas.ChatCompletionRequest, + RoutingInfo: routingInfoFor(schemas.OpenAI, "gpt-4o"), CacheDebug: &schemas.BifrostCacheDebug{ CacheHit: false, ProviderUsed: &embProvider, @@ -1246,7 +1326,7 @@ func TestCalculateCost_SemanticCacheMiss(t *testing.T) { }, } - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) // Base cost: 1000*0.000005 + 500*0.000015 = 0.005 + 0.0075 = 0.0125 // Embedding cost: 500 * 0.00000002 = 0.00001 // Total: 0.01251 @@ -1254,7 +1334,7 @@ func TestCalculateCost_SemanticCacheMiss(t *testing.T) { } func TestCalculateCost_SemanticCacheHitNoEmbeddingInfo(t *testing.T) { - mc := testCatalogWithPricing(nil) + s := testStoreWithPricing(nil) resp := &schemas.BifrostResponse{ ChatResponse: &schemas.BifrostChatResponse{ @@ -1267,7 +1347,7 @@ func TestCalculateCost_SemanticCacheHitNoEmbeddingInfo(t *testing.T) { }, } - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) assert.Equal(t, 0.0, cost) } @@ -1276,12 +1356,12 @@ func TestCalculateCost_SemanticCacheHitNoEmbeddingInfo(t *testing.T) { // ========================================================================= func TestCalculateCost_NilResponse(t *testing.T) { - mc := testCatalogWithPricing(nil) - assert.Equal(t, 0.0, mc.CalculateCost(nil, nil)) + s := testStoreWithPricing(nil) + assert.Equal(t, 0.0, s.CalculateCost(nil, nil)) } func TestCalculateCost_ProviderComputedCostPassthrough(t *testing.T) { - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("gpt-4o", "openai", "chat"): chatPricing(0.000005, 0.000015), }) @@ -1294,23 +1374,23 @@ func TestCalculateCost_ProviderComputedCostPassthrough(t *testing.T) { }, }) - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) assert.Equal(t, 0.99, cost) } func TestCalculateCost_NoUsageData(t *testing.T) { - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("gpt-4o", "openai", "chat"): chatPricing(0.000005, 0.000015), }) resp := makeChatResponse(schemas.OpenAI, "gpt-4o", nil) - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) assert.Equal(t, 0.0, cost) } func TestCalculateCost_ChatCompletion_GPT4o(t *testing.T) { // GPT-4o: $5/M input, $15/M output, cache_read=$0.5/M - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("gpt-4o", "openai", "chat"): { Model: "gpt-4o", Provider: "openai", Mode: "chat", InputCostPerToken: bifrost.Ptr(0.000005), @@ -1325,14 +1405,14 @@ func TestCalculateCost_ChatCompletion_GPT4o(t *testing.T) { TotalTokens: 12000, }) - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) // 10000*0.000005 + 2000*0.000015 = 0.05 + 0.03 = 0.08 assert.InDelta(t, 0.08, cost, 1e-12) } func TestCalculateCost_ChatCompletion_Claude35Sonnet_WithCache(t *testing.T) { // Claude 3.5 Sonnet (Bedrock): $3/M input, $15/M output, cache_read=$0.3/M, cache_creation=$3.75/M - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("anthropic.claude-3-5-sonnet-20241022-v2:0", "bedrock", "chat"): { Model: "anthropic.claude-3-5-sonnet-20241022-v2:0", Provider: "bedrock", Mode: "chat", InputCostPerToken: bifrost.Ptr(0.000003), @@ -1354,7 +1434,7 @@ func TestCalculateCost_ChatCompletion_Claude35Sonnet_WithCache(t *testing.T) { }, }) - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) // Both cached read and write tokens are input-side deductions from promptTokens. // Input: (5000-3000-500)*0.000003 + 3000*0.0000003 + 500*0.00000375 = 0.0045 + 0.0009 + 0.001875 = 0.007275 // Output: 1000*0.000015 = 0.015 @@ -1364,7 +1444,7 @@ func TestCalculateCost_ChatCompletion_Claude35Sonnet_WithCache(t *testing.T) { func TestCalculateCost_Embedding(t *testing.T) { // Titan Embed Text v1: $0.1/M input - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("amazon.titan-embed-text-v1", "bedrock", "embedding"): { Model: "amazon.titan-embed-text-v1", Provider: "bedrock", Mode: "embedding", InputCostPerToken: bifrost.Ptr(0.0000001), @@ -1377,13 +1457,13 @@ func TestCalculateCost_Embedding(t *testing.T) { TotalTokens: 10000, }) - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) // 10000 * 0.0000001 = 0.001 assert.InDelta(t, 0.001, cost, 1e-12) } func TestCalculateCost_Rerank(t *testing.T) { - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("amazon.rerank-v1:0", "bedrock", "rerank"): { Model: "amazon.rerank-v1:0", Provider: "bedrock", Mode: "rerank", InputCostPerToken: bifrost.Ptr(0.0), @@ -1396,13 +1476,13 @@ func TestCalculateCost_Rerank(t *testing.T) { TotalTokens: 500, }) - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) assert.Equal(t, 0.0, cost) } func TestCalculateCost_ImageGeneration(t *testing.T) { // dall-e-3 via aiml: output_cost_per_image=$0.052 - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("dall-e-3", "aiml", "image_generation"): { Model: "dall-e-3", Provider: "aiml", Mode: "image_generation", OutputCostPerImage: bifrost.Ptr(0.052), @@ -1413,13 +1493,13 @@ func TestCalculateCost_ImageGeneration(t *testing.T) { OutputTokensDetails: &schemas.ImageTokenDetails{NImages: 3}, }) - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) // 3 * 0.052 = 0.156 assert.InDelta(t, 0.156, cost, 1e-12) } func TestCalculateCost_StreamRequestTypeNormalized(t *testing.T) { - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("gpt-4o", "openai", "chat"): chatPricing(0.000005, 0.000015), }) @@ -1428,19 +1508,18 @@ func TestCalculateCost_StreamRequestTypeNormalized(t *testing.T) { ChatResponse: &schemas.BifrostChatResponse{ Usage: &schemas.BifrostLLMUsage{PromptTokens: 1000, CompletionTokens: 500, TotalTokens: 1500}, ExtraFields: schemas.BifrostResponseExtraFields{ - RequestType: schemas.ChatCompletionStreamRequest, - Provider: schemas.OpenAI, - OriginalModelRequested: "gpt-4o", + RequestType: schemas.ChatCompletionStreamRequest, + RoutingInfo: routingInfoFor(schemas.OpenAI, "gpt-4o"), }, }, } - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) assert.InDelta(t, 0.0125, cost, 1e-12) } func TestCalculateCost_WebSocketResponsesFallsBackToChatPricing(t *testing.T) { - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("gpt-4o", "openai", "chat"): chatPricing(0.000005, 0.000015), }) @@ -1450,24 +1529,22 @@ func TestCalculateCost_WebSocketResponsesFallsBackToChatPricing(t *testing.T) { Usage: &schemas.ResponsesResponseUsage{InputTokens: 1000, OutputTokens: 500, TotalTokens: 1500}, }, ExtraFields: schemas.BifrostResponseExtraFields{ - RequestType: schemas.WebSocketResponsesRequest, - Provider: schemas.OpenAI, - OriginalModelRequested: "gpt-4o", - ResolvedModelUsed: "gpt-4o", + RequestType: schemas.WebSocketResponsesRequest, + RoutingInfo: routingInfoFor(schemas.OpenAI, "gpt-4o"), }, }, } - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) assert.InDelta(t, 0.0125, cost, 1e-12) } func TestCalculateCost_NoPricingData(t *testing.T) { - mc := testCatalogWithPricing(nil) + s := testStoreWithPricing(nil) resp := makeChatResponse(schemas.OpenAI, "unknown-model", &schemas.BifrostLLMUsage{ PromptTokens: 1000, CompletionTokens: 500, TotalTokens: 1500, }) - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) assert.Equal(t, 0.0, cost) } @@ -1476,76 +1553,76 @@ func TestCalculateCost_NoPricingData(t *testing.T) { // ========================================================================= func TestGetPricing_DirectLookup(t *testing.T) { - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("gpt-4o", "openai", "chat"): chatPricing(0.000005, 0.000015), }) - p := mc.resolvePricing("openai", "gpt-4o", "", schemas.ChatCompletionRequest, PricingLookupScopes{Provider: "openai"}) + p := s.resolvePricing(schemas.RoutingInfo{Provider: "openai", Model: "gpt-4o"}, schemas.ChatCompletionRequest, LookupScopes{Provider: "openai"}) assert.Equal(t, 0.000005, derefF(p.InputCostPerToken)) } func TestGetPricing_GeminiFallsBackToVertex(t *testing.T) { - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("gemini-2.0-flash", "vertex", "chat"): { Model: "gemini-2.0-flash", Provider: "vertex", Mode: "chat", InputCostPerToken: bifrost.Ptr(0.0000001), OutputCostPerToken: bifrost.Ptr(0.0000004), }, }) - p := mc.resolvePricing("gemini", "gemini-2.0-flash", "", schemas.ChatCompletionRequest, PricingLookupScopes{Provider: "gemini"}) + p := s.resolvePricing(schemas.RoutingInfo{Provider: "gemini", Model: "gemini-2.0-flash"}, schemas.ChatCompletionRequest, LookupScopes{Provider: "gemini"}) assert.Equal(t, 0.0000001, derefF(p.InputCostPerToken)) } func TestGetPricing_VertexStripsProviderPrefix(t *testing.T) { - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("gemini-2.0-flash", "vertex", "chat"): chatPricing(0.0000001, 0.0000004), }) - p := mc.resolvePricing("vertex", "google/gemini-2.0-flash", "", schemas.ChatCompletionRequest, PricingLookupScopes{Provider: "vertex"}) + p := s.resolvePricing(schemas.RoutingInfo{Provider: "vertex", Model: "google/gemini-2.0-flash"}, schemas.ChatCompletionRequest, LookupScopes{Provider: "vertex"}) assert.Equal(t, 0.0000001, derefF(p.InputCostPerToken)) } func TestGetPricing_BedrockAddsAnthropicPrefix(t *testing.T) { - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("anthropic.claude-3-5-sonnet-20241022-v2:0", "bedrock", "chat"): chatPricing(0.000003, 0.000015), }) - p := mc.resolvePricing("bedrock", "claude-3-5-sonnet-20241022-v2:0", "", schemas.ChatCompletionRequest, PricingLookupScopes{Provider: "bedrock"}) + p := s.resolvePricing(schemas.RoutingInfo{Provider: "bedrock", Model: "claude-3-5-sonnet-20241022-v2:0"}, schemas.ChatCompletionRequest, LookupScopes{Provider: "bedrock"}) assert.Equal(t, 0.000003, derefF(p.InputCostPerToken)) } func TestGetPricing_ResponsesFallsBackToChat(t *testing.T) { - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("gpt-4o", "openai", "chat"): chatPricing(0.000005, 0.000015), }) - p := mc.resolvePricing("openai", "gpt-4o", "", schemas.ResponsesRequest, PricingLookupScopes{Provider: "openai"}) + p := s.resolvePricing(schemas.RoutingInfo{Provider: "openai", Model: "gpt-4o"}, schemas.ResponsesRequest, LookupScopes{Provider: "openai"}) assert.Equal(t, 0.000005, derefF(p.InputCostPerToken)) } func TestGetPricing_ResponsesStreamFallsBackToChat(t *testing.T) { - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("gpt-4o", "openai", "chat"): chatPricing(0.000005, 0.000015), }) - p := mc.resolvePricing("openai", "gpt-4o", "", schemas.ResponsesStreamRequest, PricingLookupScopes{Provider: "openai"}) + p := s.resolvePricing(schemas.RoutingInfo{Provider: "openai", Model: "gpt-4o"}, schemas.ResponsesStreamRequest, LookupScopes{Provider: "openai"}) assert.Equal(t, 0.000005, derefF(p.InputCostPerToken)) } func TestGetPricing_RealtimeFallsBackToChat(t *testing.T) { - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("gpt-4o", "openai", "chat"): chatPricing(0.000005, 0.000015), }) - p := mc.resolvePricing("openai", "gpt-4o", "", schemas.RealtimeRequest, PricingLookupScopes{Provider: "openai"}) + p := s.resolvePricing(schemas.RoutingInfo{Provider: "openai", Model: "gpt-4o"}, schemas.RealtimeRequest, LookupScopes{Provider: "openai"}) assert.Equal(t, 0.000005, derefF(p.InputCostPerToken)) } func TestGetPricing_GeminiResponsesFallsBackToVertexChat(t *testing.T) { - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("gemini-2.0-flash", "vertex", "chat"): chatPricing(0.0000001, 0.0000004), }) // gemini provider + responses request → try vertex + responses → try vertex + chat - p := mc.resolvePricing("gemini", "gemini-2.0-flash", "", schemas.ResponsesRequest, PricingLookupScopes{Provider: "gemini"}) + p := s.resolvePricing(schemas.RoutingInfo{Provider: "gemini", Model: "gemini-2.0-flash"}, schemas.ResponsesRequest, LookupScopes{Provider: "gemini"}) assert.Equal(t, 0.0000001, derefF(p.InputCostPerToken)) } func TestGetPricing_NotFound(t *testing.T) { - mc := testCatalogWithPricing(nil) - p := mc.resolvePricing("openai", "nonexistent", "", schemas.ChatCompletionRequest, PricingLookupScopes{Provider: "openai"}) + s := testStoreWithPricing(nil) + p := s.resolvePricing(schemas.RoutingInfo{Provider: "openai", Model: "nonexistent"}, schemas.ChatCompletionRequest, LookupScopes{Provider: "openai"}) assert.Nil(t, p) } @@ -1554,32 +1631,32 @@ func TestGetPricing_NotFound(t *testing.T) { // ========================================================================= func TestResolvePricing_DeploymentFallback(t *testing.T) { - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("my-deployment", "openai", "chat"): chatPricing(0.000005, 0.000015), }) // Model not found directly, but deployment matches - p := mc.resolvePricing("openai", "gpt-4o-custom", "my-deployment", schemas.ChatCompletionRequest, PricingLookupScopes{}) + p := s.resolvePricing(schemas.RoutingInfo{Provider: "openai", Model: "gpt-4o-custom", ResolvedKeyAlias: &schemas.ResolvedKeyAlias{ModelID: "my-deployment"}}, schemas.ChatCompletionRequest, LookupScopes{}) require.NotNil(t, p) assert.Equal(t, 0.000005, derefF(p.InputCostPerToken)) } func TestResolvePricing_ResolvedModelHasPriority(t *testing.T) { - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("gpt-4o", "openai", "chat"): chatPricing(0.000005, 0.000015), makeKey("my-deployment", "openai", "chat"): chatPricing(0.000001, 0.000002), }) // Resolved model ("my-deployment") is looked up first and has priority // over the originally requested model ("gpt-4o"). - p := mc.resolvePricing("openai", "gpt-4o", "my-deployment", schemas.ChatCompletionRequest, PricingLookupScopes{}) + p := s.resolvePricing(schemas.RoutingInfo{Provider: "openai", Model: "gpt-4o", ResolvedKeyAlias: &schemas.ResolvedKeyAlias{ModelID: "my-deployment"}}, schemas.ChatCompletionRequest, LookupScopes{}) require.NotNil(t, p) assert.Equal(t, 0.000001, derefF(p.InputCostPerToken)) } func TestResolvePricing_NothingFound(t *testing.T) { - mc := testCatalogWithPricing(nil) - p := mc.resolvePricing("openai", "unknown", "", schemas.ChatCompletionRequest, PricingLookupScopes{}) + s := testStoreWithPricing(nil) + p := s.resolvePricing(schemas.RoutingInfo{Provider: "openai", Model: "unknown"}, schemas.ChatCompletionRequest, LookupScopes{}) assert.Nil(t, p) } @@ -1666,7 +1743,7 @@ func TestResponsesUsageToBifrostUsage_WithTokenDetails(t *testing.T) { func TestCalculateCost_200kTier_EndToEnd(t *testing.T) { // Claude 3.5 Sonnet Bedrock with 200k tier pricing - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("anthropic.claude-3-5-sonnet-20240620-v1:0", "bedrock", "chat"): { Model: "anthropic.claude-3-5-sonnet-20240620-v1:0", Provider: "bedrock", Mode: "chat", InputCostPerToken: bifrost.Ptr(0.000003), @@ -1686,14 +1763,14 @@ func TestCalculateCost_200kTier_EndToEnd(t *testing.T) { TotalTokens: 210000, // Above 200k }) - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) // Tiered rate: input=0.000006, output=0.00003 // 190000*0.000006 + 20000*0.00003 = 1.14 + 0.6 = 1.74 assert.InDelta(t, 1.74, cost, 1e-9) } func TestCalculateCost_272kTier_EndToEnd(t *testing.T) { - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("claude-3-7-sonnet", "anthropic", "chat"): { Model: "claude-3-7-sonnet", Provider: "anthropic", @@ -1716,7 +1793,7 @@ func TestCalculateCost_272kTier_EndToEnd(t *testing.T) { TotalTokens: 280000, // Above 272k }) - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) // Tiered rate: input=0.000009, output=0.000045 // 250000*0.000009 + 30000*0.000045 = 2.25 + 1.35 = 3.60 assert.InDelta(t, 3.60, cost, 1e-9) @@ -1724,7 +1801,7 @@ func TestCalculateCost_272kTier_EndToEnd(t *testing.T) { func TestCalculateCost_272kTier_CacheReadFallbackChain(t *testing.T) { // Verifies the 272k cache read rate takes precedence over 200k and base rates - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("claude-3-7-sonnet", "anthropic", "chat"): { Model: "claude-3-7-sonnet", Provider: "anthropic", @@ -1748,7 +1825,7 @@ func TestCalculateCost_272kTier_CacheReadFallbackChain(t *testing.T) { }, }) - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) // Non-cached input: (250000-50000) * 0.000009 = 200000 * 0.000009 = 1.80 // Cached read (272k rate): 50000 * 0.0000009 = 0.045 // Output: 30000 * 0.000045 = 1.35 @@ -1860,7 +1937,7 @@ func TestComputeTextCost_PriorityCacheReadRate(t *testing.T) { func TestCalculateCost_PriorityTier_EndToEnd(t *testing.T) { tier := schemas.BifrostServiceTierPriority - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("gpt-4o", "openai", "chat"): { Model: "gpt-4o", Provider: "openai", @@ -1882,21 +1959,19 @@ func TestCalculateCost_PriorityTier_EndToEnd(t *testing.T) { }, ExtraFields: schemas.BifrostResponseExtraFields{ RequestType: schemas.ChatCompletionRequest, - Provider: schemas.OpenAI, - OriginalModelRequested: "gpt-4o", - ResolvedModelUsed: "gpt-4o", + RoutingInfo: routingInfoFor(schemas.OpenAI, "gpt-4o"), }, }, } - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) // Priority rates: 1000*0.000010 + 500*0.000030 = 0.010 + 0.015 = 0.025 assert.InDelta(t, 0.025, cost, 1e-12) } func TestCalculateCost_NonPriorityServiceTier_UsesBaseRate(t *testing.T) { tier := schemas.BifrostServiceTierAuto - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("gpt-4o", "openai", "chat"): { Model: "gpt-4o", Provider: "openai", @@ -1918,14 +1993,12 @@ func TestCalculateCost_NonPriorityServiceTier_UsesBaseRate(t *testing.T) { }, ExtraFields: schemas.BifrostResponseExtraFields{ RequestType: schemas.ChatCompletionRequest, - Provider: schemas.OpenAI, - OriginalModelRequested: "gpt-4o", - ResolvedModelUsed: "gpt-4o", + RoutingInfo: routingInfoFor(schemas.OpenAI, "gpt-4o"), }, }, } - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) // Base rates (not priority): 1000*0.000005 + 500*0.000015 = 0.005 + 0.0075 = 0.0125 assert.InDelta(t, 0.0125, cost, 1e-12) } @@ -2009,33 +2082,33 @@ func TestTieredCacheReadRate_FallbackOrder(t *testing.T) { } // ========================================================================= -// tierFromString tests +// tierFromResponse tests // ========================================================================= -func TestTierFromString_Priority(t *testing.T) { +func TestTierFromResponse_Priority(t *testing.T) { s := schemas.BifrostServiceTierPriority - tier := tierFromString(&s) + tier := tierFromResponse(&s, nil) assert.True(t, tier.isPriority) assert.False(t, tier.isFlex) } -func TestTierFromString_Flex(t *testing.T) { +func TestTierFromResponse_Flex(t *testing.T) { s := schemas.BifrostServiceTierFlex - tier := tierFromString(&s) + tier := tierFromResponse(&s, nil) assert.False(t, tier.isPriority) assert.True(t, tier.isFlex) } -func TestTierFromString_Default(t *testing.T) { +func TestTierFromResponse_Default(t *testing.T) { for _, s := range []schemas.BifrostServiceTier{schemas.BifrostServiceTierAuto, schemas.BifrostServiceTierDefault, ""} { - tier := tierFromString(&s) + tier := tierFromResponse(&s, nil) assert.False(t, tier.isPriority, "expected no priority for %q", s) assert.False(t, tier.isFlex, "expected no flex for %q", s) } } -func TestTierFromString_Nil(t *testing.T) { - tier := tierFromString(nil) +func TestTierFromResponse_Nil(t *testing.T) { + tier := tierFromResponse(nil, nil) assert.False(t, tier.isPriority) assert.False(t, tier.isFlex) } @@ -2141,7 +2214,7 @@ func TestComputeTextCost_FlexFallsBackToBaseWhenNoFlexRate(t *testing.T) { func TestCalculateCost_FlexTier_EndToEnd(t *testing.T) { tier := schemas.BifrostServiceTierFlex - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("gpt-4o", "openai", "chat"): { Model: "gpt-4o", Provider: "openai", @@ -2163,21 +2236,19 @@ func TestCalculateCost_FlexTier_EndToEnd(t *testing.T) { }, ExtraFields: schemas.BifrostResponseExtraFields{ RequestType: schemas.ChatCompletionRequest, - Provider: schemas.OpenAI, - OriginalModelRequested: "gpt-4o", - ResolvedModelUsed: "gpt-4o", + RoutingInfo: routingInfoFor(schemas.OpenAI, "gpt-4o"), }, }, } - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) // Flex rates: 1000*0.0000025 + 500*0.0000075 = 0.0025 + 0.00375 = 0.00625 assert.InDelta(t, 0.00625, cost, 1e-12) } func TestCalculateCost_FlexTier_FallsBackToBaseWhenNoFlexRate(t *testing.T) { tier := schemas.BifrostServiceTierFlex - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("gpt-4o", "openai", "chat"): chatPricing(0.000005, 0.000015), }) @@ -2191,20 +2262,18 @@ func TestCalculateCost_FlexTier_FallsBackToBaseWhenNoFlexRate(t *testing.T) { }, ExtraFields: schemas.BifrostResponseExtraFields{ RequestType: schemas.ChatCompletionRequest, - Provider: schemas.OpenAI, - OriginalModelRequested: "gpt-4o", - ResolvedModelUsed: "gpt-4o", + RoutingInfo: routingInfoFor(schemas.OpenAI, "gpt-4o"), }, }, } - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) // No flex rates configured — falls back to base: 1000*0.000005 + 500*0.000015 = 0.005 + 0.0075 = 0.0125 assert.InDelta(t, 0.0125, cost, 1e-12) } func TestCalculateCost_ProviderCostZeroTotalStillCalculates(t *testing.T) { - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("gpt-4o", "openai", "chat"): chatPricing(0.000005, 0.000015), }) @@ -2218,7 +2287,7 @@ func TestCalculateCost_ProviderCostZeroTotalStillCalculates(t *testing.T) { }, }) - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) assert.InDelta(t, 0.0125, cost, 1e-12) } @@ -2255,12 +2324,12 @@ func TestCalculateCost_ImageGeneration_NilUsage_PerImagePricing(t *testing.T) { OutputCostPerImage: bifrost.Ptr(0.04), } - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("dall-e-3", "openai", "image_generation"): pricing, }) resp := makeImageResponse("openai", "dall-e-3", nil) - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) // 1 image * $0.04 = $0.04 assert.InDelta(t, 0.04, cost, 1e-12) } @@ -2275,12 +2344,12 @@ func TestCalculateCost_ImageGeneration_NilUsage_InputAndOutputPerImage(t *testin OutputCostPerImage: bifrost.Ptr(0.04), } - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("test-image-model", "test", "image_generation"): pricing, }) resp := makeImageResponse("test", "test-image-model", nil) - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) // NumInputImages is 0 (not populated from request), so only output pricing applies // 1 output image * $0.04 = $0.04 assert.InDelta(t, 0.04, cost, 1e-12) @@ -2296,14 +2365,14 @@ func TestCalculateCost_ImageGeneration_WithInputImages(t *testing.T) { OutputCostPerImage: bifrost.Ptr(0.04), } - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("gpt-image-1", "openai", "image_generation"): pricing, }) resp := makeImageResponse("openai", "gpt-image-1", &schemas.ImageUsage{ NumInputImages: 2, }) - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) // 2 input images * $0.01 + 1 output image * $0.04 = $0.06 assert.InDelta(t, 0.06, cost, 1e-12) } @@ -2317,7 +2386,7 @@ func TestCalculateCost_ImageGeneration_OutputCountFromData(t *testing.T) { OutputCostPerImage: bifrost.Ptr(0.04), } - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("dall-e-3", "openai", "image_generation"): pricing, }) @@ -2329,13 +2398,12 @@ func TestCalculateCost_ImageGeneration_OutputCountFromData(t *testing.T) { {URL: "https://example.com/img3.png", Index: 2}, }, ExtraFields: schemas.BifrostResponseExtraFields{ - RequestType: schemas.ImageGenerationRequest, - Provider: "openai", - OriginalModelRequested: "dall-e-3", + RequestType: schemas.ImageGenerationRequest, + RoutingInfo: routingInfoFor("openai", "dall-e-3"), }, }, } - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) // 3 output images * $0.04 = $0.12 assert.InDelta(t, 0.12, cost, 1e-12) } @@ -2350,12 +2418,12 @@ func TestCalculateCost_ImageGeneration_NilUsage_NoPerImagePricing(t *testing.T) OutputCostPerToken: bifrost.Ptr(0.000002), } - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("token-only-model", "test", "image_generation"): pricing, }) resp := makeImageResponse("test", "token-only-model", nil) - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) // No per-image pricing and all tokens are zero → 0 assert.InDelta(t, 0.0, cost, 1e-12) } @@ -2369,12 +2437,12 @@ func TestCalculateCost_ImageGeneration_EmptyUsage_PerImagePricing(t *testing.T) OutputCostPerImage: bifrost.Ptr(0.04), } - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("dall-e-3", "openai", "image_generation"): pricing, }) resp := makeImageResponse("openai", "dall-e-3", &schemas.ImageUsage{}) - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) assert.InDelta(t, 0.04, cost, 1e-12) } @@ -2433,7 +2501,7 @@ func TestComputeImageCost_BothHaveTokens_IgnoresPerImage(t *testing.T) { } func TestCalculateCost_ResponsesWithCodeInterpreter(t *testing.T) { - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("gpt-4.1", "openai", "chat"): chatPricing(0.000002, 0.000008), }) @@ -2451,14 +2519,12 @@ func TestCalculateCost_ResponsesWithCodeInterpreter(t *testing.T) { }, ExtraFields: schemas.BifrostResponseExtraFields{ RequestType: schemas.ResponsesRequest, - Provider: schemas.OpenAI, - OriginalModelRequested: "gpt-4.1", - ResolvedModelUsed: "gpt-4.1", + RoutingInfo: routingInfoFor(schemas.OpenAI, "gpt-4.1"), }, }, } - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) // Token cost only: 579*0.000002 + 334*0.000008 = 0.001158 + 0.002672 = 0.003830 // Session cost is now tracked via ContainerCreateRequest, not per-response assert.InDelta(t, 0.003830, cost, 1e-6) @@ -2496,7 +2562,7 @@ func TestComputeContainerCreationCost_NilRate(t *testing.T) { // --------------------------------------------------------------------------- func TestCalculateCost_ContainerCreate_NoMemoryLimit(t *testing.T) { - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("container", "openai", "chat"): { Model: "container", Provider: "openai", @@ -2511,17 +2577,17 @@ func TestCalculateCost_ContainerCreate_NoMemoryLimit(t *testing.T) { Name: "test-container", ExtraFields: schemas.BifrostResponseExtraFields{ RequestType: schemas.ContainerCreateRequest, - Provider: schemas.OpenAI, + RoutingInfo: schemas.RoutingInfo{Provider: schemas.OpenAI}, }, }, } - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) assert.InDelta(t, 0.03, cost, 1e-12) } func TestCalculateCost_ContainerCreate_MemorySpecificEntry(t *testing.T) { - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("container", "openai", "chat"): { Model: "container", Provider: "openai", @@ -2543,17 +2609,17 @@ func TestCalculateCost_ContainerCreate_MemorySpecificEntry(t *testing.T) { MemoryLimit: "4g", ExtraFields: schemas.BifrostResponseExtraFields{ RequestType: schemas.ContainerCreateRequest, - Provider: schemas.OpenAI, + RoutingInfo: schemas.RoutingInfo{Provider: schemas.OpenAI}, }, }, } - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) assert.InDelta(t, 0.12, cost, 1e-12) } func TestCalculateCost_ContainerCreate_FallsBackToBaseEntry(t *testing.T) { - mc := testCatalogWithPricing(map[string]configstoreTables.TableModelPricing{ + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ makeKey("container", "openai", "chat"): { Model: "container", Provider: "openai", @@ -2569,18 +2635,18 @@ func TestCalculateCost_ContainerCreate_FallsBackToBaseEntry(t *testing.T) { MemoryLimit: "4g", ExtraFields: schemas.BifrostResponseExtraFields{ RequestType: schemas.ContainerCreateRequest, - Provider: schemas.OpenAI, + RoutingInfo: schemas.RoutingInfo{Provider: schemas.OpenAI}, }, }, } // No container-4g entry — should fall back to base "container" rate - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) assert.InDelta(t, 0.03, cost, 1e-12) } -func TestCalculateCost_ContainerCreate_NoPricingEntry(t *testing.T) { - mc := testCatalogWithPricing(nil) +func TestCalculateCost_ContainerCreate_NoEntry(t *testing.T) { + s := testStoreWithPricing(nil) resp := &schemas.BifrostResponse{ ContainerCreateResponse: &schemas.BifrostContainerCreateResponse{ @@ -2588,82 +2654,148 @@ func TestCalculateCost_ContainerCreate_NoPricingEntry(t *testing.T) { Name: "test-container", ExtraFields: schemas.BifrostResponseExtraFields{ RequestType: schemas.ContainerCreateRequest, - Provider: schemas.OpenAI, + RoutingInfo: schemas.RoutingInfo{Provider: schemas.OpenAI}, }, }, } - cost := mc.CalculateCost(resp, nil) + cost := s.CalculateCost(resp, nil) assert.Equal(t, 0.0, cost) } // --------------------------------------------------------------------------- -// file:// URL loading tests +// Backward-compat: RoutingInfo missing → synthesize from deprecated triplet +// +// Covers callers stuck on the legacy ExtraFields shape: +// - LoggerPlugin.RecalculateCosts replaying logs written before RoutingInfo existed +// - Third-party plugins / SDK users that haven't migrated to RoutingInfo +// +// The fallback only fires when RoutingInfo is fully empty (zero Provider, +// zero Model, nil ResolvedKeyAlias). Any partial population is trusted. // --------------------------------------------------------------------------- -func TestLoadPricingFromURL_FileScheme(t *testing.T) { - pricingData := map[string]PricingEntry{ - "gpt-4o": { - Provider: "openai", - Mode: "chat", +func TestCalculateCost_BackCompat_LegacyFieldsOnly_NoAlias(t *testing.T) { + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ + makeKey("gpt-4o", "openai", "chat"): chatPricing(0.000005, 0.000015), + }) + + // Caller populates only the deprecated triplet — no RoutingInfo. + // Pricing should fall back to Provider + OriginalModelRequested. + resp := &schemas.BifrostResponse{ + ChatResponse: &schemas.BifrostChatResponse{ + Usage: &schemas.BifrostLLMUsage{PromptTokens: 1000, CompletionTokens: 500, TotalTokens: 1500}, + ExtraFields: schemas.BifrostResponseExtraFields{ + RequestType: schemas.ChatCompletionRequest, + Provider: schemas.OpenAI, + OriginalModelRequested: "gpt-4o", + }, }, } - data, err := json.Marshal(pricingData) - require.NoError(t, err) - - f, err := os.CreateTemp(t.TempDir(), "pricing-*.json") - require.NoError(t, err) - _, err = f.Write(data) - require.NoError(t, err) - f.Close() - mc := newTestCatalog(nil, nil) - mc.logger = noOpLogger{} - mc.pricingURL = "file://" + f.Name() - - result, err := mc.loadPricingFromURL(context.Background()) - require.NoError(t, err) - require.Len(t, result, 1) - assert.Equal(t, "openai", result["gpt-4o"].Provider) + cost := s.CalculateCost(resp, nil) + // 1000 * 0.000005 + 500 * 0.000015 = 0.005 + 0.0075 = 0.0125 + assert.InDelta(t, 0.0125, cost, 1e-12) } -func TestLoadPricingFromURL_FileMissing(t *testing.T) { - mc := newTestCatalog(nil, nil) - mc.logger = noOpLogger{} - mc.pricingURL = "file:///nonexistent/path/pricing.json" +func TestCalculateCost_BackCompat_LegacyFieldsOnly_WithAlias(t *testing.T) { + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ + makeKey("my-deployment", "openai", "chat"): chatPricing(0.000005, 0.000015), + }) + + // Caller populates only the deprecated triplet with a distinct + // ResolvedModelUsed (i.e. an alias was matched at original request time + // and the wire model differs from the caller-facing name). The fallback + // should route ResolvedModelUsed into ResolvedKeyAlias.ModelID so the + // catalog lookup hits the deployment-keyed entry. + resp := &schemas.BifrostResponse{ + ChatResponse: &schemas.BifrostChatResponse{ + Usage: &schemas.BifrostLLMUsage{PromptTokens: 1000, CompletionTokens: 500, TotalTokens: 1500}, + ExtraFields: schemas.BifrostResponseExtraFields{ + RequestType: schemas.ChatCompletionRequest, + Provider: schemas.OpenAI, + OriginalModelRequested: "my-alias-name", + ResolvedModelUsed: "my-deployment", + }, + }, + } - _, err := mc.loadPricingFromURL(context.Background()) - require.Error(t, err) + cost := s.CalculateCost(resp, nil) + // 1000 * 0.000005 + 500 * 0.000015 = 0.0125, charged via the deployment-keyed entry + assert.InDelta(t, 0.0125, cost, 1e-12) } -func TestLoadModelParametersFromURL_FileScheme(t *testing.T) { - paramsData := map[string]json.RawMessage{ - "gpt-4o": json.RawMessage(`{"max_output_tokens":4096}`), +func TestCalculateCost_BackCompat_RoutingInfoWinsOverLegacyFields(t *testing.T) { + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ + makeKey("gpt-4o", "openai", "chat"): chatPricing(0.000005, 0.000015), + makeKey("gemini-2.0-flash", "gemini", "chat"): { + Model: "gemini-2.0-flash", + Provider: "gemini", + Mode: "chat", + InputCostPerToken: bifrost.Ptr(0.0000001), + OutputCostPerToken: bifrost.Ptr(0.0000004), + }, + }) + + // Both populated. The modern fields (RoutingInfo) must win — the + // fallback only fires when RoutingInfo is fully unset. + resp := &schemas.BifrostResponse{ + ChatResponse: &schemas.BifrostChatResponse{ + Usage: &schemas.BifrostLLMUsage{PromptTokens: 1000, CompletionTokens: 500, TotalTokens: 1500}, + ExtraFields: schemas.BifrostResponseExtraFields{ + RequestType: schemas.ChatCompletionRequest, + RoutingInfo: routingInfoFor(schemas.OpenAI, "gpt-4o"), + Provider: schemas.Gemini, + OriginalModelRequested: "gemini-2.0-flash", + }, + }, } - data, err := json.Marshal(paramsData) - require.NoError(t, err) - f, err := os.CreateTemp(t.TempDir(), "model-parameters-*.json") - require.NoError(t, err) - _, err = f.Write(data) - require.NoError(t, err) - f.Close() + cost := s.CalculateCost(resp, nil) + // Priced via RoutingInfo → openai/gpt-4o → 0.0125 (not the gemini rate). + assert.InDelta(t, 0.0125, cost, 1e-12) +} - mc := newTestCatalog(nil, nil) - mc.logger = noOpLogger{} - mc.modelParametersURL = "file://" + f.Name() +func TestCalculateCost_BackCompat_BothEmpty_ReturnsZero(t *testing.T) { + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ + makeKey("gpt-4o", "openai", "chat"): chatPricing(0.000005, 0.000015), + }) - result, err := mc.loadModelParametersFromURL(context.Background()) - require.NoError(t, err) - require.Len(t, result, 1) - assert.JSONEq(t, `{"max_output_tokens":4096}`, string(result["gpt-4o"])) + // Neither RoutingInfo nor the deprecated triplet are populated. + // Pricing has no way to identify the model; cost is 0. + resp := &schemas.BifrostResponse{ + ChatResponse: &schemas.BifrostChatResponse{ + Usage: &schemas.BifrostLLMUsage{PromptTokens: 1000, CompletionTokens: 500, TotalTokens: 1500}, + ExtraFields: schemas.BifrostResponseExtraFields{ + RequestType: schemas.ChatCompletionRequest, + }, + }, + } + + cost := s.CalculateCost(resp, nil) + assert.Equal(t, 0.0, cost) } -func TestLoadModelParametersFromURL_FileMissing(t *testing.T) { - mc := newTestCatalog(nil, nil) - mc.logger = noOpLogger{} - mc.modelParametersURL = "file:///nonexistent/path/model-parameters.json" +func TestCalculateCost_BackCompat_PartialRoutingInfo_NoFallback(t *testing.T) { + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ + makeKey("gpt-4o", "openai", "chat"): chatPricing(0.000005, 0.000015), + }) + + // RoutingInfo has Model but no Provider. The legacy Provider field is + // also set. The fallback MUST NOT fire — partial RoutingInfo means the + // caller intended to use RoutingInfo. With Provider unset on RoutingInfo, + // the catalog lookup fails and cost is 0. (This guards the trigger + // against false positives.) + resp := &schemas.BifrostResponse{ + ChatResponse: &schemas.BifrostChatResponse{ + Usage: &schemas.BifrostLLMUsage{PromptTokens: 1000, CompletionTokens: 500, TotalTokens: 1500}, + ExtraFields: schemas.BifrostResponseExtraFields{ + RequestType: schemas.ChatCompletionRequest, + RoutingInfo: schemas.RoutingInfo{Model: "gpt-4o"}, + Provider: schemas.OpenAI, + }, + }, + } - _, err := mc.loadModelParametersFromURL(context.Background()) - require.Error(t, err) + cost := s.CalculateCost(resp, nil) + assert.Equal(t, 0.0, cost) } diff --git a/framework/modelcatalog/pricing_overrides.go b/framework/modelcatalog/datasheet/overrides.go similarity index 59% rename from framework/modelcatalog/pricing_overrides.go rename to framework/modelcatalog/datasheet/overrides.go index baecf51347..67c335e4b7 100644 --- a/framework/modelcatalog/pricing_overrides.go +++ b/framework/modelcatalog/datasheet/overrides.go @@ -1,4 +1,4 @@ -package modelcatalog +package datasheet import ( "context" @@ -11,95 +11,8 @@ import ( configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" ) -// PricingLookupScopes carries the runtime identifiers used to resolve scoped -// pricing overrides during cost calculation. -type PricingLookupScopes struct { - VirtualKeyID string - SelectedKeyID string - Provider string -} - -// PricingLookupScopesFromContext builds a PricingLookupScopes from a BifrostContext. -// It reads the governance virtual key ID (not the raw VK token) and the selected key ID. -// provider should be the provider name string (e.g. "openai"), pass "" if unavailable. -// Returns nil only when ctx is nil. An empty scopes value is still returned when all fields -// are empty so that global-scope overrides are always evaluated. -// DO NOT USE THIS FUNCTION IN A GO ROUTINE. This is because it reads from ctx which is cancelled when the request ends. -// Better to call it in PostHooks synchronously and then pass the scopes object to the pricing manager. -// Only use this in go routines when you know for sure that the request will not end before the go routine completes. -func PricingLookupScopesFromContext(ctx *schemas.BifrostContext, provider string) *PricingLookupScopes { - if ctx == nil { - return nil - } - virtualKeyID, _ := ctx.Value(schemas.BifrostContextKeyGovernanceVirtualKeyID).(string) - selectedKeyID, _ := ctx.Value(schemas.BifrostContextKeySelectedKeyID).(string) - return &PricingLookupScopes{ - VirtualKeyID: virtualKeyID, - SelectedKeyID: selectedKeyID, - Provider: provider, - } -} - -// ScopeKind identifies which governance scope an override applies to. -type ScopeKind string - -const ( - ScopeKindGlobal ScopeKind = "global" - ScopeKindProvider ScopeKind = "provider" - ScopeKindProviderKey ScopeKind = "provider_key" - ScopeKindVirtualKey ScopeKind = "virtual_key" - ScopeKindVirtualKeyProvider ScopeKind = "virtual_key_provider" - ScopeKindVirtualKeyProviderKey ScopeKind = "virtual_key_provider_key" -) - -// MatchType controls how an override pattern is matched against model names. -type MatchType string - -const ( - MatchTypeExact MatchType = "exact" - MatchTypeWildcard MatchType = "wildcard" -) - -// PricingOverride describes a scoped pricing override shared across config storage, -// model catalog compilation, and governance APIs. -type PricingOverride struct { - ID string `json:"id"` - Name string `json:"name"` - ScopeKind ScopeKind `json:"scope_kind"` - VirtualKeyID *string `json:"virtual_key_id,omitempty"` - ProviderID *string `json:"provider_id,omitempty"` - ProviderKeyID *string `json:"provider_key_id,omitempty"` - MatchType MatchType `json:"match_type"` - Pattern string `json:"pattern"` - RequestTypes []schemas.RequestType `json:"request_types,omitempty"` - Options PricingOptions `json:"options"` -} - -// customPricingEntry is a single flattened override ready for lookup. -type customPricingEntry struct { - id string - scopeKind ScopeKind - virtualKeyID string - providerID string - providerKeyID string - pattern string // exact model name, or wildcard prefix (trailing * stripped) - wildcard bool - requestModes map[string]struct{} // always non-nil for valid overrides - options PricingOptions -} - -// customPricingData is the in-memory lookup structure for pricing overrides. -// Exact matches are indexed by model name; wildcards are a flat slice. -type customPricingData struct { - exact map[string][]customPricingEntry - wildcard []customPricingEntry -} - -// IsValid validates the shared pricing override contract before persistence or runtime use. -// -// Input: override — the PricingOverride to validate (receiver). -// Output: error — non-nil if any scope, pattern, or request-type constraint is violated. -func (override *PricingOverride) IsValid() error { +// IsValid validates the shared override contract before persistence or runtime use. +func (override *Override) IsValid() error { if err := override.validateScopeKind(); err != nil { return err } @@ -109,11 +22,7 @@ func (override *PricingOverride) IsValid() error { return override.validateRequestTypes() } -// validateScopeKind validates the scope identifiers required by override.ScopeKind. -// -// Input: override — receiver; ScopeKind and the three optional ID fields are inspected. -// Output: error — non-nil when required identifiers are absent or forbidden ones are present. -func (override *PricingOverride) validateScopeKind() error { +func (override *Override) validateScopeKind() error { switch override.ScopeKind { case ScopeKindGlobal: if override.VirtualKeyID != nil || override.ProviderID != nil || override.ProviderKeyID != nil { @@ -157,13 +66,7 @@ func (override *PricingOverride) validateScopeKind() error { return nil } -// validatePattern checks that Pattern is non-empty and consistent with MatchType. -// -// Input: override — receiver; Pattern and MatchType are inspected. -// Output: error — non-nil when the pattern is empty, contains a wildcard for exact mode, -// -// or does not end with a single trailing "*" for wildcard mode. -func (override *PricingOverride) validatePattern() error { +func (override *Override) validatePattern() error { pattern := strings.TrimSpace(override.Pattern) if pattern == "" { return fmt.Errorf("pattern is required") @@ -186,13 +89,10 @@ func (override *PricingOverride) validatePattern() error { return nil } -// validateRequestTypes checks that RequestTypes is non-empty and that every entry is a -// supported base request type. Stream variants (e.g. chat_completion_stream) are rejected — -// the base type (chat_completion) already covers both streaming and non-streaming requests. -// -// Input: override — receiver; RequestTypes slice is inspected. -// Output: error — non-nil if RequestTypes is empty, or contains an unsupported or stream variant. -func (override *PricingOverride) validateRequestTypes() error { +// validateRequestTypes checks that RequestTypes is non-empty and that every +// entry is a supported base request type. Stream variants are rejected — the +// base type already covers both streaming and non-streaming requests. +func (override *Override) validateRequestTypes() error { if len(override.RequestTypes) == 0 { return fmt.Errorf("request_types is required and must contain at least one value") } @@ -207,11 +107,9 @@ func (override *PricingOverride) validateRequestTypes() error { return nil } -// matchesScope reports whether the entry's governance scope matches the runtime identifiers. -// -// Input: scopes — runtime VirtualKeyID, SelectedKeyID, and Provider to match against. -// Output: bool — true when the entry's scope kind and stored IDs align with scopes. -func (e *customPricingEntry) matchesScope(scopes PricingLookupScopes) bool { +// matchesScope reports whether the entry's governance scope matches the +// runtime identifiers. +func (e *customPricingEntry) matchesScope(scopes LookupScopes) bool { switch e.scopeKind { case ScopeKindGlobal: return true @@ -229,25 +127,14 @@ func (e *customPricingEntry) matchesScope(scopes PricingLookupScopes) bool { return false } -// matchesMode reports whether the entry applies to the given normalized request mode. -// -// Input: mode — normalized request type string (e.g. "chat", "embedding"). -// Output: bool — true when requestModes contains mode. func (e *customPricingEntry) matchesMode(mode string) bool { _, ok := e.requestModes[mode] return ok } // resolve walks the 6-scope priority hierarchy and returns the first matching -// pricing patch for the given model, request mode, and runtime scopes. -// -// Input: model — exact model name being priced. -// -// mode — normalized request type string (e.g. "chat", "embedding"). -// scopes — runtime governance identifiers used to narrow the scope search. -// -// Output: *PricingOptions — pointer to the first matching override's options, or nil if none match. -func (c *customPricingData) resolve(model, mode string, scopes PricingLookupScopes) *PricingOptions { +// pricing patch for the given model, mode, and runtime scopes. +func (c *customPricingData) resolve(model, mode string, scopes LookupScopes) *Options { for _, scopeKind := range scopePriorityOrder(scopes) { for i := range c.exact[model] { e := &c.exact[model][i] @@ -267,10 +154,7 @@ func (c *customPricingData) resolve(model, mode string, scopes PricingLookupScop // scopePriorityOrder returns scope kinds in most-specific-first order, // skipping scopes that can't match given the available runtime identifiers. -// -// Input: scopes — runtime governance identifiers; empty fields cause the corresponding scope kinds to be omitted. -// Output: []ScopeKind — ordered list from most-specific (VirtualKeyProviderKey) to least-specific (Global). -func scopePriorityOrder(scopes PricingLookupScopes) []ScopeKind { +func scopePriorityOrder(scopes LookupScopes) []ScopeKind { order := make([]ScopeKind, 0, 6) if scopes.VirtualKeyID != "" && scopes.Provider != "" && scopes.SelectedKeyID != "" { order = append(order, ScopeKindVirtualKeyProviderKey) @@ -291,11 +175,10 @@ func scopePriorityOrder(scopes PricingLookupScopes) []ScopeKind { return order } -// buildCustomPricingData constructs a customPricingData lookup structure from a raw override slice. -// -// Input: overrides — slice of validated PricingOverride records loaded from the config store. -// Output: *customPricingData — ready-to-query structure with exact and wildcard indexes populated. -func buildCustomPricingData(overrides []PricingOverride) *customPricingData { +// buildCustomPricingData constructs the lookup structure from a raw override +// slice. Wildcards are sorted longest-prefix-first so more specific patterns +// (e.g. "gpt-4*") win over broader ones ("gpt-*") deterministically. +func buildCustomPricingData(overrides []Override) *customPricingData { data := &customPricingData{ exact: make(map[string][]customPricingEntry, len(overrides)), } @@ -329,60 +212,38 @@ func buildCustomPricingData(overrides []PricingOverride) *customPricingData { data.wildcard = append(data.wildcard, entry) } } - // Sort wildcards by descending prefix length so more-specific patterns (e.g. "gpt-4*") - // are checked before broader ones (e.g. "gpt-*"), making precedence deterministic. sort.Slice(data.wildcard, func(i, j int) bool { return len(data.wildcard[i].pattern) > len(data.wildcard[j].pattern) }) return data } -// applyPricingOverrides resolves any active scoped pricing override for the given model -// and request type, then patches the catalog base pricing with the override values. -// It returns the original pricing unchanged when no custom pricing tree is loaded or -// when the request type cannot be mapped to a known pricing mode. -// -// Input: model — exact model name being priced. -// -// requestType — the request type used to derive the pricing mode. -// pricing — base pricing row from the catalog to patch. -// scopes — runtime governance identifiers used to narrow the override scope. -// -// Output: TableModelPricing — patched pricing row, or pricing unchanged if no override matches. -// bool — true when an override was applied, false otherwise. -func (mc *ModelCatalog) applyPricingOverrides(model string, requestType schemas.RequestType, pricing configstoreTables.TableModelPricing, scopes PricingLookupScopes) (configstoreTables.TableModelPricing, bool) { - mc.overridesMu.RLock() - custom := mc.customPricing - mc.overridesMu.RUnlock() +// applyPricingOverrides resolves any active scoped override for (model, +// requestType) and patches the catalog base pricing. Returns the original +// pricing unchanged when no override matches or the request type can't be +// mapped to a known pricing mode. +func (s *Store) applyPricingOverrides(model string, requestType schemas.RequestType, pricing configstoreTables.TableModelPricing, scopes LookupScopes) (configstoreTables.TableModelPricing, bool) { + s.overridesMu.RLock() + custom := s.customPricing + s.overridesMu.RUnlock() if custom == nil { return pricing, false } - mode := normalizeRequestType(requestType) if mode == "unknown" { return pricing, false } - if patch := custom.resolve(model, mode, scopes); patch != nil { return patchPricing(pricing, *patch), true } return pricing, false } -// patchPricing applies override values onto a copy of the base pricing row. -// For all fields, a non-nil override pointer replaces the corresponding destination value; -// a nil override leaves the base value intact. -// The original pricing row is never modified; a patched copy is always returned. -// -// Input: pricing — base pricing row from the catalog. -// -// override — pricing options sourced from the matched override entry. -// -// Output: TableModelPricing — shallow copy of pricing with override fields applied. -func patchPricing(pricing configstoreTables.TableModelPricing, override PricingOptions) configstoreTables.TableModelPricing { +// patchPricing returns a copy of pricing with override fields applied. Nil +// fields in override leave the corresponding base values intact. +func patchPricing(pricing configstoreTables.TableModelPricing, override Options) configstoreTables.TableModelPricing { patched := pricing - for _, field := range []struct { dst **float64 src *float64 @@ -393,6 +254,8 @@ func patchPricing(pricing configstoreTables.TableModelPricing, override PricingO {dst: &patched.OutputCostPerTokenPriority, src: override.OutputCostPerTokenPriority}, {dst: &patched.InputCostPerTokenFlex, src: override.InputCostPerTokenFlex}, {dst: &patched.OutputCostPerTokenFlex, src: override.OutputCostPerTokenFlex}, + {dst: &patched.InputCostPerTokenFast, src: override.InputCostPerTokenFast}, + {dst: &patched.OutputCostPerTokenFast, src: override.OutputCostPerTokenFast}, {dst: &patched.InputCostPerVideoPerSecond, src: override.InputCostPerVideoPerSecond}, {dst: &patched.OutputCostPerVideoPerSecond, src: override.OutputCostPerVideoPerSecond}, {dst: &patched.OutputCostPerSecond, src: override.OutputCostPerSecond}, @@ -458,13 +321,86 @@ func patchPricing(pricing configstoreTables.TableModelPricing, override PricingO return patched } -func (mc *ModelCatalog) loadPricingOverridesFromStore(ctx context.Context) error { - if mc.configStore == nil { +// LoadOverridesFromStore reloads all overrides from the config store. Called +// at bootstrap and after force-reload paths. +func (s *Store) LoadOverridesFromStore(ctx context.Context) error { + if s.configStore == nil { return nil } - rows, err := mc.configStore.GetPricingOverrides(ctx, configstore.PricingOverrideFilters{}) + rows, err := s.configStore.GetPricingOverrides(ctx, configstore.PricingOverrideFilters{}) if err != nil { return err } - return mc.SetPricingOverrides(rows) + return s.SetOverrides(rows) +} + +// SetOverrides replaces the full in-memory override set. Duplicate IDs in +// the input keep the last-seen entry (matching today's behavior). +func (s *Store) SetOverrides(rows []configstoreTables.TablePricingOverride) error { + seen := make(map[string]int, len(rows)) + overrides := make([]Override, 0, len(rows)) + for i := range rows { + o, err := convertTableOverride(&rows[i]) + if err != nil { + return err + } + if idx, exists := seen[o.ID]; exists { + overrides[idx] = o + } else { + seen[o.ID] = len(overrides) + overrides = append(overrides, o) + } + } + s.overridesMu.Lock() + s.rawOverrides = overrides + s.customPricing = buildCustomPricingData(overrides) + s.overridesMu.Unlock() + return nil +} + +// UpsertOverrides inserts or replaces one or more overrides, rebuilding the +// lookup map exactly once at the end. +func (s *Store) UpsertOverrides(rows ...*configstoreTables.TablePricingOverride) error { + seenIncoming := make(map[string]int, len(rows)) + overrides := make([]Override, 0, len(rows)) + for _, row := range rows { + o, err := convertTableOverride(row) + if err != nil { + return err + } + if idx, exists := seenIncoming[o.ID]; exists { + overrides[idx] = o + } else { + seenIncoming[o.ID] = len(overrides) + overrides = append(overrides, o) + } + } + + s.overridesMu.Lock() + defer s.overridesMu.Unlock() + + updated := make([]Override, 0, len(s.rawOverrides)+len(overrides)) + for _, o := range s.rawOverrides { + if _, replacing := seenIncoming[o.ID]; !replacing { + updated = append(updated, o) + } + } + updated = append(updated, overrides...) + s.rawOverrides = updated + s.customPricing = buildCustomPricingData(updated) + return nil +} + +// DeleteOverride removes an override by ID. +func (s *Store) DeleteOverride(id string) { + s.overridesMu.Lock() + defer s.overridesMu.Unlock() + updated := make([]Override, 0, len(s.rawOverrides)) + for _, o := range s.rawOverrides { + if o.ID != id { + updated = append(updated, o) + } + } + s.rawOverrides = updated + s.customPricing = buildCustomPricingData(updated) } diff --git a/framework/modelcatalog/pricing_overrides_test.go b/framework/modelcatalog/datasheet/overrides_test.go similarity index 73% rename from framework/modelcatalog/pricing_overrides_test.go rename to framework/modelcatalog/datasheet/overrides_test.go index 8593aad89a..79a1c8d819 100644 --- a/framework/modelcatalog/pricing_overrides_test.go +++ b/framework/modelcatalog/datasheet/overrides_test.go @@ -1,4 +1,4 @@ -package modelcatalog +package datasheet import ( "testing" @@ -23,10 +23,22 @@ func (noOpLogger) LogHTTPRequest(schemas.LogLevel, string) schemas.LogEventBuild return schemas.NoopLogEvent } +// newTestStore builds a minimal Store for unit tests. Callers seed pricingData +// directly and use SetOverrides for overrides. +func newTestStore() *Store { + return &Store{ + logger: noOpLogger{}, + pricingData: map[string]configstoreTables.TableModelPricing{}, + baseModelIndex: map[string]string{}, + supportedResponseTypes: map[string][]string{}, + supportedParams: map[string][]string{}, + datasheetByProvider: map[schemas.ModelProvider][]string{}, + } +} + func TestGetPricing_OverridePrecedenceExactWildcard(t *testing.T) { - mc := newTestCatalog(nil, nil) - mc.logger = noOpLogger{} - mc.pricingData[makeKey("gpt-4o", "openai", "chat")] = configstoreTables.TableModelPricing{ + s := newTestStore() + s.pricingData[makeKey("gpt-4o", "openai", "chat")] = configstoreTables.TableModelPricing{ Model: "gpt-4o", Provider: "openai", Mode: "chat", @@ -35,7 +47,7 @@ func TestGetPricing_OverridePrecedenceExactWildcard(t *testing.T) { } providerID := "openai" - require.NoError(t, mc.SetPricingOverrides([]configstoreTables.TablePricingOverride{ + require.NoError(t, s.SetOverrides([]configstoreTables.TablePricingOverride{ { ID: "openai-override-0", ScopeKind: string(ScopeKindProvider), @@ -56,7 +68,7 @@ func TestGetPricing_OverridePrecedenceExactWildcard(t *testing.T) { }, })) - pricing := mc.resolvePricing("openai", "gpt-4o", "", schemas.ChatCompletionRequest, PricingLookupScopes{Provider: "openai"}) + pricing := s.resolvePricing(schemas.RoutingInfo{Provider: "openai", Model: "gpt-4o"}, schemas.ChatCompletionRequest, LookupScopes{Provider: "openai"}) require.NotNil(t, pricing) require.NotNil(t, pricing.InputCostPerToken) assert.Equal(t, 20.0, *pricing.InputCostPerToken) @@ -64,9 +76,8 @@ func TestGetPricing_OverridePrecedenceExactWildcard(t *testing.T) { func TestGetPricing_RequestTypeSpecificOverrideBeatsGeneric(t *testing.T) { t.Skip() - mc := newTestCatalog(nil, nil) - mc.logger = noOpLogger{} - mc.pricingData[makeKey("gpt-4o", "openai", "responses")] = configstoreTables.TableModelPricing{ + s := newTestStore() + s.pricingData[makeKey("gpt-4o", "openai", "responses")] = configstoreTables.TableModelPricing{ Model: "gpt-4o", Provider: "openai", Mode: "responses", @@ -75,7 +86,7 @@ func TestGetPricing_RequestTypeSpecificOverrideBeatsGeneric(t *testing.T) { } providerID := "openai" - require.NoError(t, mc.SetPricingOverrides([]configstoreTables.TablePricingOverride{ + require.NoError(t, s.SetOverrides([]configstoreTables.TablePricingOverride{ { ID: "openai-generic", ScopeKind: string(ScopeKindProvider), @@ -95,16 +106,15 @@ func TestGetPricing_RequestTypeSpecificOverrideBeatsGeneric(t *testing.T) { }, })) - pricing := mc.resolvePricing("openai", "gpt-4o", "", schemas.ResponsesRequest, PricingLookupScopes{Provider: "openai"}) + pricing := s.resolvePricing(schemas.RoutingInfo{Provider: "openai", Model: "gpt-4o"}, schemas.ResponsesRequest, LookupScopes{Provider: "openai"}) require.NotNil(t, pricing) assert.Equal(t, 15.0, pricing.InputCostPerToken) } func TestGetPricing_AppliesOverrideAfterFallbackResolution(t *testing.T) { t.Skip() - mc := newTestCatalog(nil, nil) - mc.logger = noOpLogger{} - mc.pricingData[makeKey("gpt-4o", "vertex", "chat")] = configstoreTables.TableModelPricing{ + s := newTestStore() + s.pricingData[makeKey("gpt-4o", "vertex", "chat")] = configstoreTables.TableModelPricing{ Model: "gpt-4o", Provider: "vertex", Mode: "chat", @@ -113,7 +123,7 @@ func TestGetPricing_AppliesOverrideAfterFallbackResolution(t *testing.T) { } geminiProviderID := "gemini" - require.NoError(t, mc.SetPricingOverrides([]configstoreTables.TablePricingOverride{ + require.NoError(t, s.SetOverrides([]configstoreTables.TablePricingOverride{ { ID: "gemini-override", ScopeKind: string(ScopeKindProvider), @@ -124,15 +134,14 @@ func TestGetPricing_AppliesOverrideAfterFallbackResolution(t *testing.T) { }, })) - pricing := mc.resolvePricing("gemini", "gpt-4o", "", schemas.ChatCompletionRequest, PricingLookupScopes{Provider: "gemini"}) + pricing := s.resolvePricing(schemas.RoutingInfo{Provider: "gemini", Model: "gpt-4o"}, schemas.ChatCompletionRequest, LookupScopes{Provider: "gemini"}) require.NotNil(t, pricing) assert.Equal(t, 7.0, pricing.InputCostPerToken) } func TestGetPricing_DeploymentLookupUsesResolvedModelForOverrideMatching(t *testing.T) { - mc := newTestCatalog(nil, nil) - mc.logger = noOpLogger{} - mc.pricingData[makeKey("dep-gpt4o", "openai", "chat")] = configstoreTables.TableModelPricing{ + s := newTestStore() + s.pricingData[makeKey("dep-gpt4o", "openai", "chat")] = configstoreTables.TableModelPricing{ Model: "dep-gpt4o", Provider: "openai", Mode: "chat", @@ -141,7 +150,7 @@ func TestGetPricing_DeploymentLookupUsesResolvedModelForOverrideMatching(t *test } providerID := "openai" - require.NoError(t, mc.SetPricingOverrides([]configstoreTables.TablePricingOverride{ + require.NoError(t, s.SetOverrides([]configstoreTables.TablePricingOverride{ { ID: "resolved-model-override", ScopeKind: string(ScopeKindProvider), @@ -155,16 +164,15 @@ func TestGetPricing_DeploymentLookupUsesResolvedModelForOverrideMatching(t *test // Override pattern matches the resolved model name ("dep-gpt4o"), not the // originally requested name ("gpt-4o"), because resolved model has priority. - pricing := mc.resolvePricing("openai", "gpt-4o", "dep-gpt4o", schemas.ChatCompletionRequest, PricingLookupScopes{Provider: "openai"}) + pricing := s.resolvePricing(schemas.RoutingInfo{Provider: "openai", Model: "gpt-4o", ResolvedKeyAlias: &schemas.ResolvedKeyAlias{ModelID: "dep-gpt4o"}}, schemas.ChatCompletionRequest, LookupScopes{Provider: "openai"}) require.NotNil(t, pricing) require.NotNil(t, pricing.InputCostPerToken) assert.Equal(t, 7.0, *pricing.InputCostPerToken) } func TestGetPricing_FallbackUsesRequestedProviderForScopeMatching(t *testing.T) { - mc := newTestCatalog(nil, nil) - mc.logger = noOpLogger{} - mc.pricingData[makeKey("gpt-4o", "vertex", "chat")] = configstoreTables.TableModelPricing{ + s := newTestStore() + s.pricingData[makeKey("gpt-4o", "vertex", "chat")] = configstoreTables.TableModelPricing{ Model: "gpt-4o", Provider: "vertex", Mode: "chat", @@ -174,7 +182,7 @@ func TestGetPricing_FallbackUsesRequestedProviderForScopeMatching(t *testing.T) geminiProviderID := "gemini" vertexProviderID := "vertex" - require.NoError(t, mc.SetPricingOverrides([]configstoreTables.TablePricingOverride{ + require.NoError(t, s.SetOverrides([]configstoreTables.TablePricingOverride{ { ID: "gemini-provider-override", ScopeKind: string(ScopeKindProvider), @@ -195,7 +203,7 @@ func TestGetPricing_FallbackUsesRequestedProviderForScopeMatching(t *testing.T) }, })) - pricing := mc.resolvePricing("gemini", "gpt-4o", "", schemas.ChatCompletionRequest, PricingLookupScopes{Provider: "gemini"}) + pricing := s.resolvePricing(schemas.RoutingInfo{Provider: "gemini", Model: "gpt-4o"}, schemas.ChatCompletionRequest, LookupScopes{Provider: "gemini"}) require.NotNil(t, pricing) require.NotNil(t, pricing.InputCostPerToken) assert.Equal(t, 5.0, *pricing.InputCostPerToken) @@ -203,9 +211,8 @@ func TestGetPricing_FallbackUsesRequestedProviderForScopeMatching(t *testing.T) func TestGetPricing_ExactOverrideDoesNotMatchProviderPrefixedModel(t *testing.T) { t.Skip() - mc := newTestCatalog(nil, nil) - mc.logger = noOpLogger{} - mc.pricingData[makeKey("openai/gpt-4o", "openai", "chat")] = configstoreTables.TableModelPricing{ + s := newTestStore() + s.pricingData[makeKey("openai/gpt-4o", "openai", "chat")] = configstoreTables.TableModelPricing{ Model: "openai/gpt-4o", Provider: "openai", Mode: "chat", @@ -214,7 +221,7 @@ func TestGetPricing_ExactOverrideDoesNotMatchProviderPrefixedModel(t *testing.T) } providerID := "openai" - require.NoError(t, mc.SetPricingOverrides([]configstoreTables.TablePricingOverride{ + require.NoError(t, s.SetOverrides([]configstoreTables.TablePricingOverride{ { ID: "openai-override-0", ScopeKind: string(ScopeKindProvider), @@ -225,17 +232,16 @@ func TestGetPricing_ExactOverrideDoesNotMatchProviderPrefixedModel(t *testing.T) }, })) - pricing := mc.resolvePricing("openai", "openai/gpt-4o", "", schemas.ChatCompletionRequest, PricingLookupScopes{Provider: "openai"}) + pricing := s.resolvePricing(schemas.RoutingInfo{Provider: "openai", Model: "openai/gpt-4o"}, schemas.ChatCompletionRequest, LookupScopes{Provider: "openai"}) require.NotNil(t, pricing) assert.Equal(t, 1.0, pricing.InputCostPerToken) } func TestGetPricing_NoMatchingOverrideLeavesPricingUnchanged(t *testing.T) { t.Skip() - mc := newTestCatalog(nil, nil) - mc.logger = noOpLogger{} + s := newTestStore() baseCacheRead := 0.4 - mc.pricingData[makeKey("gpt-4o", "openai", "chat")] = configstoreTables.TableModelPricing{ + s.pricingData[makeKey("gpt-4o", "openai", "chat")] = configstoreTables.TableModelPricing{ Model: "gpt-4o", Provider: "openai", Mode: "chat", @@ -245,7 +251,7 @@ func TestGetPricing_NoMatchingOverrideLeavesPricingUnchanged(t *testing.T) { } providerID := "openai" - require.NoError(t, mc.SetPricingOverrides([]configstoreTables.TablePricingOverride{ + require.NoError(t, s.SetOverrides([]configstoreTables.TablePricingOverride{ { ID: "openai-override-0", ScopeKind: string(ScopeKindProvider), @@ -256,7 +262,7 @@ func TestGetPricing_NoMatchingOverrideLeavesPricingUnchanged(t *testing.T) { }, })) - pricing := mc.resolvePricing("openai", "gpt-4o", "", schemas.ChatCompletionRequest, PricingLookupScopes{Provider: "openai"}) + pricing := s.resolvePricing(schemas.RoutingInfo{Provider: "openai", Model: "gpt-4o"}, schemas.ChatCompletionRequest, LookupScopes{Provider: "openai"}) require.NotNil(t, pricing) assert.Equal(t, 1.0, pricing.InputCostPerToken) assert.Equal(t, 2.0, pricing.OutputCostPerToken) @@ -264,11 +270,10 @@ func TestGetPricing_NoMatchingOverrideLeavesPricingUnchanged(t *testing.T) { assert.Equal(t, 0.4, *pricing.CacheReadInputTokenCost) } -func TestDeleteProviderPricingOverrides_StopsApplying(t *testing.T) { +func TestDeleteProviderOverrides_StopsApplying(t *testing.T) { t.Skip() - mc := newTestCatalog(nil, nil) - mc.logger = noOpLogger{} - mc.pricingData[makeKey("gpt-4o", "openai", "chat")] = configstoreTables.TableModelPricing{ + s := newTestStore() + s.pricingData[makeKey("gpt-4o", "openai", "chat")] = configstoreTables.TableModelPricing{ Model: "gpt-4o", Provider: "openai", Mode: "chat", @@ -277,7 +282,7 @@ func TestDeleteProviderPricingOverrides_StopsApplying(t *testing.T) { } providerID := "openai" - require.NoError(t, mc.SetPricingOverrides([]configstoreTables.TablePricingOverride{ + require.NoError(t, s.SetOverrides([]configstoreTables.TablePricingOverride{ { ID: "openai-override-0", ScopeKind: string(ScopeKindProvider), @@ -288,22 +293,21 @@ func TestDeleteProviderPricingOverrides_StopsApplying(t *testing.T) { }, })) - pricing := mc.resolvePricing("openai", "gpt-4o", "", schemas.ChatCompletionRequest, PricingLookupScopes{Provider: "openai"}) + pricing := s.resolvePricing(schemas.RoutingInfo{Provider: "openai", Model: "gpt-4o"}, schemas.ChatCompletionRequest, LookupScopes{Provider: "openai"}) require.NotNil(t, pricing) assert.Equal(t, 11.0, pricing.InputCostPerToken) - require.NoError(t, mc.SetPricingOverrides(nil)) + require.NoError(t, s.SetOverrides(nil)) - pricing = mc.resolvePricing("openai", "gpt-4o", "", schemas.ChatCompletionRequest, PricingLookupScopes{Provider: "openai"}) + pricing = s.resolvePricing(schemas.RoutingInfo{Provider: "openai", Model: "gpt-4o"}, schemas.ChatCompletionRequest, LookupScopes{Provider: "openai"}) require.NotNil(t, pricing) assert.Equal(t, 1.0, pricing.InputCostPerToken) } func TestGetPricing_WildcardSpecificityLongerLiteralWins(t *testing.T) { t.Skip() - mc := newTestCatalog(nil, nil) - mc.logger = noOpLogger{} - mc.pricingData[makeKey("gpt-4o-mini", "openai", "chat")] = configstoreTables.TableModelPricing{ + s := newTestStore() + s.pricingData[makeKey("gpt-4o-mini", "openai", "chat")] = configstoreTables.TableModelPricing{ Model: "gpt-4o-mini", Provider: "openai", Mode: "chat", @@ -312,7 +316,7 @@ func TestGetPricing_WildcardSpecificityLongerLiteralWins(t *testing.T) { } providerID := "openai" - require.NoError(t, mc.SetPricingOverrides([]configstoreTables.TablePricingOverride{ + require.NoError(t, s.SetOverrides([]configstoreTables.TablePricingOverride{ { ID: "openai-override-0", ScopeKind: string(ScopeKindProvider), @@ -331,17 +335,14 @@ func TestGetPricing_WildcardSpecificityLongerLiteralWins(t *testing.T) { }, })) - pricing := mc.resolvePricing("openai", "gpt-4o-mini", "", schemas.ChatCompletionRequest, PricingLookupScopes{Provider: "openai"}) + pricing := s.resolvePricing(schemas.RoutingInfo{Provider: "openai", Model: "gpt-4o-mini"}, schemas.ChatCompletionRequest, LookupScopes{Provider: "openai"}) require.NotNil(t, pricing) assert.Equal(t, 6.0, pricing.InputCostPerToken) } -// TestGetPricing_FirstInsertionWinsOnTie verifies that when multiple wildcard overrides -// match the same model and scope, the first one inserted takes precedence. func TestGetPricing_FirstInsertionWinsOnTie(t *testing.T) { - mc := newTestCatalog(nil, nil) - mc.logger = noOpLogger{} - mc.pricingData[makeKey("gpt-4o-mini", "openai", "chat")] = configstoreTables.TableModelPricing{ + s := newTestStore() + s.pricingData[makeKey("gpt-4o-mini", "openai", "chat")] = configstoreTables.TableModelPricing{ Model: "gpt-4o-mini", Provider: "openai", Mode: "chat", @@ -350,7 +351,7 @@ func TestGetPricing_FirstInsertionWinsOnTie(t *testing.T) { } providerID := "openai" - require.NoError(t, mc.SetPricingOverrides([]configstoreTables.TablePricingOverride{ + require.NoError(t, s.SetOverrides([]configstoreTables.TablePricingOverride{ { ID: "a-override", ScopeKind: string(ScopeKindProvider), @@ -371,7 +372,7 @@ func TestGetPricing_FirstInsertionWinsOnTie(t *testing.T) { }, })) - pricing := mc.resolvePricing("openai", "gpt-4o-mini", "", schemas.ChatCompletionRequest, PricingLookupScopes{Provider: "openai"}) + pricing := s.resolvePricing(schemas.RoutingInfo{Provider: "openai", Model: "gpt-4o-mini"}, schemas.ChatCompletionRequest, LookupScopes{Provider: "openai"}) require.NotNil(t, pricing) require.NotNil(t, pricing.InputCostPerToken) assert.Equal(t, 8.0, *pricing.InputCostPerToken) @@ -392,7 +393,7 @@ func TestPatchPricing_PartialPatchOnlyChangesSpecifiedFields(t *testing.T) { } cacheRead := 0.9 - patched := patchPricing(base, PricingOptions{ + patched := patchPricing(base, Options{ InputCostPerToken: bifrost.Ptr(3.0), CacheReadInputTokenCost: &cacheRead, }) @@ -406,15 +407,14 @@ func TestPatchPricing_PartialPatchOnlyChangesSpecifiedFields(t *testing.T) { assert.Equal(t, 0.7, *patched.InputCostPerImage) } -func TestApplyScopedPricingOverrides_ScopePrecedence(t *testing.T) { - mc := newTestCatalog(nil, nil) - mc.logger = noOpLogger{} +func TestApplyScopedOverrides_ScopePrecedence(t *testing.T) { + s := newTestStore() providerScopeID := "openai" providerKeyScopeID := "provider-key-1" virtualKeyScopeID := "virtual-key-1" - require.NoError(t, mc.SetPricingOverrides([]configstoreTables.TablePricingOverride{ + require.NoError(t, s.SetOverrides([]configstoreTables.TablePricingOverride{ { ID: "global", ScopeKind: string(ScopeKindGlobal), @@ -462,12 +462,12 @@ func TestApplyScopedPricingOverrides_ScopePrecedence(t *testing.T) { tests := []struct { name string - scopes PricingLookupScopes + scopes LookupScopes expected float64 }{ { name: "virtual key wins over provider key, provider and global", - scopes: PricingLookupScopes{ + scopes: LookupScopes{ VirtualKeyID: virtualKeyScopeID, SelectedKeyID: providerKeyScopeID, Provider: providerScopeID, @@ -476,7 +476,7 @@ func TestApplyScopedPricingOverrides_ScopePrecedence(t *testing.T) { }, { name: "provider key wins over provider and global", - scopes: PricingLookupScopes{ + scopes: LookupScopes{ SelectedKeyID: providerKeyScopeID, Provider: providerScopeID, }, @@ -484,21 +484,21 @@ func TestApplyScopedPricingOverrides_ScopePrecedence(t *testing.T) { }, { name: "provider wins over global", - scopes: PricingLookupScopes{ + scopes: LookupScopes{ Provider: providerScopeID, }, expected: 3.0, }, { name: "global applies when no narrower scope is provided", - scopes: PricingLookupScopes{}, + scopes: LookupScopes{}, expected: 2.0, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - patched, applied := mc.applyPricingOverrides("gpt-5-nano", schemas.ChatCompletionRequest, base, tc.scopes) + patched, applied := s.applyPricingOverrides("gpt-5-nano", schemas.ChatCompletionRequest, base, tc.scopes) require.True(t, applied) require.NotNil(t, patched.InputCostPerToken) assert.Equal(t, tc.expected, *patched.InputCostPerToken) diff --git a/framework/modelcatalog/datasheet/params.go b/framework/modelcatalog/datasheet/params.go new file mode 100644 index 0000000000..86e990f7be --- /dev/null +++ b/framework/modelcatalog/datasheet/params.go @@ -0,0 +1,258 @@ +package datasheet + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "slices" + + bifrost "github.com/maximhq/bifrost/core" + providerUtils "github.com/maximhq/bifrost/core/providers/utils" + "github.com/maximhq/bifrost/core/schemas" + configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" + "github.com/tidwall/gjson" + "gorm.io/gorm" +) + +// LoadModelParamsFromDB bulk-loads model parameters from the DB into the +// provider-utils cache and the in-memory supportedResponseTypes / +// supportedParams indexes. Returns the row count so the composer can decide +// whether to background-sync from URL afterwards. +// +// The provider-utils cache-miss handler in the composer still loads one row +// at a time when an unknown model is queried; both paths use the same JSON +// shape stored in the table. +func (s *Store) LoadModelParamsFromDB(ctx context.Context) (int, error) { + if s.configStore == nil { + return 0, nil + } + rows, err := s.configStore.GetModelParameters(ctx) + if err != nil { + return 0, fmt.Errorf("failed to load model parameters from database: %w", err) + } + if len(rows) == 0 { + if s.logger != nil { + s.logger.Debug("no model parameters rows in database") + } + return 0, nil + } + paramsData := make(map[string]json.RawMessage, len(rows)) + for _, row := range rows { + paramsData[row.Model] = json.RawMessage(row.Data) + } + applied := s.applyModelParameters(paramsData) + if s.logger != nil { + s.logger.Debug("loaded %d model parameters records from database into cache (%d rows scanned)", applied, len(rows)) + } + return applied, nil +} + +// SyncModelParamsFromURL fetches model parameters from the configured URL, +// persists to DB (when configStore != nil), and refreshes the in-memory +// indexes. On URL failure it falls back to DB records when any exist. +func (s *Store) SyncModelParamsFromURL(ctx context.Context) error { + if s.logger != nil { + s.logger.Debug("starting model parameters synchronization") + } + + paramsData, err := withRetries(ctx, urlFetchMaxRetries, urlFetchMaxBackoff, func() (map[string]json.RawMessage, error) { + return s.loadModelParametersFromURL(ctx) + }) + if err != nil { + if s.configStore != nil { + rows, dbErr := s.configStore.GetModelParameters(ctx) + if dbErr == nil && len(rows) > 0 { + if s.logger != nil { + s.logger.Error("failed to load model parameters from URL, falling back to existing database records: %v", err) + } + return nil + } + } + return fmt.Errorf("failed to load model parameters from URL and no existing data in database: %w", err) + } + + if s.configStore != nil { + err = s.configStore.ExecuteTransaction(ctx, func(tx *gorm.DB) error { + for model, data := range paramsData { + params := &configstoreTables.TableModelParameters{ + Model: model, + Data: string(data), + } + if err := s.configStore.UpsertModelParameters(ctx, params, tx); err != nil { + return fmt.Errorf("failed to upsert model parameters for model %s: %w", model, err) + } + } + return nil + }) + if err != nil { + return fmt.Errorf("failed to sync model parameters to database: %w", err) + } + } + + s.applyModelParameters(paramsData) + if s.logger != nil { + s.logger.Info("successfully synced %d model parameters records", len(paramsData)) + } + return nil +} + +// LoadModelParamsFromURLIntoMemory fetches model parameters from the URL and +// applies them in-memory only. Used when there's no config store. +func (s *Store) LoadModelParamsFromURLIntoMemory(ctx context.Context) error { + paramsData, err := withRetries(ctx, urlFetchMaxRetries, urlFetchMaxBackoff, func() (map[string]json.RawMessage, error) { + return s.loadModelParametersFromURL(ctx) + }) + if err != nil { + return fmt.Errorf("failed to load model parameters from URL: %w", err) + } + s.applyModelParameters(paramsData) + return nil +} + +// loadModelParametersFromURL fetches and parses the model parameters +// datasheet at the configured URL. +func (s *Store) loadModelParametersFromURL(ctx context.Context) (map[string]json.RawMessage, error) { + s.syncCfgMu.RLock() + rawURL := s.modelParametersURL + s.syncCfgMu.RUnlock() + + parsed, err := url.Parse(rawURL) + if err != nil { + return nil, fmt.Errorf("failed to parse model parameters URL: %w", err) + } + + var data []byte + + if parsed.Scheme == "file" { + data, err = os.ReadFile(parsed.Path) + if err != nil { + return nil, fmt.Errorf("failed to read model parameters file: %w", err) + } + } else { + if err := bifrost.ValidateExternalURL(rawURL, true); err != nil { + return nil, fmt.Errorf("model parameters URL validation failed: %w", err) + } + client := &http.Client{Timeout: DefaultModelParametersTimeout} + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, fmt.Errorf("failed to create HTTP request: %w", err) + } + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to download model parameters data: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to download model parameters data: HTTP %d", resp.StatusCode) + } + data, err = io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read model parameters response: %w", err) + } + } + var paramsData map[string]json.RawMessage + if err := json.Unmarshal(data, ¶msData); err != nil { + return nil, fmt.Errorf("failed to unmarshal model parameters data: %w", err) + } + if s.logger != nil { + s.logger.Debug("successfully downloaded and parsed %d model parameters records", len(paramsData)) + } + return paramsData, nil +} + +// applyModelParameters parses the raw model-parameters JSON and updates: +// - supportedResponseTypes (per-model normalized output types) +// - supportedParams (per-model accepted request parameter names) +// - the provider-utils ModelParams cache (max_output_tokens, +// vertex_multi_region_only) +// +// Models with no useful info contribute nothing — the indexes are wholly +// replaced under mu.Lock so readers see either the pre- or post-sync state, +// never a partial mix. Returns the count of successfully parsed entries so +// bootstrap callers can distinguish "DB had rows but all malformed" from +// "DB had usable rows". +func (s *Store) applyModelParameters(paramsData map[string]json.RawMessage) int { + modelParamsEntries := make(map[string]providerUtils.ModelParams, len(paramsData)) + newResponseTypes := make(map[string][]string, len(paramsData)) + newParamsIndex := make(map[string][]string, len(paramsData)) + applied := 0 + + for model, rawData := range paramsData { + var parsed modelParametersParseResult + if err := json.Unmarshal(rawData, &parsed); err != nil { + if s.logger != nil { + s.logger.Warn("model-parameters-sync: skipping malformed parameters for model %s: %v", model, err) + } + continue + } + applied++ + + outputs := make([]string, 0, len(parsed.SupportedEndpoints)) + for _, endpoint := range parsed.SupportedEndpoints { + if normalized := normalizeEndpointToOutputType(endpoint); normalized != "" && !slices.Contains(outputs, normalized) { + outputs = append(outputs, normalized) + } + } + if parsed.Mode != nil { + if normalized := normalizeModeToOutputType(*parsed.Mode); normalized != "" && !slices.Contains(outputs, normalized) { + outputs = append(outputs, normalized) + } + } + + // Backfill text_completion when the pricing catalog has a row for it + // even though supported_endpoints didn't mention /completions. + if !slices.Contains(outputs, "text_completion") { + if provider := gjson.GetBytes(rawData, "provider"); provider.Exists() { + key := makeKey(model, normalizeProvider(provider.String()), normalizeRequestType(schemas.TextCompletionRequest)) + s.mu.RLock() + _, ok := s.pricingData[key] + s.mu.RUnlock() + if ok { + outputs = append(outputs, "text_completion") + } + } + } + + if len(outputs) > 0 { + newResponseTypes[model] = outputs + } + + if supported := extractSupportedParams(&parsed); len(supported) > 0 { + newParamsIndex[model] = supported + } + + var p struct { + MaxOutputTokens *int `json:"max_output_tokens"` + } + if err := json.Unmarshal(rawData, &p); err == nil && (p.MaxOutputTokens != nil || parsed.VertexMultiRegionOnly != nil) { + modelParamsEntries[model] = providerUtils.ModelParams{ + MaxOutputTokens: p.MaxOutputTokens, + IsVertexMultiRegionOnly: parsed.VertexMultiRegionOnly, + } + } + } + + s.mu.Lock() + s.supportedResponseTypes = newResponseTypes + s.supportedParams = newParamsIndex + s.mu.Unlock() + + if len(modelParamsEntries) > 0 { + providerUtils.BulkSetModelParams(modelParamsEntries) + } + return applied +} + +// GetModelParametersByModel reads a single model-parameter row from the DB. +// Used by the composer's cache-miss handler — installed via +// providerUtils.SetCacheMissHandler in the composer's Init. +func (s *Store) GetModelParametersByModel(ctx context.Context, model string) (*configstoreTables.TableModelParameters, error) { + if s.configStore == nil { + return nil, nil + } + return s.configStore.GetModelParametersByModel(ctx, model) +} diff --git a/framework/modelcatalog/datasheet/store.go b/framework/modelcatalog/datasheet/store.go new file mode 100644 index 0000000000..4c3ee8b6dc --- /dev/null +++ b/framework/modelcatalog/datasheet/store.go @@ -0,0 +1,452 @@ +package datasheet + +import ( + "slices" + "strings" + "sync" + "time" + + bifrost "github.com/maximhq/bifrost/core" + "github.com/maximhq/bifrost/core/schemas" + "github.com/maximhq/bifrost/framework/configstore" + configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" +) + +// Defaults for sync configuration and timeouts. Exposed so the composer can +// fall back to these when the framework Config leaves fields nil. +const ( + DefaultURL = "https://getbifrost.ai/datasheet" + DefaultModelParametersURL = "https://getbifrost.ai/datasheet/model-parameters" + DefaultSyncInterval = 24 * time.Hour + DefaultPricingTimeout = 45 * time.Second + DefaultModelParametersTimeout = 45 * time.Second +) + +// Config groups the values the composer hands to New / UpdateSyncConfig. +// Zero values fall back to the Default* constants. +type Config struct { + URL string + ModelParametersURL string + SyncInterval time.Duration +} + +func (c Config) resolved() Config { + if c.URL == "" { + c.URL = DefaultURL + } + if c.ModelParametersURL == "" { + c.ModelParametersURL = DefaultModelParametersURL + } + if c.SyncInterval <= 0 { + c.SyncInterval = DefaultSyncInterval + } + return c +} + +// Store owns the pricing catalog (canonical rows, base-model index, derived +// datasheet view, supported request types/parameters) and pricing overrides. +// +// All I/O is driven by the composer — Store does not own a ticker or the +// distributed lock; it exposes SyncFromURL / LoadFromDB / UpdateSyncConfig as +// the surface the composer calls. +type Store struct { + configStore configstore.ConfigStore + logger schemas.Logger + + // Canonical pricing state, protected by mu. Read paths take RLock and + // return defensive copies of any slice/map they expose. + mu sync.RWMutex + pricingData map[string]configstoreTables.TableModelPricing // model|provider|mode → row + baseModelIndex map[string]string // model → canonical base name + supportedResponseTypes map[string][]string // model → [chat_completion, responses, …] + supportedParams map[string][]string // model → [temperature, top_p, …] + datasheetByProvider map[schemas.ModelProvider][]string // rebuilt every reload + + // Overrides under their own mutex: writes here don't block pricing reads + // (the hot CalculateCost path takes mu.RLock and overridesMu.RLock + // independently and the orderings never invert). + overridesMu sync.RWMutex + rawOverrides []Override + customPricing *customPricingData + + // Sync configuration owned here so UpdateSyncConfig is atomic w.r.t. the + // URL accessors in sync.go. The composer's ticker reads SyncInterval() + // and LastSyncedAt() to schedule. + syncCfgMu sync.RWMutex + url string + modelParametersURL string + syncInterval time.Duration + lastSyncedAt time.Time +} + +// New constructs a Store with the given config. The store is empty; callers +// (composer) drive bootstrap via LoadFromDB or LoadFromURLIntoMemory. +func New(configStore configstore.ConfigStore, logger schemas.Logger, cfg Config) *Store { + cfg = cfg.resolved() + return &Store{ + configStore: configStore, + logger: logger, + pricingData: make(map[string]configstoreTables.TableModelPricing), + baseModelIndex: make(map[string]string), + supportedResponseTypes: make(map[string][]string), + supportedParams: make(map[string][]string), + datasheetByProvider: make(map[schemas.ModelProvider][]string), + url: cfg.URL, + modelParametersURL: cfg.ModelParametersURL, + syncInterval: cfg.SyncInterval, + } +} + +// UpdateSyncConfig replaces URL / params URL / interval atomically. The +// composer is responsible for triggering a fresh sync after this returns. +func (s *Store) UpdateSyncConfig(cfg Config) { + cfg = cfg.resolved() + s.syncCfgMu.Lock() + s.url = cfg.URL + s.modelParametersURL = cfg.ModelParametersURL + s.syncInterval = cfg.SyncInterval + s.syncCfgMu.Unlock() +} + +// URL returns a snapshot of the pricing URL. +func (s *Store) URL() string { + s.syncCfgMu.RLock() + defer s.syncCfgMu.RUnlock() + return s.url +} + +// ModelParametersURL returns a snapshot of the model-parameters URL. +func (s *Store) ModelParametersURL() string { + s.syncCfgMu.RLock() + defer s.syncCfgMu.RUnlock() + return s.modelParametersURL +} + +// SyncInterval returns the minimum elapsed time between background syncs. +func (s *Store) SyncInterval() time.Duration { + s.syncCfgMu.RLock() + defer s.syncCfgMu.RUnlock() + return s.syncInterval +} + +// LastSyncedAt returns the last successful URL→DB sync timestamp; zero +// before any sync has completed. +func (s *Store) LastSyncedAt() time.Time { + s.syncCfgMu.RLock() + defer s.syncCfgMu.RUnlock() + return s.lastSyncedAt +} + +// MarkSynced records the timestamp of a successful sync — called by the +// composer's ticker after a successful tick. +func (s *Store) MarkSynced(t time.Time) { + s.syncCfgMu.Lock() + s.lastSyncedAt = t + s.syncCfgMu.Unlock() +} + +// --- Reads --- + +// Get returns the raw pricing row for (model, provider, requestType) or nil. +// Useful for callers that need exact pricing without override resolution. +func (s *Store) Get(model string, provider schemas.ModelProvider, requestType schemas.RequestType) *configstoreTables.TableModelPricing { + key := makeKey(model, string(provider), normalizeRequestType(requestType)) + s.mu.RLock() + defer s.mu.RUnlock() + row, ok := s.pricingData[key] + if !ok { + return nil + } + return &row +} + +// GetPricingEntryForModel returns the first pricing entry found across known +// modes. Preserved for callers (inference handler) that want any pricing row +// for the model without specifying a request type. +func (s *Store) GetPricingEntryForModel(model string, provider schemas.ModelProvider) *Entry { + s.mu.RLock() + defer s.mu.RUnlock() + for _, mode := range []schemas.RequestType{ + schemas.TextCompletionRequest, + schemas.ChatCompletionRequest, + schemas.ResponsesRequest, + schemas.EmbeddingRequest, + schemas.RerankRequest, + schemas.SpeechRequest, + schemas.TranscriptionRequest, + schemas.ImageGenerationRequest, + schemas.ImageEditRequest, + schemas.ImageVariationRequest, + schemas.VideoGenerationRequest, + schemas.OCRRequest, + } { + key := makeKey(model, string(provider), normalizeRequestType(mode)) + if pricing, ok := s.pricingData[key]; ok { + return convertTablePricingToEntry(&pricing) + } + } + return nil +} + +// GetCapabilityEntry returns capability metadata (context length, supported +// modes, etc.) for a (model, provider) pair. Prefers chat → responses → +// text-completion entries; falls back to the lexicographically first mode if +// none of the preferred modes match. Tries the exact model first, then the +// canonical base model. +func (s *Store) GetCapabilityEntry(model string, provider schemas.ModelProvider) *Entry { + s.mu.RLock() + defer s.mu.RUnlock() + + if entry := s.capabilityEntryForExactUnsafe(model, provider); entry != nil { + return entry + } + + baseModel := s.baseModelNameUnsafe(model) + if baseModel != model { + if entry := s.capabilityEntryForExactUnsafe(baseModel, provider); entry != nil { + return entry + } + } + + if entry := s.capabilityEntryForFamilyUnsafe(baseModel, provider); entry != nil { + return entry + } + return nil +} + +// BaseModelName returns the canonical base model name. Uses the pre-computed +// base_model from the pricing catalog when present, falling back to +// algorithmic date/version stripping for unknown models. +// +// "gpt-4o" → "gpt-4o" +// "openai/gpt-4o" → "gpt-4o" +// "gpt-4o-2024-08-06" → "gpt-4o" (algorithmic fallback) +func (s *Store) BaseModelName(model string) string { + s.mu.RLock() + defer s.mu.RUnlock() + return s.baseModelNameUnsafe(model) +} + +// baseModelNameUnsafe — caller MUST hold s.mu. Used by GetCapabilityEntry +// and other hot paths that already hold the lock. +func (s *Store) baseModelNameUnsafe(model string) string { + if base, ok := s.baseModelIndex[model]; ok { + return base + } + _, baseName := schemas.ParseModelString(model, "") + if baseName != model { + if base, ok := s.baseModelIndex[baseName]; ok { + return base + } + } + return schemas.BaseModelName(baseName) +} + +// IsSameModel reports whether two model strings refer to the same underlying +// model after normalization. +func (s *Store) IsSameModel(model1, model2 string) bool { + if model1 == model2 { + return true + } + return s.BaseModelName(model1) == s.BaseModelName(model2) +} + +// DistinctBaseModelNames returns every unique base name from the catalog. +// Used by governance for cross-provider model selection. +func (s *Store) DistinctBaseModelNames() []string { + s.mu.RLock() + defer s.mu.RUnlock() + seen := make(map[string]struct{}) + for _, baseName := range s.baseModelIndex { + seen[baseName] = struct{}{} + } + out := make([]string, 0, len(seen)) + for name := range seen { + out = append(out, name) + } + return out +} + +// DatasheetModelsForProvider returns the per-provider model slice derived +// from pricing data on the last load/sync. Composer unions this with +// live.ModelsForProvider on read. +func (s *Store) DatasheetModelsForProvider(provider schemas.ModelProvider) []string { + s.mu.RLock() + defer s.mu.RUnlock() + models, ok := s.datasheetByProvider[provider] + if !ok { + return nil + } + out := make([]string, len(models)) + copy(out, models) + return out +} + +// DatasheetProviders returns every provider that has at least one pricing +// row in the datasheet view. Composer unions this with live + keyconfig to +// enumerate "all known providers" for GetProvidersForModel. +func (s *Store) DatasheetProviders() []schemas.ModelProvider { + s.mu.RLock() + defer s.mu.RUnlock() + out := make([]schemas.ModelProvider, 0, len(s.datasheetByProvider)) + for p := range s.datasheetByProvider { + out = append(out, p) + } + return out +} + +// IsRequestTypeSupported checks whether a model declares support for the +// given request type via the model-parameters datasheet. +func (s *Store) IsRequestTypeSupported(model string, requestType schemas.RequestType) bool { + s.mu.RLock() + defer s.mu.RUnlock() + outputs, ok := s.supportedResponseTypes[model] + return ok && slices.Contains(outputs, string(requestType)) +} + +// GetSupportedParameters returns the list of OpenAI-compatible parameter +// names a model accepts (e.g. temperature, top_p, tools). nil for unknown. +func (s *Store) GetSupportedParameters(model string) []string { + s.mu.RLock() + params, ok := s.supportedParams[model] + s.mu.RUnlock() + if !ok { + return nil + } + out := make([]string, len(params)) + copy(out, params) + return out +} + +// IsTextCompletionSupported checks whether a model has a text_completion +// pricing entry — used by litellmcompat to decide whether to convert text +// completion requests into chat completion requests. +func (s *Store) IsTextCompletionSupported(model string, provider schemas.ModelProvider) bool { + s.mu.RLock() + defer s.mu.RUnlock() + key := makeKey(model, normalizeProvider(string(provider)), normalizeRequestType(schemas.TextCompletionRequest)) + _, ok := s.pricingData[key] + return ok +} + +// --- Private capability helpers (caller holds mu) --- + +func (s *Store) capabilityEntryForExactUnsafe(model string, provider schemas.ModelProvider) *Entry { + preferredModes := []schemas.RequestType{ + schemas.ChatCompletionRequest, + schemas.ResponsesRequest, + schemas.TextCompletionRequest, + } + for _, mode := range preferredModes { + key := makeKey(model, string(provider), normalizeRequestType(mode)) + if pricing, ok := s.pricingData[key]; ok { + return convertTablePricingToEntry(&pricing) + } + } + + prefix := model + "|" + string(provider) + "|" + var matchingKeys []string + for key := range s.pricingData { + if strings.HasPrefix(key, prefix) { + matchingKeys = append(matchingKeys, key) + } + } + return s.selectCapabilityEntryFromKeysUnsafe(matchingKeys) +} + +func (s *Store) capabilityEntryForFamilyUnsafe(baseModel string, provider schemas.ModelProvider) *Entry { + if baseModel == "" { + return nil + } + var matchingKeys []string + for key, pricing := range s.pricingData { + if normalizeProvider(pricing.Provider) != string(provider) { + continue + } + if s.baseModelNameUnsafe(pricing.Model) != baseModel { + continue + } + matchingKeys = append(matchingKeys, key) + } + return s.selectCapabilityEntryFromKeysUnsafe(matchingKeys) +} + +func (s *Store) selectCapabilityEntryFromKeysUnsafe(matchingKeys []string) *Entry { + if len(matchingKeys) == 0 { + return nil + } + preferredModes := []string{ + normalizeRequestType(schemas.ChatCompletionRequest), + normalizeRequestType(schemas.ResponsesRequest), + normalizeRequestType(schemas.TextCompletionRequest), + } + for _, mode := range preferredModes { + var modeMatches []string + for _, key := range matchingKeys { + parts := strings.SplitN(key, "|", 3) + if len(parts) != 3 || parts[2] != mode { + continue + } + modeMatches = append(modeMatches, key) + } + if len(modeMatches) == 0 { + continue + } + slices.Sort(modeMatches) + pricing := s.pricingData[modeMatches[0]] + return convertTablePricingToEntry(&pricing) + } + slices.Sort(matchingKeys) + pricing := s.pricingData[matchingKeys[0]] + return convertTablePricingToEntry(&pricing) +} + +// NewTestStore constructs a minimal Store for unit tests without I/O. +// Optionally seed baseModelIndex so BaseModelName lookups resolve. A no-op +// logger is wired so cost / pricing paths (which assume Store.logger is +// non-nil) don't panic from external test code. +func NewTestStore(baseModelIndex map[string]string) *Store { + if baseModelIndex == nil { + baseModelIndex = make(map[string]string) + } + return &Store{ + logger: bifrost.NewNoOpLogger(), + pricingData: make(map[string]configstoreTables.TableModelPricing), + baseModelIndex: baseModelIndex, + supportedResponseTypes: make(map[string][]string), + supportedParams: make(map[string][]string), + datasheetByProvider: make(map[schemas.ModelProvider][]string), + } +} + +// --- Internal: rebuild the datasheet view from current pricingData --- + +// rebuildDatasheetViewUnsafe regenerates baseModelIndex and datasheetByProvider +// from pricingData. Caller MUST hold s.mu write-lock. Called after every +// pricingData mutation in sync.go / params.go. +func (s *Store) rebuildDatasheetViewUnsafe() { + s.baseModelIndex = make(map[string]string) + providerModels := make(map[schemas.ModelProvider]map[string]struct{}) + + for _, pricing := range s.pricingData { + normalized := schemas.ModelProvider(normalizeProvider(pricing.Provider)) + if providerModels[normalized] == nil { + providerModels[normalized] = make(map[string]struct{}) + } + providerModels[normalized][pricing.Model] = struct{}{} + + if pricing.BaseModel != "" { + s.baseModelIndex[pricing.Model] = pricing.BaseModel + } + } + + s.datasheetByProvider = make(map[schemas.ModelProvider][]string, len(providerModels)) + for provider, modelSet := range providerModels { + models := make([]string, 0, len(modelSet)) + for m := range modelSet { + models = append(models, m) + } + slices.Sort(models) + s.datasheetByProvider[provider] = models + } +} diff --git a/framework/modelcatalog/datasheet/sync.go b/framework/modelcatalog/datasheet/sync.go new file mode 100644 index 0000000000..4d3b9749c6 --- /dev/null +++ b/framework/modelcatalog/datasheet/sync.go @@ -0,0 +1,219 @@ +package datasheet + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "time" + + bifrost "github.com/maximhq/bifrost/core" + providerUtils "github.com/maximhq/bifrost/core/providers/utils" + configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" + "gorm.io/gorm" +) + +const ( + urlFetchMaxRetries = 3 // retries after first attempt (4 attempts total) + urlFetchMaxBackoff = 10 * time.Second // cap for exponential backoff (steps start at 1s) +) + +// SyncFromURL fetches the upstream pricing datasheet, persists it to the DB +// (when configStore != nil), and refreshes the in-memory cache + derived +// datasheet view. On URL failure it falls back to existing DB records when +// any exist, otherwise propagates the error. +// +// The composer owns the distributed lock, the ticker, and the after-sync +// gossip hook — none of that lives here. SyncFromURL is the pure +// "URL → DB → memory" step. +func (s *Store) SyncFromURL(ctx context.Context) error { + pricingData, err := withRetries(ctx, urlFetchMaxRetries, urlFetchMaxBackoff, func() (map[string]Entry, error) { + return s.loadPricingFromURL(ctx) + }) + if err != nil { + // URL failed — fall back to existing DB records when we have them. + if s.configStore != nil { + records, dbErr := s.configStore.GetModelPrices(ctx) + if dbErr != nil { + return fmt.Errorf("failed to get pricing records: %w", dbErr) + } + if len(records) > 0 { + if s.logger != nil { + s.logger.Warn("failed to fetch pricing from URL, falling back to existing database records: %v", err) + } + return nil + } + } + return fmt.Errorf("failed to load pricing data from URL and no existing data available: %w", err) + } + + if s.configStore != nil { + err = s.configStore.ExecuteTransaction(ctx, func(tx *gorm.DB) error { + seen := make(map[string]struct{}) + for modelKey, entry := range pricingData { + pricing := convertEntryToTablePricing(modelKey, entry) + key := makeKey(pricing.Model, pricing.Provider, pricing.Mode) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + if err := s.configStore.UpsertModelPrices(ctx, &pricing, tx); err != nil { + return fmt.Errorf("failed to create pricing record for model %s: %w", pricing.Model, err) + } + } + return nil + }) + if err != nil { + return fmt.Errorf("failed to sync pricing data to database: %w", err) + } + + if err := s.LoadFromDB(ctx); err != nil { + return fmt.Errorf("failed to reload pricing cache: %w", err) + } + } else { + // No config store — apply the parsed data directly to in-memory state. + s.applyPricingData(pricingData) + } + + // Populate provider-utils model params cache from any max_output_tokens + // fields in the pricing entries so providers can read those without a + // separate model-parameters sync round-trip. + s.populateModelParamsFromPricing(pricingData) + + if s.logger != nil { + s.logger.Debug("successfully synced %d pricing records", len(pricingData)) + } + return nil +} + +// LoadFromDB reloads the in-memory pricing cache + datasheet view from the +// config store. Used by the composer at bootstrap and as the gossip +// ReloadFromDB handler on non-leader pods. +func (s *Store) LoadFromDB(ctx context.Context) error { + if s.configStore == nil { + return nil + } + records, err := s.configStore.GetModelPrices(ctx) + if err != nil { + return fmt.Errorf("failed to load pricing from database: %w", err) + } + + s.mu.Lock() + s.pricingData = make(map[string]configstoreTables.TableModelPricing, len(records)) + for _, pricing := range records { + key := makeKey(pricing.Model, pricing.Provider, pricing.Mode) + s.pricingData[key] = pricing + } + s.rebuildDatasheetViewUnsafe() + s.mu.Unlock() + + if s.logger != nil { + s.logger.Debug("loaded %d pricing records from database into memory", len(records)) + } + return nil +} + +// LoadFromURLIntoMemory loads pricing from the URL directly into memory +// (no DB). Used when the composer was built without a config store. +func (s *Store) LoadFromURLIntoMemory(ctx context.Context) error { + pricingData, err := withRetries(ctx, urlFetchMaxRetries, urlFetchMaxBackoff, func() (map[string]Entry, error) { + return s.loadPricingFromURL(ctx) + }) + if err != nil { + return fmt.Errorf("failed to load pricing data from URL: %w", err) + } + s.applyPricingData(pricingData) + s.populateModelParamsFromPricing(pricingData) + return nil +} + +// applyPricingData replaces the in-memory pricing cache + datasheet view +// from a freshly-parsed URL payload. Used by the LoadFromURLIntoMemory +// path and the no-configstore SyncFromURL fallback. +func (s *Store) applyPricingData(pricingData map[string]Entry) { + s.mu.Lock() + s.pricingData = make(map[string]configstoreTables.TableModelPricing, len(pricingData)) + for modelKey, entry := range pricingData { + pricing := convertEntryToTablePricing(modelKey, entry) + key := makeKey(pricing.Model, pricing.Provider, pricing.Mode) + s.pricingData[key] = pricing + } + s.rebuildDatasheetViewUnsafe() + s.mu.Unlock() +} + +// loadPricingFromURL fetches and parses the pricing datasheet at the +// configured URL. Honors ctx for cancellation. +func (s *Store) loadPricingFromURL(ctx context.Context) (map[string]Entry, error) { + s.syncCfgMu.RLock() + rawURL := s.url + s.syncCfgMu.RUnlock() + + parsed, err := url.Parse(rawURL) + if err != nil { + return nil, fmt.Errorf("failed to parse pricing URL: %w", err) + } + + var data []byte + + if parsed.Scheme == "file" { + data, err = os.ReadFile(parsed.Path) + if err != nil { + return nil, fmt.Errorf("failed to read pricing file: %w", err) + } + } else { + if err := bifrost.ValidateExternalURL(rawURL, true); err != nil { + return nil, fmt.Errorf("pricing URL validation failed: %w", err) + } + client := &http.Client{Timeout: DefaultPricingTimeout} + req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.URL(), nil) + if err != nil { + return nil, fmt.Errorf("failed to create HTTP request: %w", err) + } + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to download pricing data: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to download pricing data: HTTP %d", resp.StatusCode) + } + data, err = io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read pricing data response: %w", err) + } + } + var pricingData map[string]Entry + if err := json.Unmarshal(data, &pricingData); err != nil { + return nil, fmt.Errorf("failed to unmarshal pricing data: %w", err) + } + if s.logger != nil { + s.logger.Debug("successfully downloaded and parsed %d pricing records", len(pricingData)) + } + + return pricingData, nil +} + +// populateModelParamsFromPricing extracts max_output_tokens from pricing +// entries and seeds the provider-utils model params cache so providers can +// look up max output tokens without a separate model-parameters sync. +func (s *Store) populateModelParamsFromPricing(pricingData map[string]Entry) { + modelParamsEntries := make(map[string]providerUtils.ModelParams) + for modelKey, entry := range pricingData { + if entry.MaxOutputTokens != nil { + modelName := extractModelName(modelKey) + modelParamsEntries[modelName] = providerUtils.ModelParams{ + MaxOutputTokens: entry.MaxOutputTokens, + } + } + } + if len(modelParamsEntries) > 0 { + providerUtils.BulkSetModelParams(modelParamsEntries) + if s.logger != nil { + s.logger.Debug("populated %d model params entries from pricing datasheet", len(modelParamsEntries)) + } + } +} diff --git a/framework/modelcatalog/datasheet/types.go b/framework/modelcatalog/datasheet/types.go new file mode 100644 index 0000000000..b1a520252b --- /dev/null +++ b/framework/modelcatalog/datasheet/types.go @@ -0,0 +1,724 @@ +// Package pricing owns the pricing/model-parameters catalog (compartments A + B + E): +// canonical pricing rows fetched from the upstream datasheet, per-provider +// datasheet-derived model views, supported request types and parameters, +// and scoped pricing overrides. It also computes per-response cost. +// +// The package performs no list-models I/O — that's the live store's domain. +// The hourly sync ticker lives on the composer (ModelCatalog), not here; the +// composer calls SyncFromURL / LoadFromDB / Sync*ModelParams*. Reads are +// hot-path and lock-free where possible. +package datasheet + +import ( + "context" + "slices" + "strings" + "time" + + "github.com/bytedance/sonic" + "github.com/maximhq/bifrost/core/schemas" + configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" +) + +// Tier boundaries for tiered token pricing. Matches the upstream datasheet +// keys (input_cost_per_token_above_k_tokens). +const ( + TokenTierAbove272K = 272000 + TokenTierAbove200K = 200000 + TokenTierAbove128K = 128000 +) + +// retryBackoffMin is the initial wait before the first retry; subsequent +// retries scale exponentially up to maxBackoff. +const retryBackoffMin = time.Second + +// Entry represents a single model's pricing information. Field names and +// JSON tags match the datasheet schema exactly. AdditionalAttributes carries +// editorial metadata stored on the pricing row — never populated from the +// URL datasheet, only from DB reads via the management API. +type Entry struct { + BaseModel string `json:"base_model,omitempty"` + Provider string `json:"provider"` + Mode string `json:"mode"` + + ContextLength *int `json:"context_length,omitempty"` + MaxInputTokens *int `json:"max_input_tokens,omitempty"` + MaxOutputTokens *int `json:"max_output_tokens,omitempty"` + Architecture *schemas.Architecture `json:"architecture,omitempty"` + + // AdditionalAttributes carries editorial metadata stored on the pricing + // row (e.g. description). Populated from the DB read path only; the + // json:"-" tag prevents URL datasheet payloads from ever feeding into + // this field via json.Unmarshal. + AdditionalAttributes map[string]string `json:"-"` + + Options +} + +// UnmarshalJSON handles the special case where search_context_cost_per_query +// may arrive as either a plain float64 or a tiered object +// {"search_context_size_low":…, "search_context_size_medium":…, "search_context_size_high":…}. +func (p *Entry) UnmarshalJSON(data []byte) error { + type entryAlias Entry + var raw struct { + entryAlias + SearchContextCostPerQuery *struct { + Low *float64 `json:"search_context_size_low"` + Medium *float64 `json:"search_context_size_medium"` + High *float64 `json:"search_context_size_high"` + } `json:"search_context_cost_per_query,omitempty"` + } + if err := sonic.Unmarshal(data, &raw); err != nil { + return err + } + *p = Entry(raw.entryAlias) + + // search_context_cost_per_query arrives as a tiered object — all three values are + // equal for non-Perplexity providers; we prefer medium, then low, then high. + // Perplexity always returns a pre-computed total_cost so the per-query rate is + // never consumed for that provider. + if q := raw.SearchContextCostPerQuery; q != nil { + switch { + case q.Medium != nil: + p.SearchContextCostPerQuery = q.Medium + case q.Low != nil: + p.SearchContextCostPerQuery = q.Low + case q.High != nil: + p.SearchContextCostPerQuery = q.High + } + } + return nil +} + +// Options holds every individual cost field. Embedded into Entry and reused +// as the patch shape for Override. +type Options struct { + // Costs - Text + InputCostPerToken *float64 `json:"input_cost_per_token,omitempty"` + OutputCostPerToken *float64 `json:"output_cost_per_token,omitempty"` + InputCostPerTokenBatches *float64 `json:"input_cost_per_token_batches,omitempty"` + OutputCostPerTokenBatches *float64 `json:"output_cost_per_token_batches,omitempty"` + InputCostPerTokenPriority *float64 `json:"input_cost_per_token_priority,omitempty"` + OutputCostPerTokenPriority *float64 `json:"output_cost_per_token_priority,omitempty"` + InputCostPerTokenFlex *float64 `json:"input_cost_per_token_flex,omitempty"` + OutputCostPerTokenFlex *float64 `json:"output_cost_per_token_flex,omitempty"` + // Fast mode (Anthropic research preview, speed:"fast" on Opus 4.6/4.7/4.8). + // Flat rate across the full context window — no 128k/200k/272k tiering. + InputCostPerTokenFast *float64 `json:"input_cost_per_token_fast,omitempty"` + OutputCostPerTokenFast *float64 `json:"output_cost_per_token_fast,omitempty"` + InputCostPerCharacter *float64 `json:"input_cost_per_character,omitempty"` + // Costs - 128k Tier + InputCostPerTokenAbove128kTokens *float64 `json:"input_cost_per_token_above_128k_tokens,omitempty"` + InputCostPerImageAbove128kTokens *float64 `json:"input_cost_per_image_above_128k_tokens,omitempty"` + InputCostPerVideoPerSecondAbove128kTokens *float64 `json:"input_cost_per_video_per_second_above_128k_tokens,omitempty"` + InputCostPerAudioPerSecondAbove128kTokens *float64 `json:"input_cost_per_audio_per_second_above_128k_tokens,omitempty"` + OutputCostPerTokenAbove128kTokens *float64 `json:"output_cost_per_token_above_128k_tokens,omitempty"` + // Costs - 200k Tier + InputCostPerTokenAbove200kTokens *float64 `json:"input_cost_per_token_above_200k_tokens,omitempty"` + InputCostPerTokenAbove200kTokensPriority *float64 `json:"input_cost_per_token_above_200k_tokens_priority,omitempty"` + OutputCostPerTokenAbove200kTokens *float64 `json:"output_cost_per_token_above_200k_tokens,omitempty"` + OutputCostPerTokenAbove200kTokensPriority *float64 `json:"output_cost_per_token_above_200k_tokens_priority,omitempty"` + // Costs - 272k Tier + InputCostPerTokenAbove272kTokens *float64 `json:"input_cost_per_token_above_272k_tokens,omitempty"` + InputCostPerTokenAbove272kTokensPriority *float64 `json:"input_cost_per_token_above_272k_tokens_priority,omitempty"` + OutputCostPerTokenAbove272kTokens *float64 `json:"output_cost_per_token_above_272k_tokens,omitempty"` + OutputCostPerTokenAbove272kTokensPriority *float64 `json:"output_cost_per_token_above_272k_tokens_priority,omitempty"` + + // Costs - Cache + CacheCreationInputTokenCost *float64 `json:"cache_creation_input_token_cost,omitempty"` + CacheReadInputTokenCost *float64 `json:"cache_read_input_token_cost,omitempty"` + CacheCreationInputTokenCostAbove200kTokens *float64 `json:"cache_creation_input_token_cost_above_200k_tokens,omitempty"` + CacheReadInputTokenCostAbove200kTokens *float64 `json:"cache_read_input_token_cost_above_200k_tokens,omitempty"` + CacheReadInputTokenCostAbove200kTokensPriority *float64 `json:"cache_read_input_token_cost_above_200k_tokens_priority,omitempty"` + CacheCreationInputTokenCostAbove1hr *float64 `json:"cache_creation_input_token_cost_above_1hr,omitempty"` + CacheCreationInputTokenCostAbove1hrAbove200kTokens *float64 `json:"cache_creation_input_token_cost_above_1hr_above_200k_tokens,omitempty"` + CacheCreationInputAudioTokenCost *float64 `json:"cache_creation_input_audio_token_cost,omitempty"` + CacheReadInputTokenCostPriority *float64 `json:"cache_read_input_token_cost_priority,omitempty"` + CacheReadInputTokenCostFlex *float64 `json:"cache_read_input_token_cost_flex,omitempty"` + CacheReadInputImageTokenCost *float64 `json:"cache_read_input_image_token_cost,omitempty"` + CacheReadInputTokenCostAbove272kTokens *float64 `json:"cache_read_input_token_cost_above_272k_tokens,omitempty"` + CacheReadInputTokenCostAbove272kTokensPriority *float64 `json:"cache_read_input_token_cost_above_272k_tokens_priority,omitempty"` + + // Costs - Image + InputCostPerImage *float64 `json:"input_cost_per_image,omitempty"` + InputCostPerPixel *float64 `json:"input_cost_per_pixel,omitempty"` + OutputCostPerImage *float64 `json:"output_cost_per_image,omitempty"` + OutputCostPerPixel *float64 `json:"output_cost_per_pixel,omitempty"` + OutputCostPerImagePremiumImage *float64 `json:"output_cost_per_image_premium_image,omitempty"` + OutputCostPerImageAbove512x512Pixels *float64 `json:"output_cost_per_image_above_512_and_512_pixels,omitempty"` + OutputCostPerImageAbove512x512PixelsPremium *float64 `json:"output_cost_per_image_above_512_and_512_pixels_and_premium_image,omitempty"` + OutputCostPerImageAbove1024x1024Pixels *float64 `json:"output_cost_per_image_above_1024_and_1024_pixels,omitempty"` + OutputCostPerImageAbove1024x1024PixelsPremium *float64 `json:"output_cost_per_image_above_1024_and_1024_pixels_and_premium_image,omitempty"` + OutputCostPerImageAbove2048x2048Pixels *float64 `json:"output_cost_per_image_above_2048_and_2048_pixels,omitempty"` + OutputCostPerImageAbove4096x4096Pixels *float64 `json:"output_cost_per_image_above_4096_and_4096_pixels,omitempty"` + OutputCostPerImageLowQuality *float64 `json:"output_cost_per_image_low_quality,omitempty"` + OutputCostPerImageMediumQuality *float64 `json:"output_cost_per_image_medium_quality,omitempty"` + OutputCostPerImageHighQuality *float64 `json:"output_cost_per_image_high_quality,omitempty"` + OutputCostPerImageAutoQuality *float64 `json:"output_cost_per_image_auto_quality,omitempty"` + InputCostPerImageToken *float64 `json:"input_cost_per_image_token,omitempty"` + OutputCostPerImageToken *float64 `json:"output_cost_per_image_token,omitempty"` + + // Costs - Audio/Video + InputCostPerAudioToken *float64 `json:"input_cost_per_audio_token,omitempty"` + InputCostPerAudioPerSecond *float64 `json:"input_cost_per_audio_per_second,omitempty"` + InputCostPerSecond *float64 `json:"input_cost_per_second,omitempty"` + InputCostPerVideoPerSecond *float64 `json:"input_cost_per_video_per_second,omitempty"` + OutputCostPerAudioToken *float64 `json:"output_cost_per_audio_token,omitempty"` + OutputCostPerVideoPerSecond *float64 `json:"output_cost_per_video_per_second,omitempty"` + OutputCostPerSecond *float64 `json:"output_cost_per_second,omitempty"` + + // Costs - Other. + // + // SearchContextCostPerQuery is stored as a single float64, but the upstream datasheet + // represents it as a tiered object. See Entry.UnmarshalJSON. + SearchContextCostPerQuery *float64 `json:"search_context_cost_per_query,omitempty"` + CodeInterpreterCostPerSession *float64 `json:"code_interpreter_cost_per_session,omitempty"` + + // Costs - OCR + OCRCostPerPage *float64 `json:"ocr_cost_per_page,omitempty"` + AnnotationCostPerPage *float64 `json:"annotation_cost_per_page,omitempty"` +} + +// LookupScopes carries the runtime identifiers used to resolve scoped pricing +// overrides during cost calculation. +type LookupScopes struct { + VirtualKeyID string + SelectedKeyID string + Provider string +} + +// LookupScopesFromContext builds a LookupScopes from a BifrostContext. Reads +// the governance virtual key ID (not the raw VK token) and the selected key +// ID. provider should be the provider name string (e.g. "openai"); pass "" if +// unavailable. Returns nil only when ctx is nil. An empty scopes value is +// still returned when all fields are empty so global-scope overrides remain +// evaluable. +// +// NOT SAFE in a goroutine — reads from ctx which is cancelled when the +// request ends. Call synchronously in PostHooks and pass the result by value +// to anything that may outlive the request. +func LookupScopesFromContext(ctx *schemas.BifrostContext, provider string) *LookupScopes { + if ctx == nil { + return nil + } + virtualKeyID, _ := ctx.Value(schemas.BifrostContextKeyGovernanceVirtualKeyID).(string) + selectedKeyID, _ := ctx.Value(schemas.BifrostContextKeySelectedKeyID).(string) + return &LookupScopes{ + VirtualKeyID: virtualKeyID, + SelectedKeyID: selectedKeyID, + Provider: provider, + } +} + +// ScopeKind identifies which governance scope an override applies to. +type ScopeKind string + +const ( + ScopeKindGlobal ScopeKind = "global" + ScopeKindProvider ScopeKind = "provider" + ScopeKindProviderKey ScopeKind = "provider_key" + ScopeKindVirtualKey ScopeKind = "virtual_key" + ScopeKindVirtualKeyProvider ScopeKind = "virtual_key_provider" + ScopeKindVirtualKeyProviderKey ScopeKind = "virtual_key_provider_key" +) + +// MatchType controls how an override pattern is matched against model names. +type MatchType string + +const ( + MatchTypeExact MatchType = "exact" + MatchTypeWildcard MatchType = "wildcard" +) + +// Override describes a scoped pricing override shared across config storage, +// model catalog compilation, and governance APIs. +type Override struct { + ID string `json:"id"` + Name string `json:"name"` + ScopeKind ScopeKind `json:"scope_kind"` + VirtualKeyID *string `json:"virtual_key_id,omitempty"` + ProviderID *string `json:"provider_id,omitempty"` + ProviderKeyID *string `json:"provider_key_id,omitempty"` + MatchType MatchType `json:"match_type"` + Pattern string `json:"pattern"` + RequestTypes []schemas.RequestType `json:"request_types,omitempty"` + Options Options `json:"options"` +} + +// serviceTier captures the OpenAI service_tier value from a response. +// Add new tier flags here as OpenAI introduces them. +type serviceTier struct { + isPriority bool // true when service_tier == "priority" + isFlex bool // true when service_tier == "flex" + isFast bool // true when usage.speed == "fast" (Anthropic fast mode) +} + +// costInput holds the extracted usage data from a BifrostResponse, +// normalized for the pricing engine. +type costInput struct { + usage *schemas.BifrostLLMUsage + audioTextInputChars int + audioSeconds *int + audioTokenDetails *schemas.TranscriptionUsageInputTokenDetails + imageUsage *schemas.ImageUsage + imageSize string // e.g. "1024x1024", used for per-pixel pricing + imageQuality string // "low", "medium", "high", "auto" (gpt-image-1.5); empty = use base rate + videoSeconds *int + ocrProcessedPages *int + ocrIsAnnotated *bool + // containerIdentifierString, when non-empty, replaces the actual requested/resolved + // model names during pricing lookup. Used for request types whose cost is not + // tied to a specific model. Currently only used for container creates. + containerIdentifierString string + tier serviceTier +} + +// customPricingEntry is one flattened override ready for lookup. +type customPricingEntry struct { + id string + scopeKind ScopeKind + virtualKeyID string + providerID string + providerKeyID string + pattern string // exact model name, or wildcard prefix (trailing * stripped) + wildcard bool + requestModes map[string]struct{} // always non-nil for valid overrides + options Options +} + +// customPricingData is the in-memory lookup structure for pricing overrides. +// Exact matches are indexed by model name; wildcards are a flat slice. +type customPricingData struct { + exact map[string][]customPricingEntry + wildcard []customPricingEntry +} + +// modelParametersParseResult is the parsed result type used by +// buildSupportedOutputsIndex (consumed by params.go's applyModelParameters). +type modelParametersParseResult struct { + Mode *string `json:"mode,omitempty"` + SupportedEndpoints []string `json:"supported_endpoints,omitempty"` + ModelParameters []struct { + ID string `json:"id"` + } `json:"model_parameters,omitempty"` + SupportsAssistantPrefill *bool `json:"supports_assistant_prefill,omitempty"` + SupportsFunctionCalling *bool `json:"supports_function_calling,omitempty"` + SupportsParallelFunctionCalling *bool `json:"supports_parallel_function_calling,omitempty"` + SupportsToolChoice *bool `json:"supports_tool_choice,omitempty"` + SupportsReasoning *bool `json:"supports_reasoning,omitempty"` + SupportsResponseSchema *bool `json:"supports_response_schema,omitempty"` + SupportsServiceTier *bool `json:"supports_service_tier,omitempty"` + SupportsPromptCaching *bool `json:"supports_prompt_caching,omitempty"` + VertexMultiRegionOnly *bool `json:"vertex_multi_region_only,omitempty"` +} + +// --- private helpers (shared across pricing/*.go files) --- + +// makeKey is the composite map key used by pricingData: model|provider|mode. +func makeKey(model, provider, mode string) string { + return model + "|" + provider + "|" + mode +} + +// normalizeProvider folds upstream-datasheet provider name variants +// (vertex_ai, google-vertex, etc.) onto bifrost's canonical provider names. +func normalizeProvider(p string) string { + switch { + case strings.Contains(p, "vertex_ai") || p == "google-vertex": + return string(schemas.Vertex) + case strings.Contains(p, "bedrock"): + return string(schemas.Bedrock) + case strings.Contains(p, "cohere"): + return string(schemas.Cohere) + case strings.Contains(p, "runwayml"): + return string(schemas.Runway) + case strings.Contains(p, "fireworks_ai"): + return string(schemas.Fireworks) + default: + return p + } +} + +// normalizeRequestType collapses streaming and non-streaming variants of a +// request type to a single pricing mode string. +func normalizeRequestType(reqType schemas.RequestType) string { + switch reqType { + case schemas.TextCompletionRequest, schemas.TextCompletionStreamRequest: + return "completion" + case schemas.ChatCompletionRequest, schemas.ChatCompletionStreamRequest: + return "chat" + case schemas.ResponsesRequest, schemas.ResponsesStreamRequest, schemas.WebSocketResponsesRequest, schemas.RealtimeRequest, schemas.CompactionRequest: + return "responses" + case schemas.EmbeddingRequest: + return "embedding" + case schemas.RerankRequest: + return "rerank" + case schemas.SpeechRequest, schemas.SpeechStreamRequest: + return "audio_speech" + case schemas.TranscriptionRequest, schemas.TranscriptionStreamRequest: + return "audio_transcription" + case schemas.ImageGenerationRequest, schemas.ImageGenerationStreamRequest, schemas.ImageVariationRequest: + return "image_generation" + case schemas.ImageEditRequest, schemas.ImageEditStreamRequest: + return "image_edit" + case schemas.VideoGenerationRequest, schemas.VideoRemixRequest: + return "video_generation" + case schemas.OCRRequest: + return "ocr" + case schemas.ContainerCreateRequest: + return "container_create" + } + return "unknown" +} + +// normalizeStreamRequestType maps a stream variant to its non-stream base type. +// Idempotent — passing a non-stream type returns it unchanged. +func normalizeStreamRequestType(rt schemas.RequestType) schemas.RequestType { + switch rt { + case schemas.TextCompletionStreamRequest: + return schemas.TextCompletionRequest + case schemas.ChatCompletionStreamRequest: + return schemas.ChatCompletionRequest + case schemas.ResponsesStreamRequest, schemas.WebSocketResponsesRequest: + return schemas.ResponsesRequest + case schemas.RealtimeRequest: + return schemas.RealtimeRequest + case schemas.SpeechStreamRequest: + return schemas.SpeechRequest + case schemas.TranscriptionStreamRequest: + return schemas.TranscriptionRequest + case schemas.ImageGenerationStreamRequest: + return schemas.ImageGenerationRequest + case schemas.ImageEditStreamRequest: + return schemas.ImageEditRequest + default: + return rt + } +} + +// extractModelName strips a leading "provider/" prefix from a model key. +func extractModelName(modelKey string) string { + if idx := strings.Index(modelKey, "/"); idx >= 0 { + return modelKey[idx+1:] + } + return modelKey +} + +// normalizeEndpointToOutputType converts a supported_endpoints URL path to a +// normalized output type. Empty string for unrecognized endpoints. +func normalizeEndpointToOutputType(endpoint string) string { + switch { + case strings.Contains(endpoint, "/chat/completions"): + return "chat_completion" + case strings.Contains(endpoint, "/responses"): + return "responses" + case strings.Contains(endpoint, "/completions"): + return "text_completion" + default: + return "" + } +} + +// normalizeModeToOutputType converts mode to a normalized output type. +func normalizeModeToOutputType(mode string) string { + switch mode { + case "chat": + return "chat_completion" + case "completion": + return "text_completion" + case "responses": + return "responses" + default: + return "" + } +} + +// extractSupportedParams builds a list of supported OpenAI-compatible parameter +// names from model_parameters[].id values and supports_* boolean flags. +func extractSupportedParams(parsed *modelParametersParseResult) []string { + var supported []string + addParam := func(name string) { + if !slices.Contains(supported, name) { + supported = append(supported, name) + } + } + + for _, mp := range parsed.ModelParameters { + switch mp.ID { + case "reasoning_effort", "reasoning_summary": + addParam("reasoning") + case "web_search": + addParam("web_search_options") + case "promptTools", "image_detail", "stream": + // skip — not top-level request parameters + default: + addParam(mp.ID) + } + } + + if parsed.SupportsAssistantPrefill != nil && *parsed.SupportsAssistantPrefill { + // Not an actual request parameter; if present, trailing assistant messages + // for anthropic and bedrock's anthropic models will not be trimmed. + addParam("assistant_prefill") + } + if parsed.SupportsFunctionCalling != nil && *parsed.SupportsFunctionCalling { + addParam("tools") + } + if parsed.SupportsParallelFunctionCalling != nil && *parsed.SupportsParallelFunctionCalling { + addParam("parallel_tool_calls") + } + if parsed.SupportsToolChoice != nil && *parsed.SupportsToolChoice { + addParam("tool_choice") + } + if parsed.SupportsReasoning != nil && *parsed.SupportsReasoning { + addParam("reasoning") + } + if parsed.SupportsResponseSchema != nil && *parsed.SupportsResponseSchema { + addParam("response_format") + addParam("text") + } + if parsed.SupportsServiceTier != nil && *parsed.SupportsServiceTier { + addParam("service_tier") + } + if parsed.SupportsPromptCaching != nil && *parsed.SupportsPromptCaching { + addParam("cachePoint") + addParam("cache_control") + addParam("prompt_cache_key") + addParam("prompt_cache_retention") + } + + return supported +} + +// withRetries runs op until it succeeds or maxRetries retries are exhausted +// (1 initial attempt + maxRetries retries). After each failure it waits with +// exponential backoff starting at 1 second (retryBackoffMin), capped at +// maxBackoff when > 0. If maxBackoff is zero, the delay grows unbounded. +func withRetries[T any](ctx context.Context, maxRetries int, maxBackoff time.Duration, op func() (T, error)) (T, error) { + var zero T + if maxRetries < 0 { + maxRetries = 0 + } + var lastErr error + for attempt := 0; attempt <= maxRetries; attempt++ { + select { + case <-ctx.Done(): + return zero, ctx.Err() + default: + } + + if attempt > 0 { + backoff := retryBackoffMin * time.Duration(1< 0 && backoff > maxBackoff { + backoff = maxBackoff + } + select { + case <-ctx.Done(): + return zero, ctx.Err() + case <-time.After(backoff): + } + } + v, err := op() + if err == nil { + return v, nil + } + lastErr = err + } + return zero, lastErr +} + +// convertEntryToTablePricing converts a parsed Entry from the upstream +// datasheet into the row shape persisted in the config store. +func convertEntryToTablePricing(modelKey string, entry Entry) configstoreTables.TableModelPricing { + provider := normalizeProvider(entry.Provider) + modelName := extractModelName(modelKey) + return configstoreTables.TableModelPricing{ + Model: modelName, + BaseModel: entry.BaseModel, + Provider: provider, + Mode: entry.Mode, + ContextLength: entry.ContextLength, + MaxInputTokens: entry.MaxInputTokens, + MaxOutputTokens: entry.MaxOutputTokens, + Architecture: entry.Architecture, + + InputCostPerToken: entry.InputCostPerToken, + OutputCostPerToken: entry.OutputCostPerToken, + InputCostPerTokenBatches: entry.InputCostPerTokenBatches, + OutputCostPerTokenBatches: entry.OutputCostPerTokenBatches, + InputCostPerTokenPriority: entry.InputCostPerTokenPriority, + OutputCostPerTokenPriority: entry.OutputCostPerTokenPriority, + InputCostPerTokenFlex: entry.InputCostPerTokenFlex, + OutputCostPerTokenFlex: entry.OutputCostPerTokenFlex, + InputCostPerTokenFast: entry.InputCostPerTokenFast, + OutputCostPerTokenFast: entry.OutputCostPerTokenFast, + InputCostPerTokenAbove200kTokens: entry.InputCostPerTokenAbove200kTokens, + InputCostPerTokenAbove200kTokensPriority: entry.InputCostPerTokenAbove200kTokensPriority, + OutputCostPerTokenAbove200kTokens: entry.OutputCostPerTokenAbove200kTokens, + OutputCostPerTokenAbove200kTokensPriority: entry.OutputCostPerTokenAbove200kTokensPriority, + InputCostPerTokenAbove272kTokens: entry.InputCostPerTokenAbove272kTokens, + InputCostPerTokenAbove272kTokensPriority: entry.InputCostPerTokenAbove272kTokensPriority, + OutputCostPerTokenAbove272kTokens: entry.OutputCostPerTokenAbove272kTokens, + OutputCostPerTokenAbove272kTokensPriority: entry.OutputCostPerTokenAbove272kTokensPriority, + InputCostPerCharacter: entry.InputCostPerCharacter, + InputCostPerTokenAbove128kTokens: entry.InputCostPerTokenAbove128kTokens, + InputCostPerImageAbove128kTokens: entry.InputCostPerImageAbove128kTokens, + InputCostPerVideoPerSecondAbove128kTokens: entry.InputCostPerVideoPerSecondAbove128kTokens, + InputCostPerAudioPerSecondAbove128kTokens: entry.InputCostPerAudioPerSecondAbove128kTokens, + OutputCostPerTokenAbove128kTokens: entry.OutputCostPerTokenAbove128kTokens, + + CacheCreationInputTokenCost: entry.CacheCreationInputTokenCost, + CacheReadInputTokenCost: entry.CacheReadInputTokenCost, + CacheCreationInputTokenCostAbove200kTokens: entry.CacheCreationInputTokenCostAbove200kTokens, + CacheReadInputTokenCostAbove200kTokens: entry.CacheReadInputTokenCostAbove200kTokens, + CacheReadInputTokenCostAbove200kTokensPriority: entry.CacheReadInputTokenCostAbove200kTokensPriority, + CacheCreationInputTokenCostAbove1hr: entry.CacheCreationInputTokenCostAbove1hr, + CacheCreationInputTokenCostAbove1hrAbove200kTokens: entry.CacheCreationInputTokenCostAbove1hrAbove200kTokens, + CacheCreationInputAudioTokenCost: entry.CacheCreationInputAudioTokenCost, + CacheReadInputTokenCostPriority: entry.CacheReadInputTokenCostPriority, + CacheReadInputTokenCostFlex: entry.CacheReadInputTokenCostFlex, + CacheReadInputImageTokenCost: entry.CacheReadInputImageTokenCost, + CacheReadInputTokenCostAbove272kTokens: entry.CacheReadInputTokenCostAbove272kTokens, + CacheReadInputTokenCostAbove272kTokensPriority: entry.CacheReadInputTokenCostAbove272kTokensPriority, + + InputCostPerImage: entry.InputCostPerImage, + InputCostPerPixel: entry.InputCostPerPixel, + OutputCostPerImage: entry.OutputCostPerImage, + OutputCostPerPixel: entry.OutputCostPerPixel, + OutputCostPerImagePremiumImage: entry.OutputCostPerImagePremiumImage, + OutputCostPerImageAbove512x512Pixels: entry.OutputCostPerImageAbove512x512Pixels, + OutputCostPerImageAbove512x512PixelsPremium: entry.OutputCostPerImageAbove512x512PixelsPremium, + OutputCostPerImageAbove1024x1024Pixels: entry.OutputCostPerImageAbove1024x1024Pixels, + OutputCostPerImageAbove1024x1024PixelsPremium: entry.OutputCostPerImageAbove1024x1024PixelsPremium, + OutputCostPerImageAbove2048x2048Pixels: entry.OutputCostPerImageAbove2048x2048Pixels, + OutputCostPerImageAbove4096x4096Pixels: entry.OutputCostPerImageAbove4096x4096Pixels, + OutputCostPerImageLowQuality: entry.OutputCostPerImageLowQuality, + OutputCostPerImageMediumQuality: entry.OutputCostPerImageMediumQuality, + OutputCostPerImageHighQuality: entry.OutputCostPerImageHighQuality, + OutputCostPerImageAutoQuality: entry.OutputCostPerImageAutoQuality, + InputCostPerImageToken: entry.InputCostPerImageToken, + OutputCostPerImageToken: entry.OutputCostPerImageToken, + + InputCostPerAudioToken: entry.InputCostPerAudioToken, + InputCostPerAudioPerSecond: entry.InputCostPerAudioPerSecond, + InputCostPerSecond: entry.InputCostPerSecond, + InputCostPerVideoPerSecond: entry.InputCostPerVideoPerSecond, + OutputCostPerAudioToken: entry.OutputCostPerAudioToken, + OutputCostPerVideoPerSecond: entry.OutputCostPerVideoPerSecond, + OutputCostPerSecond: entry.OutputCostPerSecond, + + SearchContextCostPerQuery: entry.SearchContextCostPerQuery, + CodeInterpreterCostPerSession: entry.CodeInterpreterCostPerSession, + + OCRCostPerPage: entry.OCRCostPerPage, + AnnotationCostPerPage: entry.AnnotationCostPerPage, + } +} + +// convertTablePricingToEntry converts a TableModelPricing row from the DB back +// into the Entry shape callers consume. +func convertTablePricingToEntry(pricing *configstoreTables.TableModelPricing) *Entry { + options := Options{ + InputCostPerToken: pricing.InputCostPerToken, + OutputCostPerToken: pricing.OutputCostPerToken, + InputCostPerTokenBatches: pricing.InputCostPerTokenBatches, + OutputCostPerTokenBatches: pricing.OutputCostPerTokenBatches, + InputCostPerTokenPriority: pricing.InputCostPerTokenPriority, + OutputCostPerTokenPriority: pricing.OutputCostPerTokenPriority, + InputCostPerTokenFlex: pricing.InputCostPerTokenFlex, + OutputCostPerTokenFlex: pricing.OutputCostPerTokenFlex, + InputCostPerTokenFast: pricing.InputCostPerTokenFast, + OutputCostPerTokenFast: pricing.OutputCostPerTokenFast, + InputCostPerTokenAbove200kTokens: pricing.InputCostPerTokenAbove200kTokens, + InputCostPerTokenAbove200kTokensPriority: pricing.InputCostPerTokenAbove200kTokensPriority, + OutputCostPerTokenAbove200kTokens: pricing.OutputCostPerTokenAbove200kTokens, + OutputCostPerTokenAbove200kTokensPriority: pricing.OutputCostPerTokenAbove200kTokensPriority, + InputCostPerTokenAbove272kTokens: pricing.InputCostPerTokenAbove272kTokens, + InputCostPerTokenAbove272kTokensPriority: pricing.InputCostPerTokenAbove272kTokensPriority, + OutputCostPerTokenAbove272kTokens: pricing.OutputCostPerTokenAbove272kTokens, + OutputCostPerTokenAbove272kTokensPriority: pricing.OutputCostPerTokenAbove272kTokensPriority, + InputCostPerCharacter: pricing.InputCostPerCharacter, + InputCostPerTokenAbove128kTokens: pricing.InputCostPerTokenAbove128kTokens, + InputCostPerImageAbove128kTokens: pricing.InputCostPerImageAbove128kTokens, + InputCostPerVideoPerSecondAbove128kTokens: pricing.InputCostPerVideoPerSecondAbove128kTokens, + InputCostPerAudioPerSecondAbove128kTokens: pricing.InputCostPerAudioPerSecondAbove128kTokens, + OutputCostPerTokenAbove128kTokens: pricing.OutputCostPerTokenAbove128kTokens, + + CacheCreationInputTokenCost: pricing.CacheCreationInputTokenCost, + CacheReadInputTokenCost: pricing.CacheReadInputTokenCost, + CacheCreationInputTokenCostAbove200kTokens: pricing.CacheCreationInputTokenCostAbove200kTokens, + CacheReadInputTokenCostAbove200kTokens: pricing.CacheReadInputTokenCostAbove200kTokens, + CacheReadInputTokenCostAbove200kTokensPriority: pricing.CacheReadInputTokenCostAbove200kTokensPriority, + CacheCreationInputTokenCostAbove1hr: pricing.CacheCreationInputTokenCostAbove1hr, + CacheCreationInputTokenCostAbove1hrAbove200kTokens: pricing.CacheCreationInputTokenCostAbove1hrAbove200kTokens, + CacheCreationInputAudioTokenCost: pricing.CacheCreationInputAudioTokenCost, + CacheReadInputTokenCostPriority: pricing.CacheReadInputTokenCostPriority, + CacheReadInputTokenCostFlex: pricing.CacheReadInputTokenCostFlex, + CacheReadInputImageTokenCost: pricing.CacheReadInputImageTokenCost, + CacheReadInputTokenCostAbove272kTokens: pricing.CacheReadInputTokenCostAbove272kTokens, + CacheReadInputTokenCostAbove272kTokensPriority: pricing.CacheReadInputTokenCostAbove272kTokensPriority, + + InputCostPerImage: pricing.InputCostPerImage, + InputCostPerPixel: pricing.InputCostPerPixel, + OutputCostPerImage: pricing.OutputCostPerImage, + OutputCostPerPixel: pricing.OutputCostPerPixel, + OutputCostPerImagePremiumImage: pricing.OutputCostPerImagePremiumImage, + OutputCostPerImageAbove512x512Pixels: pricing.OutputCostPerImageAbove512x512Pixels, + OutputCostPerImageAbove512x512PixelsPremium: pricing.OutputCostPerImageAbove512x512PixelsPremium, + OutputCostPerImageAbove1024x1024Pixels: pricing.OutputCostPerImageAbove1024x1024Pixels, + OutputCostPerImageAbove1024x1024PixelsPremium: pricing.OutputCostPerImageAbove1024x1024PixelsPremium, + OutputCostPerImageAbove2048x2048Pixels: pricing.OutputCostPerImageAbove2048x2048Pixels, + OutputCostPerImageAbove4096x4096Pixels: pricing.OutputCostPerImageAbove4096x4096Pixels, + OutputCostPerImageLowQuality: pricing.OutputCostPerImageLowQuality, + OutputCostPerImageMediumQuality: pricing.OutputCostPerImageMediumQuality, + OutputCostPerImageHighQuality: pricing.OutputCostPerImageHighQuality, + OutputCostPerImageAutoQuality: pricing.OutputCostPerImageAutoQuality, + InputCostPerImageToken: pricing.InputCostPerImageToken, + OutputCostPerImageToken: pricing.OutputCostPerImageToken, + + InputCostPerAudioToken: pricing.InputCostPerAudioToken, + InputCostPerAudioPerSecond: pricing.InputCostPerAudioPerSecond, + InputCostPerSecond: pricing.InputCostPerSecond, + InputCostPerVideoPerSecond: pricing.InputCostPerVideoPerSecond, + OutputCostPerAudioToken: pricing.OutputCostPerAudioToken, + OutputCostPerVideoPerSecond: pricing.OutputCostPerVideoPerSecond, + OutputCostPerSecond: pricing.OutputCostPerSecond, + + SearchContextCostPerQuery: pricing.SearchContextCostPerQuery, + CodeInterpreterCostPerSession: pricing.CodeInterpreterCostPerSession, + + OCRCostPerPage: pricing.OCRCostPerPage, + AnnotationCostPerPage: pricing.AnnotationCostPerPage, + } + return &Entry{ + BaseModel: pricing.BaseModel, + Provider: pricing.Provider, + Mode: pricing.Mode, + ContextLength: pricing.ContextLength, + MaxInputTokens: pricing.MaxInputTokens, + MaxOutputTokens: pricing.MaxOutputTokens, + Architecture: pricing.Architecture, + AdditionalAttributes: pricing.AdditionalAttributes, + Options: options, + } +} + +// convertTableOverride converts a TablePricingOverride to an Override. +func convertTableOverride(override *configstoreTables.TablePricingOverride) (Override, error) { + var options Options + if err := sonic.Unmarshal([]byte(override.PricingPatchJSON), &options); err != nil { + return Override{}, err + } + return Override{ + ID: override.ID, + Name: override.Name, + ScopeKind: ScopeKind(override.ScopeKind), + VirtualKeyID: override.VirtualKeyID, + ProviderID: override.ProviderID, + ProviderKeyID: override.ProviderKeyID, + MatchType: MatchType(override.MatchType), + Pattern: override.Pattern, + RequestTypes: override.RequestTypes, + Options: options, + }, nil +} + diff --git a/framework/modelcatalog/keyconfig/store.go b/framework/modelcatalog/keyconfig/store.go new file mode 100644 index 0000000000..8c29f8bd6f --- /dev/null +++ b/framework/modelcatalog/keyconfig/store.go @@ -0,0 +1,371 @@ +// Package keyconfig caches per-key configuration (allowed/blacklisted +// models, aliases) for every configured provider. It is a pure +// transformation: callers push raw schemas.Key slices in via SetProvider / +// Replace, and the store exposes derived views (aggregated allow/block +// lists, alias-owner index, per-key entries) for routing-time queries. +// +// The store performs no I/O — it does not know about configstore, the +// network, or persistence. The composer (ModelCatalog) owns fetching keys +// from the config store and pushing them in. +// +// Aggregation semantics — provider-level blacklist as the intersection across +// enabled keys, last-enabled-key-wins on alias collisions — are ported from +// bifrost-enterprise/core/loadbalancing/plugin.go and extended with per-key +// alias retention so routing can resolve (provider, model) → (keyID, AliasConfig). +package keyconfig + +import ( + "slices" + "strings" + "sync" + + bifrost "github.com/maximhq/bifrost/core" + "github.com/maximhq/bifrost/core/schemas" +) + +// KeyEntry is the per-key configuration snapshot the store maintains. Slice +// and map fields are owned by the store; callers must not mutate. +type KeyEntry struct { + KeyID string + Enabled bool + Allowed schemas.WhiteList + Blacklisted schemas.BlackList + Aliases schemas.KeyAliases +} + +// AliasOwner identifies which key owns an alias and carries its AliasConfig. +// Routing uses KeyID to pick credentials; Config carries the deployment / +// region overrides that must be applied alongside. +type AliasOwner struct { + KeyID string + Config schemas.AliasConfig +} + +// providerState is the immutable snapshot stored per provider. Writers +// build a fresh providerState off-lock and slot it into the entries map +// under the write lock, so readers always see a consistent set of derived +// views — never a torn mix of pre- and post-refresh state. +type providerState struct { + entries []KeyEntry + allowed schemas.WhiteList + blacklisted schemas.BlackList + aliasIndex map[string]AliasOwner +} + +type Store struct { + // mu serializes writers (Replace / SetProvider / RemoveProvider) and + // gates concurrent readers. Replace holds the write lock for its full + // build-and-swap so readers see either the full old snapshot or the + // full new one — never an interleaving. + mu sync.RWMutex + entries map[schemas.ModelProvider]*providerState + logger schemas.Logger +} + +// New constructs an empty Store. +func New(logger schemas.Logger) *Store { + if logger == nil { + logger = bifrost.NewNoOpLogger() + } + return &Store{ + entries: make(map[schemas.ModelProvider]*providerState), + logger: logger, + } +} + +// Replace resets the store to reflect the snapshot. Providers present in the +// previous state but absent from snapshot are dropped. The new map is +// swapped in atomically under the write lock: readers see either the full +// old snapshot or the full new one, never an interleaving. Use on initial +// load and on full cross-pod resyncs. +func (s *Store) Replace(snapshot map[schemas.ModelProvider][]schemas.Key) { + s.mu.Lock() + defer s.mu.Unlock() + next := make(map[schemas.ModelProvider]*providerState, len(snapshot)) + for p, keys := range snapshot { + if st := s.buildState(p, keys); st != nil { + next[p] = st + } + } + s.entries = next +} + +// SetProvider replaces the cached state for one provider. Call after a +// successful key add / update / delete for that provider. +func (s *Store) SetProvider(provider schemas.ModelProvider, keys []schemas.Key) { + st := s.buildState(provider, keys) + s.mu.Lock() + defer s.mu.Unlock() + if st == nil { + delete(s.entries, provider) + return + } + s.entries[provider] = st +} + +// RemoveProvider drops all state for the provider. Call on provider delete. +func (s *Store) RemoveProvider(provider schemas.ModelProvider) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.entries, provider) +} + +// EntriesFor returns all per-key entries for the provider (including +// individually-disabled keys), or nil when the provider has no routable +// keys at all and was dropped from the store. Returns a defensive slice +// copy; KeyEntry fields share underlying memory with the store and must +// not be mutated. +func (s *Store) EntriesFor(provider schemas.ModelProvider) []KeyEntry { + st := s.load(provider) + if st == nil { + return nil + } + out := make([]KeyEntry, len(st.entries)) + copy(out, st.entries) + return out +} + +// EntryFor returns the entry for one (provider, keyID), or false if absent. +func (s *Store) EntryFor(provider schemas.ModelProvider, keyID string) (KeyEntry, bool) { + st := s.load(provider) + if st == nil { + return KeyEntry{}, false + } + for _, e := range st.entries { + if e.KeyID == keyID { + return e, true + } + } + return KeyEntry{}, false +} + +// AllowedFor returns the aggregated whitelist: union of enabled keys' Models +// minus per-key Blacklisted, or ["*"] when any enabled key is unrestricted or +// the provider is keyless non-standard. +func (s *Store) AllowedFor(provider schemas.ModelProvider) schemas.WhiteList { + st := s.load(provider) + if st == nil { + return nil + } + return slices.Clone(st.allowed) +} + +// BlacklistedFor returns the intersection of enabled keys' BlacklistedModels. +// A model is provider-wide blocked only when *every* enabled key blacklists +// it — matching the LB semantics that a model is only fully unavailable when +// no key can serve it. Entries preserve the original casing of the first key +// that blacklisted the model (the intersection itself is case-insensitive), +// consistent with AllowedFor. +func (s *Store) BlacklistedFor(provider schemas.ModelProvider) schemas.BlackList { + st := s.load(provider) + if st == nil { + return nil + } + return slices.Clone(st.blacklisted) +} + +// IsAllowed reports whether at least one enabled key can actually serve the +// model on this provider — a key whose allow-list permits it and whose +// blacklist does not block it. Returns false for unknown providers (no state ⇒ +// no allowance). This mirrors KeysAllowingModel (true ⇔ KeysAllowingModel +// returns a non-empty set), so it is safe to use as a routing pre-filter: a +// true result guarantees a routable key exists. Keyless unrestricted providers +// (custom providers configured without keys) allow everything, since there is +// no per-key allow-list to route through. +func (s *Store) IsAllowed(provider schemas.ModelProvider, model string) bool { + st := s.load(provider) + if st == nil { + return false + } + // Keyless unrestricted provider: no per-key entries to gate on, but the + // aggregated allow-list ("*") governs and ambient/IAM auth routes without a key. + if len(st.entries) == 0 { + return st.allowed.IsAllowed(model) && !st.blacklisted.IsBlocked(model) + } + return anyKeyAllows(st, model) +} + +// anyKeyAllows reports whether any enabled key in the snapshot can serve the +// model (allowed and not blacklisted). Shares the per-key gating predicate with +// KeysAllowingModel. +func anyKeyAllows(st *providerState, model string) bool { + for _, e := range st.entries { + if !e.Enabled || e.Blacklisted.IsBlockAll() || e.Blacklisted.IsBlocked(model) { + continue + } + if e.Allowed.IsAllowed(model) { + return true + } + } + return false +} + +// ResolveAlias returns which key owns the alias on this provider and its +// AliasConfig. AliasConfig is a value copy — safe to mutate without +// affecting store state (though inner pointer fields like Region remain +// shared; treat the returned Config as read-only). +func (s *Store) ResolveAlias(provider schemas.ModelProvider, model string) (AliasOwner, bool) { + st := s.load(provider) + if st == nil { + return AliasOwner{}, false + } + // aliasIndex is keyed lowercase; match the case-insensitive contract + // of schemas.KeyAliases.ResolveConfig. + owner, ok := st.aliasIndex[strings.ToLower(model)] + return owner, ok +} + +// Providers returns every provider with cached state. Used by callers +// (notably the load balancer) that need to enumerate the routing-eligible +// provider set. +func (s *Store) Providers() []schemas.ModelProvider { + s.mu.RLock() + defer s.mu.RUnlock() + out := make([]schemas.ModelProvider, 0, len(s.entries)) + for p := range s.entries { + out = append(out, p) + } + return out +} + +// KeysAllowingModel returns the IDs of enabled keys whose Allowed list +// includes the model and whose Blacklisted list does not block it. Used by +// routing to skip keys that cannot serve the request without scanning each +// key's Models slice itself. +func (s *Store) KeysAllowingModel(provider schemas.ModelProvider, model string) []string { + st := s.load(provider) + if st == nil { + return nil + } + var out []string + for _, e := range st.entries { + if !e.Enabled || e.Blacklisted.IsBlockAll() || e.Blacklisted.IsBlocked(model) { + continue + } + if e.Allowed.IsAllowed(model) { + out = append(out, e.KeyID) + } + } + return out +} + +// load returns the providerState for one provider under an RLock. Returns +// nil when the provider isn't in the store. The returned *providerState is +// immutable once published — safe to read without holding the lock. +func (s *Store) load(provider schemas.ModelProvider) *providerState { + s.mu.RLock() + defer s.mu.RUnlock() + return s.entries[provider] +} + +// buildState applies the aggregation rules: skip disabled keys and full +// blacklists, union allowed minus per-key blacklisted, intersect blacklists +// across enabled keys for the provider-level set, last-enabled-key-wins on +// alias collisions with a warning. Returns nil when the provider has no +// routable keys (all disabled or fully block-all) and isn't a keyless +// non-standard provider — such providers are dropped from the store +// entirely, since this cache exists for routing-time queries, not +// inspection. The configstore remains the source of truth for the full +// configured-key set. +// +// Safe to call without holding s.mu — it only reads its parameters and +// s.logger, and writes only to local variables. +func (s *Store) buildState(provider schemas.ModelProvider, keys []schemas.Key) *providerState { + var ( + allModelsAllowed bool + enabledKeysCount int + allowed schemas.WhiteList + // blacklistAgg accumulates the cross-key blacklist intersection. Keyed by + // lowercased model for case-insensitive counting; name holds the original + // casing of the first key that blacklisted it, so the emitted blacklist + // preserves casing like allowed does. + blacklistAgg = make(map[string]struct { + count int + name string + }) + aliasIndex = make(map[string]AliasOwner) + entries = make([]KeyEntry, 0, len(keys)) + ) + + // Keyless non-standard providers (custom providers configured without keys) + // are unrestricted — there's no allow-list to derive from. + if len(keys) == 0 && !bifrost.IsStandardProvider(provider) { + allModelsAllowed = true + } + + for _, key := range keys { + enabled := key.Enabled == nil || *key.Enabled + entries = append(entries, KeyEntry{ + KeyID: key.ID, + Enabled: enabled, + Allowed: key.Models, + Blacklisted: key.BlacklistedModels, + Aliases: key.Aliases, + }) + + if !enabled || key.BlacklistedModels.IsBlockAll() { + continue + } + enabledKeysCount++ + + for _, m := range key.BlacklistedModels { + lower := strings.ToLower(m) + agg := blacklistAgg[lower] + if agg.count == 0 { + agg.name = m + } + agg.count++ + blacklistAgg[lower] = agg + } + + if key.Models.IsUnrestricted() { + allModelsAllowed = true + } else { + for _, m := range key.Models { + if key.BlacklistedModels.IsBlocked(m) { + continue + } + if !allowed.Contains(m) { + allowed = append(allowed, m) + } + } + } + + for aliasName, cfg := range key.Aliases { + // Normalize to lowercase so the cross-key alias index matches the + // case-insensitive semantic of schemas.KeyAliases.ResolveConfig + // and schemas.KeyAliases.Validate's intra-key uniqueness check. + // Without this, two keys that differ only in alias casing would + // silently both land in the index and the collision warning + // would never fire. + normalizedAlias := strings.ToLower(aliasName) + if prev, exists := aliasIndex[normalizedAlias]; exists && prev.KeyID != key.ID { + s.logger.Debug("keyconfig: alias %q on provider %s defined by both key %s and key %s; last enabled key wins", + aliasName, provider, prev.KeyID, key.ID) + } + aliasIndex[normalizedAlias] = AliasOwner{KeyID: key.ID, Config: cfg} + } + } + + if enabledKeysCount == 0 && !allModelsAllowed { + return nil + } + + if allModelsAllowed { + allowed = schemas.WhiteList{"*"} + } + + var blacklisted schemas.BlackList + for _, agg := range blacklistAgg { + if agg.count == enabledKeysCount { + blacklisted = append(blacklisted, agg.name) + } + } + + return &providerState{ + entries: entries, + allowed: allowed, + blacklisted: blacklisted, + aliasIndex: aliasIndex, + } +} diff --git a/framework/modelcatalog/keyconfig/store_test.go b/framework/modelcatalog/keyconfig/store_test.go new file mode 100644 index 0000000000..e368fa9e30 --- /dev/null +++ b/framework/modelcatalog/keyconfig/store_test.go @@ -0,0 +1,721 @@ +package keyconfig + +import ( + "slices" + "sort" + "sync" + "testing" + + "github.com/maximhq/bifrost/core/schemas" +) + +// --- test logger --- + +type recordingLogger struct { + mu sync.Mutex + debugs []string +} + +func (l *recordingLogger) Debug(format string, args ...any) { + l.mu.Lock() + defer l.mu.Unlock() + l.debugs = append(l.debugs, format) +} +func (l *recordingLogger) Info(format string, args ...any) {} +func (l *recordingLogger) Warn(format string, args ...any) {} +func (l *recordingLogger) Error(format string, args ...any) {} +func (l *recordingLogger) Fatal(format string, args ...any) {} +func (l *recordingLogger) SetLevel(level schemas.LogLevel) {} +func (l *recordingLogger) SetOutputType(outputType schemas.LoggerOutputType) {} +func (l *recordingLogger) LogHTTPRequest(level schemas.LogLevel, msg string) schemas.LogEventBuilder { + return schemas.NoopLogEvent +} + +func (l *recordingLogger) DebugCount() int { + l.mu.Lock() + defer l.mu.Unlock() + return len(l.debugs) +} + +func ptrBool(b bool) *bool { return &b } + +func newStoreFromFixture(fixture map[schemas.ModelProvider][]schemas.Key) (*Store, *recordingLogger) { + log := &recordingLogger{} + s := New(log) + s.Replace(fixture) + return s, log +} + +// --- behavioral tests --- + +func TestEmptyKeysAndStandardProviderRemoves(t *testing.T) { + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: nil, + }) + if got := s.AllowedFor(schemas.OpenAI); got != nil { + t.Errorf("standard provider with no keys: AllowedFor = %v, want nil", got) + } + if s.IsAllowed(schemas.OpenAI, "gpt-4o") { + t.Error("IsAllowed = true for standard provider with no keys") + } +} + +func TestKeylessNonStandardProviderUnrestricted(t *testing.T) { + custom := schemas.ModelProvider("my-custom") + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{ + custom: nil, + }) + allowed := s.AllowedFor(custom) + if !slices.Equal(allowed, schemas.WhiteList{"*"}) { + t.Errorf("keyless custom: AllowedFor = %v, want [*]", allowed) + } + if !s.IsAllowed(custom, "anything") { + t.Error("keyless custom: IsAllowed = false for arbitrary model") + } +} + +func TestUnrestrictedKeyImpliesWildcard(t *testing.T) { + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: { + {ID: "k1", Enabled: ptrBool(true), Models: schemas.WhiteList{"*"}}, + }, + }) + if got := s.AllowedFor(schemas.OpenAI); !slices.Equal(got, schemas.WhiteList{"*"}) { + t.Errorf("AllowedFor = %v, want [*]", got) + } + if !s.IsAllowed(schemas.OpenAI, "gpt-4o") { + t.Error("IsAllowed = false on wildcard key") + } +} + +func TestExplicitAllowFiltersBlacklistedPerKey(t *testing.T) { + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: { + { + ID: "k1", + Enabled: ptrBool(true), + Models: schemas.WhiteList{"gpt-4o", "o1"}, + BlacklistedModels: schemas.BlackList{"o1"}, + }, + }, + }) + allowed := s.AllowedFor(schemas.OpenAI) + sort.Strings(allowed) + if !slices.Equal(allowed, schemas.WhiteList{"gpt-4o"}) { + t.Errorf("AllowedFor = %v, want [gpt-4o] (o1 filtered by key blacklist)", allowed) + } +} + +func TestBlacklistIntersectionAcrossKeys(t *testing.T) { + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: { + {ID: "k1", Enabled: ptrBool(true), Models: schemas.WhiteList{"gpt-4o", "o1"}, + BlacklistedModels: schemas.BlackList{"o1", "gpt-3.5"}}, + {ID: "k2", Enabled: ptrBool(true), Models: schemas.WhiteList{"gpt-4o"}, + BlacklistedModels: schemas.BlackList{"o1"}}, + }, + }) + // o1 is blacklisted by both → provider-level blocked. gpt-3.5 only by one → not. + bl := s.BlacklistedFor(schemas.OpenAI) + sort.Strings(bl) + if !slices.Equal(bl, schemas.BlackList{"o1"}) { + t.Errorf("BlacklistedFor = %v, want [o1] (intersection)", bl) + } + if s.IsAllowed(schemas.OpenAI, "o1") { + t.Error("IsAllowed o1 = true, want false (provider-blocked)") + } +} + +func TestDisabledKeyDoesNotAffectAggregates(t *testing.T) { + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: { + {ID: "k1", Enabled: ptrBool(true), Models: schemas.WhiteList{"gpt-4o"}}, + {ID: "k2", Enabled: ptrBool(false), Models: schemas.WhiteList{"o1"}, BlacklistedModels: schemas.BlackList{"gpt-4o"}}, + }, + }) + allowed := s.AllowedFor(schemas.OpenAI) + if !slices.Equal(allowed, schemas.WhiteList{"gpt-4o"}) { + t.Errorf("AllowedFor = %v, want [gpt-4o] (k2 disabled)", allowed) + } + if !s.IsAllowed(schemas.OpenAI, "gpt-4o") { + t.Error("IsAllowed gpt-4o = false: disabled key's blacklist should not affect") + } +} + +func TestBlockAllKeySkipped(t *testing.T) { + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: { + {ID: "k1", Enabled: ptrBool(true), Models: schemas.WhiteList{"gpt-4o"}}, + {ID: "k2", Enabled: ptrBool(true), Models: schemas.WhiteList{"o1"}, BlacklistedModels: schemas.BlackList{"*"}}, + }, + }) + allowed := s.AllowedFor(schemas.OpenAI) + if !slices.Equal(allowed, schemas.WhiteList{"gpt-4o"}) { + t.Errorf("AllowedFor = %v, want [gpt-4o]; block-all key should be skipped", allowed) + } +} + +func TestEntriesForIncludesDisabled(t *testing.T) { + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: { + {ID: "k1", Enabled: ptrBool(true), Models: schemas.WhiteList{"gpt-4o"}}, + {ID: "k2", Enabled: ptrBool(false), Models: schemas.WhiteList{"o1"}}, + }, + }) + entries := s.EntriesFor(schemas.OpenAI) + if len(entries) != 2 { + t.Fatalf("EntriesFor returned %d entries, want 2 (including disabled)", len(entries)) + } + hasDisabled := false + for _, e := range entries { + if e.KeyID == "k2" && !e.Enabled { + hasDisabled = true + } + } + if !hasDisabled { + t.Error("disabled entry missing from EntriesFor") + } +} + +func TestEntryForLookup(t *testing.T) { + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: { + {ID: "k1", Enabled: ptrBool(true), Models: schemas.WhiteList{"gpt-4o"}}, + }, + }) + e, ok := s.EntryFor(schemas.OpenAI, "k1") + if !ok { + t.Fatal("EntryFor returned !ok for existing key") + } + if e.KeyID != "k1" { + t.Errorf("KeyID = %q, want k1", e.KeyID) + } + if _, ok := s.EntryFor(schemas.OpenAI, "missing"); ok { + t.Error("EntryFor returned ok for missing key") + } +} + +func TestResolveAliasReturnsOwner(t *testing.T) { + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: { + { + ID: "k1", Enabled: ptrBool(true), Models: schemas.WhiteList{"*"}, + Aliases: schemas.KeyAliases{ + "my-prod": schemas.AliasConfig{ModelID: "gpt-4o-2024-08-06"}, + }, + }, + }, + }) + owner, ok := s.ResolveAlias(schemas.OpenAI, "my-prod") + if !ok { + t.Fatal("ResolveAlias !ok for known alias") + } + if owner.KeyID != "k1" { + t.Errorf("owner.KeyID = %q, want k1", owner.KeyID) + } + if owner.Config.ModelID != "gpt-4o-2024-08-06" { + t.Errorf("owner.Config.ModelID = %q, want gpt-4o-2024-08-06", owner.Config.ModelID) + } +} + +func TestResolveAliasMissing(t *testing.T) { + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: {{ID: "k1", Enabled: ptrBool(true), Models: schemas.WhiteList{"*"}}}, + }) + if _, ok := s.ResolveAlias(schemas.OpenAI, "nope"); ok { + t.Error("ResolveAlias ok for missing alias") + } + if _, ok := s.ResolveAlias(schemas.ModelProvider("absent"), "anything"); ok { + t.Error("ResolveAlias ok for absent provider") + } +} + +func TestAliasCollisionLastEnabledWinsAndLogs(t *testing.T) { + s, log := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: { + { + ID: "k1", Enabled: ptrBool(true), Models: schemas.WhiteList{"*"}, + Aliases: schemas.KeyAliases{"my-prod": schemas.AliasConfig{ModelID: "from-k1"}}, + }, + { + ID: "k2", Enabled: ptrBool(true), Models: schemas.WhiteList{"*"}, + Aliases: schemas.KeyAliases{"my-prod": schemas.AliasConfig{ModelID: "from-k2"}}, + }, + }, + }) + owner, _ := s.ResolveAlias(schemas.OpenAI, "my-prod") + if owner.KeyID != "k2" { + t.Errorf("owner.KeyID = %q, want k2 (last key in slice wins)", owner.KeyID) + } + if log.DebugCount() == 0 { + t.Error("expected collision debug log, got none") + } +} + +func TestKeysAllowingModel(t *testing.T) { + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: { + {ID: "k1", Enabled: ptrBool(true), Models: schemas.WhiteList{"gpt-4o", "o1"}}, + {ID: "k2", Enabled: ptrBool(true), Models: schemas.WhiteList{"gpt-4o"}, BlacklistedModels: schemas.BlackList{"o1"}}, + {ID: "k3", Enabled: ptrBool(false), Models: schemas.WhiteList{"gpt-4o"}}, + {ID: "k4", Enabled: ptrBool(true), Models: schemas.WhiteList{"*"}}, + }, + }) + got := s.KeysAllowingModel(schemas.OpenAI, "gpt-4o") + sort.Strings(got) + if !slices.Equal(got, []string{"k1", "k2", "k4"}) { + t.Errorf("KeysAllowingModel gpt-4o = %v, want [k1 k2 k4]", got) + } + got = s.KeysAllowingModel(schemas.OpenAI, "o1") + sort.Strings(got) + if !slices.Equal(got, []string{"k1", "k4"}) { + t.Errorf("KeysAllowingModel o1 = %v, want [k1 k4] (k2 blacklists, k3 disabled)", got) + } +} + +func TestSetProviderIsolated(t *testing.T) { + s := New(nil) + s.Replace(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: {{ID: "k1", Enabled: ptrBool(true), Models: schemas.WhiteList{"gpt-4o"}}}, + schemas.Anthropic: {{ID: "k2", Enabled: ptrBool(true), Models: schemas.WhiteList{"claude-3-5-sonnet"}}}, + }) + + s.SetProvider(schemas.OpenAI, []schemas.Key{ + {ID: "k1", Enabled: ptrBool(true), Models: schemas.WhiteList{"gpt-4o-new"}}, + }) + + if got := s.AllowedFor(schemas.OpenAI); !slices.Equal(got, schemas.WhiteList{"gpt-4o-new"}) { + t.Errorf("openai after SetProvider = %v, want [gpt-4o-new]", got) + } + if got := s.AllowedFor(schemas.Anthropic); !slices.Equal(got, schemas.WhiteList{"claude-3-5-sonnet"}) { + t.Errorf("anthropic perturbed by openai SetProvider: %v", got) + } +} + +func TestReplaceDropsDisappearedProviders(t *testing.T) { + s := New(nil) + s.Replace(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: {{ID: "k1", Enabled: ptrBool(true), Models: schemas.WhiteList{"gpt-4o"}}}, + schemas.Anthropic: {{ID: "k2", Enabled: ptrBool(true), Models: schemas.WhiteList{"claude-3-5-sonnet"}}}, + }) + + s.Replace(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: {{ID: "k1", Enabled: ptrBool(true), Models: schemas.WhiteList{"gpt-4o"}}}, + }) + + if got := s.AllowedFor(schemas.Anthropic); got != nil { + t.Errorf("anthropic still present after Replace dropped it: %v", got) + } +} + +func TestSetProviderToEmptyDropsStandard(t *testing.T) { + s := New(nil) + s.Replace(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: {{ID: "k1", Enabled: ptrBool(true), Models: schemas.WhiteList{"gpt-4o"}}}, + }) + + s.SetProvider(schemas.OpenAI, nil) + + if got := s.AllowedFor(schemas.OpenAI); got != nil { + t.Errorf("after SetProvider(empty), AllowedFor = %v, want nil", got) + } +} + +func TestRemoveProvider(t *testing.T) { + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: {{ID: "k1", Enabled: ptrBool(true), Models: schemas.WhiteList{"gpt-4o"}}}, + }) + s.RemoveProvider(schemas.OpenAI) + if got := s.AllowedFor(schemas.OpenAI); got != nil { + t.Errorf("after RemoveProvider, AllowedFor = %v, want nil", got) + } +} + +func TestEntriesForReturnsDefensiveCopy(t *testing.T) { + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: {{ID: "k1", Enabled: ptrBool(true), Models: schemas.WhiteList{"gpt-4o"}}}, + }) + entries := s.EntriesFor(schemas.OpenAI) + entries[0].KeyID = "MUTATED" + again := s.EntriesFor(schemas.OpenAI) + if again[0].KeyID != "k1" { + t.Errorf("store mutated through EntriesFor: KeyID = %q", again[0].KeyID) + } +} + +// These lock down behaviors that the LB plugin used to maintain locally and +// that the keyconfig store now owns. The "aliases don't leak into allowed" +// suite documents the structural isolation: Aliases write to aliasIndex, +// Models write to allowed — they never mix. + +func TestAliases_WildcardModels_AllowedStaysWildcard(t *testing.T) { + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{ + schemas.Bedrock: { + { + ID: "bk1", + Enabled: ptrBool(true), + Models: schemas.WhiteList{"*"}, + Aliases: schemas.KeyAliases{ + "my-claude-alias": schemas.AliasConfig{ModelID: "anthropic.claude-3-5-sonnet-20241022-v2:0"}, + }, + }, + }, + }) + if got := s.AllowedFor(schemas.Bedrock); !slices.Equal(got, schemas.WhiteList{"*"}) { + t.Errorf("AllowedFor = %v, want [*] (Models field wins; aliases do not leak)", got) + } + if _, ok := s.ResolveAlias(schemas.Bedrock, "my-claude-alias"); !ok { + t.Error("alias missing from aliasIndex; should be present alongside ['*']") + } +} + +func TestAliases_SpecificModels_AllowedDoesNotIncludeAliasName(t *testing.T) { + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{ + schemas.Azure: { + { + ID: "az1", + Enabled: ptrBool(true), + Models: schemas.WhiteList{"gpt-4o"}, + Aliases: schemas.KeyAliases{ + "gpt4o-prod": schemas.AliasConfig{ModelID: "gpt-4o"}, + }, + }, + }, + }) + got := s.AllowedFor(schemas.Azure) + sort.Strings(got) + if !slices.Equal(got, schemas.WhiteList{"gpt-4o"}) { + t.Errorf("AllowedFor = %v, want [gpt-4o] only — alias name 'gpt4o-prod' must NOT appear in allowed", got) + } + if _, ok := s.ResolveAlias(schemas.Azure, "gpt4o-prod"); !ok { + t.Error("alias missing from aliasIndex") + } +} + +func TestAliases_EmptyModels_ProviderAbsent(t *testing.T) { + // Models=[] with aliases present: aliases are name swappers, not implicit + // model grants. Operators must explicitly list models in Models field. + // Provider should be absent from aggregates (no enabled allow path). + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{ + schemas.Bedrock: { + { + ID: "bk1", + Enabled: ptrBool(true), + Models: schemas.WhiteList{}, + Aliases: schemas.KeyAliases{ + "prod": schemas.AliasConfig{ModelID: "anthropic.claude-3-5-sonnet-20241022-v2:0"}, + }, + }, + }, + }) + if got := s.AllowedFor(schemas.Bedrock); got != nil { + t.Errorf("AllowedFor = %v, want nil — aliases alone must not produce an allowed entry", got) + } +} + +func TestAllKeysBlockAll_ProviderAbsent(t *testing.T) { + // Every key has BlacklistedModels=["*"]. All are skipped by the aggregator, + // enabledKeysCount stays 0, provider is dropped from the store. + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: { + {ID: "k1", Enabled: ptrBool(true), Models: schemas.WhiteList{"gpt-4o"}, BlacklistedModels: schemas.BlackList{"*"}}, + {ID: "k2", Enabled: ptrBool(true), Models: schemas.WhiteList{"o1"}, BlacklistedModels: schemas.BlackList{"*"}}, + }, + }) + if got := s.AllowedFor(schemas.OpenAI); got != nil { + t.Errorf("AllowedFor = %v, want nil — every key is block-all", got) + } + if s.IsAllowed(schemas.OpenAI, "gpt-4o") { + t.Error("IsAllowed = true for provider with all-block-all keys") + } +} + +func TestExplicitModels_UnionAcrossEnabledKeys(t *testing.T) { + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: { + {ID: "k1", Enabled: ptrBool(true), Models: schemas.WhiteList{"gpt-4o", "o1"}}, + {ID: "k2", Enabled: ptrBool(true), Models: schemas.WhiteList{"gpt-4o", "gpt-4.5"}}, + {ID: "k3", Enabled: ptrBool(false), Models: schemas.WhiteList{"sekret-model"}}, + }, + }) + got := s.AllowedFor(schemas.OpenAI) + sort.Strings(got) + // AllowedFor returns the deduped union of enabled keys' Models, minus + // per-key blacklisted entries. The same model appearing in multiple keys + // is collapsed to a single entry. + want := schemas.WhiteList{"gpt-4.5", "gpt-4o", "o1"} + sort.Strings(want) + if !slices.Equal(got, want) { + t.Errorf("AllowedFor = %v, want union of enabled keys' Models = %v (k3 disabled, excluded)", got, want) + } + // Cross-check via IsAllowed (the actual consumer path). + for _, m := range []string{"gpt-4o", "o1", "gpt-4.5"} { + if !s.IsAllowed(schemas.OpenAI, m) { + t.Errorf("IsAllowed(%s) = false, want true (model is in union of enabled keys)", m) + } + } + if s.IsAllowed(schemas.OpenAI, "sekret-model") { + t.Error("IsAllowed(sekret-model) = true, want false (only on disabled key k3)") + } +} + +// TestIsAllowed_ProviderAbsent ensures IsAllowed returns false +// (deny-by-default) for a provider that has no cached state. +func TestIsAllowed_ProviderAbsent(t *testing.T) { + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{}) + if s.IsAllowed(schemas.OpenAI, "gpt-4o") { + t.Error("IsAllowed on empty store = true, want false") + } +} + +// TestIsAllowed_BlacklistWinsOverAllow asserts blacklist takes precedence even +// when the model is also in the allowed set — the per-key gating uses both, +// but the aggregated view must respect the cross-key intersection blacklist. +func TestIsAllowed_BlacklistWinsOverAllow(t *testing.T) { + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: { + {ID: "k1", Enabled: ptrBool(true), Models: schemas.WhiteList{"gpt-4o"}, BlacklistedModels: schemas.BlackList{"gpt-4o"}}, + {ID: "k2", Enabled: ptrBool(true), Models: schemas.WhiteList{"gpt-4o"}, BlacklistedModels: schemas.BlackList{"gpt-4o"}}, + }, + }) + if s.IsAllowed(schemas.OpenAI, "gpt-4o") { + t.Error("IsAllowed = true; want false (every enabled key blacklists gpt-4o)") + } +} + +// TestProviders_EmptyStore exercises the no-state baseline. +func TestProviders_EmptyStore(t *testing.T) { + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{}) + if got := s.Providers(); len(got) != 0 { + t.Errorf("Providers() on empty store = %v, want []", got) + } +} + +// TestProviders_StandardWithKeys ensures a standard provider with at least +// one enabled non-block-all key is enumerated. +func TestProviders_StandardWithKeys(t *testing.T) { + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: {{ID: "k1", Enabled: ptrBool(true), Models: schemas.WhiteList{"*"}}}, + schemas.Anthropic: {{ID: "k1", Enabled: ptrBool(true), Models: schemas.WhiteList{"*"}}}, + }) + got := s.Providers() + sort.Slice(got, func(i, j int) bool { return string(got[i]) < string(got[j]) }) + want := []schemas.ModelProvider{schemas.Anthropic, schemas.OpenAI} + if !slices.Equal(got, want) { + t.Errorf("Providers() = %v, want %v", got, want) + } +} + +// TestProviders_StandardWithoutKeys verifies a standard provider with no +// enabled keys is dropped from the enumeration. +func TestProviders_StandardWithoutKeys(t *testing.T) { + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: {{ID: "k1", Enabled: ptrBool(false), Models: schemas.WhiteList{"*"}}}, + }) + if got := s.Providers(); len(got) != 0 { + t.Errorf("Providers() = %v, want [] (all keys disabled)", got) + } +} + +// TestProviders_KeylessNonStandardIncluded covers the keyless custom-provider +// branch: a non-standard provider with zero keys is still routable +// (ambient/IAM auth) and must appear in Providers(). buildState marks it +// unrestricted via the allModelsAllowed branch. +func TestProviders_KeylessNonStandardIncluded(t *testing.T) { + custom := schemas.ModelProvider("my-custom-bedrock") + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{custom: nil}) + got := s.Providers() + if !slices.Contains(got, custom) { + t.Errorf("Providers() = %v, want to include %q (keyless non-standard provider)", got, custom) + } + if !s.IsAllowed(custom, "anything") { + t.Error("IsAllowed on keyless non-standard provider = false, want true (allModelsAllowed)") + } +} + +// TestBlacklistIntersection_CaseInsensitive verifies the count-bucket +// normalisation: two keys blocking the same model under different casing +// must aggregate as one entry so the cross-key intersection reaches +// enabledKeysCount and the model gets promoted to the provider-wide blacklist. +// Without strings.ToLower in the count map, this would silently fail. +func TestBlacklistIntersection_CaseInsensitive(t *testing.T) { + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: { + {ID: "k1", Enabled: ptrBool(true), Models: schemas.WhiteList{"*"}, BlacklistedModels: schemas.BlackList{"gpt-4o"}}, + {ID: "k2", Enabled: ptrBool(true), Models: schemas.WhiteList{"*"}, BlacklistedModels: schemas.BlackList{"GPT-4o"}}, + }, + }) + if s.IsAllowed(schemas.OpenAI, "gpt-4o") { + t.Error("IsAllowed(gpt-4o) = true; want false (both keys block under different casing, intersection should still fire)") + } + if s.IsAllowed(schemas.OpenAI, "GpT-4O") { + t.Error("IsAllowed(GpT-4O) = true; want false (BlackList.IsBlocked is case-insensitive)") + } +} + +// TestIsAllowed_NoRoutableKey verifies IsAllowed reflects actual routability, +// not just the coarse aggregated allow/block: when one key is unrestricted but +// blacklists a model and another key simply doesn't list it, no key can serve +// the model, so IsAllowed must return false (matching KeysAllowingModel) rather +// than passing it through the gate only for routing to then fail. +func TestIsAllowed_NoRoutableKey(t *testing.T) { + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: { + {ID: "k1", Enabled: ptrBool(true), Models: schemas.WhiteList{"*"}, BlacklistedModels: schemas.BlackList{"gpt-4o-mini"}}, + {ID: "k2", Enabled: ptrBool(true), Models: schemas.WhiteList{"gpt-4o"}}, + }, + }) + if s.IsAllowed(schemas.OpenAI, "gpt-4o-mini") { + t.Error("IsAllowed(gpt-4o-mini) = true, want false (k1 blacklists it, k2 doesn't list it — no routable key)") + } + if got := s.KeysAllowingModel(schemas.OpenAI, "gpt-4o-mini"); len(got) != 0 { + t.Errorf("KeysAllowingModel(gpt-4o-mini) = %v, want empty — must agree with IsAllowed", got) + } + // gpt-4o is routable via k1 (unrestricted, not blacklisted) and k2. + if !s.IsAllowed(schemas.OpenAI, "gpt-4o") { + t.Error("IsAllowed(gpt-4o) = false, want true (servable by k1 and k2)") + } +} + +// TestBlacklistedFor_PreservesOriginalCasing verifies the provider-level +// blacklist emits the original casing of the first key that blacklisted the +// model (matching AllowedFor), even though the cross-key intersection is +// computed case-insensitively. +func TestBlacklistedFor_PreservesOriginalCasing(t *testing.T) { + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: { + {ID: "k1", Enabled: ptrBool(true), Models: schemas.WhiteList{"*"}, BlacklistedModels: schemas.BlackList{"GPT-4o"}}, + {ID: "k2", Enabled: ptrBool(true), Models: schemas.WhiteList{"*"}, BlacklistedModels: schemas.BlackList{"gpt-4o"}}, + }, + }) + bl := s.BlacklistedFor(schemas.OpenAI) + if !slices.Equal(bl, schemas.BlackList{"GPT-4o"}) { + t.Errorf("BlacklistedFor = %v, want [GPT-4o] (original casing from first key, not lowercased)", bl) + } +} + +// TestAliasIndex_CaseInsensitive_CollisionDetected verifies the alias-name +// normalisation: two keys defining the same alias under different casing +// must collide so the last-wins warning fires AND only one entry lands in +// the index. Without strings.ToLower, both would persist as separate entries +// and the collision warning would never fire. +func TestAliasIndex_CaseInsensitive_CollisionDetected(t *testing.T) { + s, log := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: { + { + ID: "k1", Enabled: ptrBool(true), Models: schemas.WhiteList{"*"}, + Aliases: schemas.KeyAliases{"My-Prod": schemas.AliasConfig{ModelID: "from-k1"}}, + }, + { + ID: "k2", Enabled: ptrBool(true), Models: schemas.WhiteList{"*"}, + Aliases: schemas.KeyAliases{"my-prod": schemas.AliasConfig{ModelID: "from-k2"}}, + }, + }, + }) + if log.DebugCount() == 0 { + t.Error("expected collision debug log on case-different aliases across keys, got none") + } + owner, ok := s.ResolveAlias(schemas.OpenAI, "MY-PROD") + if !ok { + t.Fatal("ResolveAlias(MY-PROD) = (_, false); want a match (lookup is case-insensitive)") + } + if owner.KeyID != "k2" { + t.Errorf("owner.KeyID = %q, want k2 (last key in slice wins after case-normalisation)", owner.KeyID) + } +} + +// TestResolveAlias_CaseInsensitiveLookup verifies the lookup side of the +// case-normalisation contract — an alias defined as "best-claude" must +// resolve regardless of the case used at the call site. +func TestResolveAlias_CaseInsensitiveLookup(t *testing.T) { + s, _ := newStoreFromFixture(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: { + { + ID: "k1", Enabled: ptrBool(true), Models: schemas.WhiteList{"*"}, + Aliases: schemas.KeyAliases{"best-claude": schemas.AliasConfig{ModelID: "claude-sonnet"}}, + }, + }, + }) + for _, q := range []string{"best-claude", "BEST-CLAUDE", "Best-Claude"} { + owner, ok := s.ResolveAlias(schemas.OpenAI, q) + if !ok || owner.KeyID != "k1" { + t.Errorf("ResolveAlias(%q) = (%+v, %v); want owner k1, true", q, owner, ok) + } + } +} + +// TestNewNilLogger_DoesNotPanic exercises the NoOpLogger default path. If +// New(nil) left logger as nil, the alias-collision Debug call would deref +// nil and crash the test. +func TestNewNilLogger_DoesNotPanic(t *testing.T) { + s := New(nil) + s.Replace(map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: { + { + ID: "k1", Enabled: ptrBool(true), Models: schemas.WhiteList{"*"}, + Aliases: schemas.KeyAliases{"my-prod": schemas.AliasConfig{ModelID: "a"}}, + }, + { + ID: "k2", Enabled: ptrBool(true), Models: schemas.WhiteList{"*"}, + Aliases: schemas.KeyAliases{"my-prod": schemas.AliasConfig{ModelID: "b"}}, + }, + }, + }) + if _, ok := s.ResolveAlias(schemas.OpenAI, "my-prod"); !ok { + t.Error("ResolveAlias = false; want true (Replace completed without panic)") + } +} + +// TestReplace_AtomicSnapshot verifies that concurrent readers during a +// Replace never observe a mid-resync provider count. Without atomic +// publish (e.g. the prior sync.Map mutate-in-place design) readers could +// see a count between len(old) and len(new). With the build-then-swap +// design every snapshot read returns either the old or the new size. +func TestReplace_AtomicSnapshot(t *testing.T) { + old := map[schemas.ModelProvider][]schemas.Key{ + schemas.OpenAI: {{ID: "k", Enabled: ptrBool(true), Models: schemas.WhiteList{"*"}}}, + schemas.Anthropic: {{ID: "k", Enabled: ptrBool(true), Models: schemas.WhiteList{"*"}}}, + schemas.Cohere: {{ID: "k", Enabled: ptrBool(true), Models: schemas.WhiteList{"*"}}}, + schemas.Gemini: {{ID: "k", Enabled: ptrBool(true), Models: schemas.WhiteList{"*"}}}, + } + next := map[schemas.ModelProvider][]schemas.Key{ + schemas.Bedrock: {{ID: "k", Enabled: ptrBool(true), Models: schemas.WhiteList{"*"}}}, + } + s, _ := newStoreFromFixture(old) + oldSize, nextSize := len(old), len(next) + + stop := make(chan struct{}) + var reader sync.WaitGroup + reader.Add(1) + bad := make(chan int, 1) + go func() { + defer reader.Done() + for { + select { + case <-stop: + return + default: + n := len(s.Providers()) + if n != oldSize && n != nextSize { + select { + case bad <- n: + default: + } + return + } + } + } + }() + + for i := 0; i < 200; i++ { + s.Replace(next) + s.Replace(old) + } + close(stop) + reader.Wait() + select { + case n := <-bad: + t.Errorf("reader observed torn snapshot size %d (want %d or %d)", n, oldSize, nextSize) + default: + } +} diff --git a/framework/modelcatalog/live/store.go b/framework/modelcatalog/live/store.go new file mode 100644 index 0000000000..b1a434323a --- /dev/null +++ b/framework/modelcatalog/live/store.go @@ -0,0 +1,125 @@ +// Package live caches the response of provider /v1/models calls per +// (provider, keyID, unfiltered). Filtered entries are pre-gated by the +// provider's ListModelsPipeline against the key's allowed/blacklisted/aliases; +// callers reading filtered entries MUST NOT reapply that gate elsewhere or +// alias-backfill rows will be dropped. +// +// The store is passive — it never calls the network. Callers (the HTTP server +// after key add/update, or a future background refresher) decide when to +// fetch and push results in via Upsert. +package live + +import ( + "slices" + "sync" + + bifrost "github.com/maximhq/bifrost/core" + "github.com/maximhq/bifrost/core/schemas" +) + +// Key identifies one cached response. KeyID is "" for keyless providers +// (Vertex workload identity, Bedrock IAM, etc). +type Key struct { + Provider schemas.ModelProvider + KeyID string + Unfiltered bool +} + +// Entry is a single cached response. +type Entry struct { + Models []string +} + +type Store struct { + mu sync.RWMutex + entries map[Key]Entry + logger schemas.Logger +} + +func New(logger schemas.Logger) *Store { + if logger == nil { + logger = bifrost.NewNoOpLogger() + } + return &Store{entries: make(map[Key]Entry), logger: logger} +} + +// Upsert stores a successful fetch. +func (s *Store) Upsert(provider schemas.ModelProvider, keyID string, unfiltered bool, models []string) { + cp := make([]string, len(models)) + copy(cp, models) + k := Key{Provider: provider, KeyID: keyID, Unfiltered: unfiltered} + s.mu.Lock() + s.entries[k] = Entry{Models: cp} + s.mu.Unlock() +} + +// Invalidate drops both filtered and unfiltered entries for one key. Called +// when the key's credential value changes (cached models were computed +// against the old credential) or when the key is deleted. +func (s *Store) Invalidate(provider schemas.ModelProvider, keyID string) { + s.mu.Lock() + delete(s.entries, Key{Provider: provider, KeyID: keyID, Unfiltered: false}) + delete(s.entries, Key{Provider: provider, KeyID: keyID, Unfiltered: true}) + s.mu.Unlock() +} + +// InvalidateProvider drops every entry for the provider across all keys and +// modes. Called on provider delete. +func (s *Store) InvalidateProvider(provider schemas.ModelProvider) { + s.mu.Lock() + for k := range s.entries { + if k.Provider == provider { + delete(s.entries, k) + } + } + s.mu.Unlock() +} + +// ModelsForProvider returns the union of filtered entries for the provider, +// sorted. Filtered entries are pre-gated so this is the effective allowed set +// across the provider's keys. +func (s *Store) ModelsForProvider(provider schemas.ModelProvider) []string { + return s.unionForProvider(provider, false) +} + +// UnfilteredModelsForProvider returns the union of unfiltered entries — the +// raw provider catalog with no key-level gating applied. +func (s *Store) UnfilteredModelsForProvider(provider schemas.ModelProvider) []string { + return s.unionForProvider(provider, true) +} + +// Snapshot returns a defensive copy of every entry for diagnostics. Slices +// are copied; the returned map is independent of store state. +func (s *Store) Snapshot() map[Key]Entry { + s.mu.RLock() + defer s.mu.RUnlock() + out := make(map[Key]Entry, len(s.entries)) + for k, e := range s.entries { + cp := make([]string, len(e.Models)) + copy(cp, e.Models) + out[k] = Entry{Models: cp} + } + return out +} + +// unionForProvider returns the sorted, deduplicated set of models across all +// entries matching the given provider and unfiltered flag. +func (s *Store) unionForProvider(provider schemas.ModelProvider, unfiltered bool) []string { + s.mu.RLock() + defer s.mu.RUnlock() + seen := make(map[string]struct{}) + for k, e := range s.entries { + if k.Provider != provider || k.Unfiltered != unfiltered { + continue + } + for _, m := range e.Models { + seen[m] = struct{}{} + } + } + out := make([]string, 0, len(seen)) + for m := range seen { + out = append(out, m) + } + slices.Sort(out) + return out +} diff --git a/framework/modelcatalog/live/store_test.go b/framework/modelcatalog/live/store_test.go new file mode 100644 index 0000000000..4b8e1b5721 --- /dev/null +++ b/framework/modelcatalog/live/store_test.go @@ -0,0 +1,141 @@ +package live + +import ( + "slices" + "testing" + + "github.com/maximhq/bifrost/core/schemas" +) + +const ( + openai = schemas.OpenAI + anthropic = schemas.Anthropic +) + +func upsertFiltered(s *Store, p schemas.ModelProvider, keyID string, models []string) { + s.Upsert(p, keyID, false, models) +} + +func upsertUnfiltered(s *Store, p schemas.ModelProvider, keyID string, models []string) { + s.Upsert(p, keyID, true, models) +} + +func TestUpsertAndReadFiltered(t *testing.T) { + s := New(nil) + upsertFiltered(s, openai, "k1", []string{"gpt-4o", "gpt-4o-mini"}) + + got := s.ModelsForProvider(openai) + want := []string{"gpt-4o", "gpt-4o-mini"} + if !slices.Equal(got, want) { + t.Fatalf("ModelsForProvider = %v, want %v", got, want) + } +} + +func TestFilteredAndUnfilteredDoNotMix(t *testing.T) { + s := New(nil) + upsertFiltered(s, openai, "k1", []string{"gpt-4o"}) + upsertUnfiltered(s, openai, "k1", []string{"gpt-4o", "gpt-4o-mini", "o1"}) + + if got := s.ModelsForProvider(openai); !slices.Equal(got, []string{"gpt-4o"}) { + t.Errorf("filtered read = %v, want [gpt-4o]", got) + } + if got := s.UnfilteredModelsForProvider(openai); !slices.Equal(got, []string{"gpt-4o", "gpt-4o-mini", "o1"}) { + t.Errorf("unfiltered read = %v, want [gpt-4o gpt-4o-mini o1]", got) + } +} + +func TestUnionAcrossKeys(t *testing.T) { + s := New(nil) + upsertFiltered(s, openai, "k1", []string{"gpt-4o", "gpt-4o-mini"}) + upsertFiltered(s, openai, "k2", []string{"gpt-4o-mini", "o1"}) + + got := s.ModelsForProvider(openai) + want := []string{"gpt-4o", "gpt-4o-mini", "o1"} + if !slices.Equal(got, want) { + t.Fatalf("ModelsForProvider = %v, want %v", got, want) + } +} + +func TestInvalidateOneKeyPreservesOthers(t *testing.T) { + s := New(nil) + upsertFiltered(s, openai, "k1", []string{"gpt-4o"}) + upsertFiltered(s, openai, "k2", []string{"o1"}) + upsertUnfiltered(s, openai, "k1", []string{"gpt-4o", "extra"}) + + s.Invalidate(openai, "k1") + + if got := s.ModelsForProvider(openai); !slices.Equal(got, []string{"o1"}) { + t.Errorf("after Invalidate, filtered = %v, want [o1]", got) + } + if got := s.UnfilteredModelsForProvider(openai); len(got) != 0 { + t.Errorf("after Invalidate, unfiltered = %v, want empty (k1's unfiltered should also drop)", got) + } +} + +func TestInvalidateProviderDropsEverything(t *testing.T) { + s := New(nil) + upsertFiltered(s, openai, "k1", []string{"gpt-4o"}) + upsertFiltered(s, openai, "k2", []string{"o1"}) + upsertFiltered(s, anthropic, "k3", []string{"claude-3-5-sonnet"}) + + s.InvalidateProvider(openai) + + if got := s.ModelsForProvider(openai); len(got) != 0 { + t.Errorf("openai after InvalidateProvider = %v, want empty", got) + } + if got := s.ModelsForProvider(anthropic); !slices.Equal(got, []string{"claude-3-5-sonnet"}) { + t.Errorf("anthropic untouched = %v, want [claude-3-5-sonnet]", got) + } +} + +func TestKeylessProviderUsesEmptyKeyID(t *testing.T) { + s := New(nil) + upsertFiltered(s, schemas.Vertex, "", []string{"gemini-2.0-flash"}) + + if got := s.ModelsForProvider(schemas.Vertex); !slices.Equal(got, []string{"gemini-2.0-flash"}) { + t.Errorf("keyless read = %v, want [gemini-2.0-flash]", got) + } +} + +func TestSnapshotIsDefensiveCopy(t *testing.T) { + s := New(nil) + upsertFiltered(s, openai, "k1", []string{"gpt-4o"}) + + snap := s.Snapshot() + k := Key{Provider: openai, KeyID: "k1", Unfiltered: false} + snap[k].Models[0] = "MUTATED" + + got := s.ModelsForProvider(openai) + if !slices.Equal(got, []string{"gpt-4o"}) { + t.Errorf("store mutated through Snapshot: %v", got) + } +} + +func TestUpsertCopiesInputSlice(t *testing.T) { + s := New(nil) + input := []string{"gpt-4o"} + s.Upsert(openai, "k1", false, input) + + input[0] = "MUTATED" + + if got := s.ModelsForProvider(openai); !slices.Equal(got, []string{"gpt-4o"}) { + t.Errorf("store mutated through input slice: %v", got) + } +} + +func TestModelsForProviderUnknownProviderReturnsEmpty(t *testing.T) { + s := New(nil) + if got := s.ModelsForProvider(openai); len(got) != 0 { + t.Errorf("unknown provider = %v, want empty", got) + } +} + +func TestUpsertOverwritesSameKey(t *testing.T) { + s := New(nil) + upsertFiltered(s, openai, "k1", []string{"gpt-4o"}) + upsertFiltered(s, openai, "k1", []string{"o1"}) + + if got := s.ModelsForProvider(openai); !slices.Equal(got, []string{"o1"}) { + t.Errorf("after re-Upsert = %v, want [o1] (overwrite, not append)", got) + } +} diff --git a/framework/modelcatalog/main.go b/framework/modelcatalog/main.go index d8bb9446ec..2a243e8f68 100644 --- a/framework/modelcatalog/main.go +++ b/framework/modelcatalog/main.go @@ -1,113 +1,103 @@ -// Package modelcatalog provides a pricing manager for the framework. +// Package modelcatalog composes three subpackages — datasheet (pricing + +// model parameters + capabilities), live (per-(provider, keyID) list-models +// cache), and keyconfig (per-provider allow/block/aliases derived from +// keys) — into the ModelCatalog facade that consumers (governance, +// telemetry, logging, server, etc.) use. +// +// The composer owns I/O orchestration: the hourly pricing sync ticker, the +// distributed lock used during sync, and the gossip after-sync hook. +// Subpackages perform no I/O directly — they expose Load/Sync methods the +// composer calls. package modelcatalog import ( "context" "encoding/json" "fmt" - "slices" "sync" "time" providerUtils "github.com/maximhq/bifrost/core/providers/utils" "github.com/maximhq/bifrost/core/schemas" "github.com/maximhq/bifrost/framework/configstore" - configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" + "github.com/maximhq/bifrost/framework/modelcatalog/datasheet" + "github.com/maximhq/bifrost/framework/modelcatalog/keyconfig" + "github.com/maximhq/bifrost/framework/modelcatalog/live" ) type ModelCatalog struct { configStore configstore.ConfigStore distributedLockManager *configstore.DistributedLockManager + logger schemas.Logger - logger schemas.Logger + datasheet *datasheet.Store + live *live.Store + keyconf *keyconfig.Store - // Configuration fields (protected by syncMu) - pricingURL string - modelParametersURL string - syncInterval time.Duration - lastSyncedAt time.Time - syncMu sync.RWMutex + // MCP library sync configuration (protected by syncMu) + mcpLibraryURL string + mcpLibrarySyncInterval time.Duration + lastMCPLibrarySyncedAt time.Time + syncMu sync.RWMutex shouldSyncGate func(ctx context.Context) bool afterSyncHook func(ctx context.Context) - // In-memory cache for fast access - direct map for O(1) lookups - pricingData map[string]configstoreTables.TableModelPricing - mu sync.RWMutex - - // rawOverrides is the canonical list of all active overrides. It exists solely - // to support incremental mutations: UpsertPricingOverrides and DeletePricingOverride - // iterate over it to rebuild the list, then derive customPricing from it. - // customPricing is the actual lookup structure used at query time. - rawOverrides []PricingOverride - customPricing *customPricingData - overridesMu sync.RWMutex - - modelPool map[schemas.ModelProvider][]string - unfilteredModelPool map[schemas.ModelProvider][]string // model pool without allowed models filtering - baseModelIndex map[string]string // model string → canonical base model name - - // Pre-parsed supported response types index (keyed by model name) - // Values are normalized response types: "chat_completion", "responses", "text_completion" - supportedResponseTypes map[string][]string - - // Pre-parsed supported parameters index (keyed by model name, populated from model parameters supported_parameters) - // Values are parameter names the model accepts (e.g., "temperature", "top_p", "tools") - supportedParams map[string][]string - - // Background sync worker + // Background sync orchestration. The ticker, distributed lock, and gossip + // hook live at this level — datasheet.Store has no internal scheduler. syncTicker *time.Ticker - done chan struct{} - wg sync.WaitGroup syncCtx context.Context syncCancel context.CancelFunc + done chan struct{} + wg sync.WaitGroup } -// Init initializes the model catalog func Init(ctx context.Context, config *Config, configStore configstore.ConfigStore, logger schemas.Logger) (*ModelCatalog, error) { - // Initialize pricing URL and sync interval pricingURL := DefaultPricingURL - if config.PricingURL != nil { + if config != nil && config.PricingURL != nil { pricingURL = *config.PricingURL } modelParametersURL := DefaultModelParametersURL - if config.ModelParametersURL != nil && *config.ModelParametersURL != "" { + if config != nil && config.ModelParametersURL != nil && *config.ModelParametersURL != "" { modelParametersURL = *config.ModelParametersURL } + mcpLibraryURL := DefaultMCPLibraryURL + if config != nil && config.MCPLibraryURL != nil && *config.MCPLibraryURL != "" { + mcpLibraryURL = *config.MCPLibraryURL + } + mcpLibrarySyncInterval := DefaultSyncInterval + if config != nil && config.MCPLibrarySyncInterval != nil && *config.MCPLibrarySyncInterval > 0 { + mcpLibrarySyncInterval = time.Duration(*config.MCPLibrarySyncInterval) * time.Second + } syncInterval := DefaultSyncInterval - if config.PricingSyncInterval != nil { + if config != nil && config.PricingSyncInterval != nil { syncInterval = time.Duration(*config.PricingSyncInterval) * time.Second } // Log the active interval and the scheduler's actual check frequency so operators // are not surprised that setting interval=1h does not mean checks happen every second. - // Actual syncs occur when: (1) the 1-hour ticker fires AND (2) time.Since(lastSync) >= pricingSyncInterval. logger.Info("pricing sync interval set to %v (scheduler checks every %v)", syncInterval, syncWorkerTickerPeriod) mc := &ModelCatalog{ - pricingURL: pricingURL, - modelParametersURL: modelParametersURL, - syncInterval: syncInterval, + mcpLibraryURL: mcpLibraryURL, + mcpLibrarySyncInterval: mcpLibrarySyncInterval, configStore: configStore, logger: logger, - pricingData: make(map[string]configstoreTables.TableModelPricing), - modelPool: make(map[schemas.ModelProvider][]string), - unfilteredModelPool: make(map[schemas.ModelProvider][]string), - baseModelIndex: make(map[string]string), - supportedResponseTypes: make(map[string][]string), - supportedParams: make(map[string][]string), - done: make(chan struct{}), distributedLockManager: configstore.NewDistributedLockManager(configStore, logger, configstore.WithDefaultTTL(30*time.Second)), + datasheet: datasheet.New(configStore, logger, datasheet.Config{ + URL: pricingURL, + ModelParametersURL: modelParametersURL, + SyncInterval: syncInterval, + }), + live: live.New(logger), + keyconf: keyconfig.New(logger), + done: make(chan struct{}), } - - // Initialize syncCtx early so background startup goroutines can use it and - // Cleanup() can cancel them. startSyncWorker is still called at the end after - // cold-start paths have completed. mc.syncCtx, mc.syncCancel = context.WithCancel(ctx) // If Init returns an error the caller never owns mc and will never call - // Cleanup(), so cancel syncCtx to stop any background goroutines that were - // already spawned before the failure. + // Cleanup(), so cancel syncCtx to stop any background goroutines that + // were already spawned before the failure. initSucceeded := false defer func() { if !initSucceeded { @@ -117,13 +107,13 @@ func Init(ctx context.Context, config *Config, configStore configstore.ConfigSto logger.Info("initializing model catalog...") if configStore != nil { - // Per-model lazy load when the in-memory cache misses (eviction, new models, or if - // startup bulk load was skipped). loadModelParametersFromDatabase still bulk-warms - // the cache on init and on ReloadFromDB so common paths avoid a DB read per model. + // Lazy load on cache miss: providers may need params for models not + // covered by the startup bulk load (e.g. just-uploaded models). The + // bulk load still warms the common case so this only fires on misses. providerUtils.SetCacheMissHandler(func(model string) *providerUtils.ModelParams { missCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() - params, err := configStore.GetModelParametersByModel(missCtx, model) + params, err := mc.datasheet.GetModelParametersByModel(missCtx, model) if err != nil || params == nil { return nil } @@ -142,34 +132,32 @@ func Init(ctx context.Context, config *Config, configStore configstore.ConfigSto IsVertexMultiRegionOnly: p.VertexMultiRegionOnly, } }) + var wg sync.WaitGroup var pricingErr, paramsErr error wg.Add(2) go func() { defer wg.Done() - if err := mc.loadPricingFromDatabase(ctx); err != nil { + if err := mc.datasheet.LoadFromDB(ctx); err != nil { pricingErr = fmt.Errorf("failed to load initial pricing data: %w", err) return } - mc.mu.RLock() - hasPricingData := len(mc.pricingData) > 0 - mc.mu.RUnlock() - if hasPricingData { - mc.logger.Info("existing pricing data found in database, syncing from URL in background") + if mc.hasPricingData() { + logger.Info("existing pricing data found in database, syncing from URL in background") mc.wg.Add(1) go func() { defer mc.wg.Done() if err := mc.withDistributedLock(mc.syncCtx, "model_catalog_pricing_startup_sync", 10, func() error { - return mc.syncPricing(mc.syncCtx) + return mc.runPricingSync(mc.syncCtx) }); err != nil { - mc.logger.Warn("background startup pricing sync failed: %v", err) + logger.Warn("background startup pricing sync failed: %v", err) } else { - mc.logger.Info("background startup pricing sync completed successfully") + logger.Info("background startup pricing sync completed successfully") } }() } else { if err := mc.withDistributedLock(ctx, "model_catalog_pricing_startup_sync", 10, func() error { - return mc.syncPricing(ctx) + return mc.runPricingSync(ctx) }); err != nil { pricingErr = fmt.Errorf("failed to sync pricing data: %w", err) } @@ -177,27 +165,27 @@ func Init(ctx context.Context, config *Config, configStore configstore.ConfigSto }() go func() { defer wg.Done() - n, err := mc.loadModelParametersFromDatabase(ctx) + n, err := mc.datasheet.LoadModelParamsFromDB(ctx) if err != nil { paramsErr = fmt.Errorf("failed to load initial model parameters: %w", err) return } if n > 0 { - mc.logger.Info("existing model parameters found in database (%d records), syncing from URL in background", n) + logger.Info("existing model parameters found in database (%d records), syncing from URL in background", n) mc.wg.Add(1) go func() { defer mc.wg.Done() if err := mc.withDistributedLock(mc.syncCtx, "model_catalog_params_startup_sync", 10, func() error { - return mc.syncModelParameters(mc.syncCtx) + return mc.runParamsSync(mc.syncCtx) }); err != nil { - mc.logger.Warn("background startup model parameters sync failed: %v", err) + logger.Warn("background startup model parameters sync failed: %v", err) } else { - mc.logger.Info("background startup model parameters sync completed successfully") + logger.Info("background startup model parameters sync completed successfully") } }() } else { if err := mc.withDistributedLock(ctx, "model_catalog_params_startup_sync", 10, func() error { - return mc.syncModelParameters(ctx) + return mc.runParamsSync(ctx) }); err != nil { paramsErr = fmt.Errorf("failed to sync model parameters data: %w", err) } @@ -210,61 +198,96 @@ func Init(ctx context.Context, config *Config, configStore configstore.ConfigSto if paramsErr != nil { return nil, paramsErr } + + // MCP library catalog follows the datasheet bootstrap pattern: if the DB + // already has catalog rows, refresh from URL in the background; if it is + // empty, block startup until the first remote sync lands so the library page + // is populated immediately after boot. + hasMCPLibraryData, err := mc.hasMCPLibraryData(ctx) + if err != nil { + return nil, fmt.Errorf("failed to load initial MCP library data: %w", err) + } + if hasMCPLibraryData { + logger.Info("existing MCP library data found in database, syncing from URL in background") + mc.wg.Add(1) + go func() { + defer mc.wg.Done() + if err := mc.withDistributedLock(mc.syncCtx, "model_catalog_mcp_library_startup_sync", 10, func() error { + return mc.syncMCPLibrary(mc.syncCtx) + }); err != nil { + mc.logger.Warn("background startup MCP library sync failed: %v", err) + } else { + mc.syncMu.Lock() + mc.lastMCPLibrarySyncedAt = time.Now() + mc.syncMu.Unlock() + } + }() + } else { + // Empty DB: attempt a blocking sync so the library page is populated + // immediately after boot. Unlike pricing, a failure here is non-fatal + // — the background worker will retry on the next tick. + if err := mc.withDistributedLock(ctx, "model_catalog_mcp_library_startup_sync", 10, func() error { + return mc.syncMCPLibrary(ctx) + }); err != nil { + logger.Warn("initial MCP library sync failed (will retry in background): %v", err) + } else { + mc.syncMu.Lock() + mc.lastMCPLibrarySyncedAt = time.Now() + mc.syncMu.Unlock() + } + } } else { - // Load pricing and model parameters from URL into memory (no config store) - if err := mc.loadPricingIntoMemoryFromURL(ctx); err != nil { - return nil, fmt.Errorf("failed to load pricing data from config memory: %w", err) + if err := mc.datasheet.LoadFromURLIntoMemory(ctx); err != nil { + return nil, fmt.Errorf("failed to load pricing data into memory: %w", err) } - if err := mc.loadModelParametersIntoMemoryFromURL(ctx); err != nil { + if err := mc.datasheet.LoadModelParamsFromURLIntoMemory(ctx); err != nil { return nil, fmt.Errorf("failed to load model parameters from URL: %w", err) } } - mc.syncMu.Lock() - mc.lastSyncedAt = time.Now() - mc.syncMu.Unlock() + mc.datasheet.MarkSynced(time.Now()) - // Populate model pool with normalized providers from pricing data - mc.populateModelPoolFromPricingData() - - if err := mc.loadPricingOverridesFromStore(ctx); err != nil { + if err := mc.datasheet.LoadOverridesFromStore(ctx); err != nil { return nil, fmt.Errorf("failed to load pricing overrides: %w", err) } - // Start background sync worker mc.startSyncWorker(mc.syncCtx) initSucceeded = true return mc, nil } -func (mc *ModelCatalog) SetShouldSyncGate(shouldSyncGate func(ctx context.Context) bool) { - mc.shouldSyncGate = shouldSyncGate +func (mc *ModelCatalog) SetShouldSyncGate(fn func(ctx context.Context) bool) { + mc.shouldSyncGate = fn } -// SetAfterSyncHook registers a callback invoked after every successful URL → DB pricing sync. -// In enterprise this is used to broadcast a gossip message so other pods reload from DB. +// SetAfterSyncHook registers a callback invoked after every successful +// URL → DB pricing sync. In enterprise this broadcasts a gossip message so +// other pods reload from DB. func (mc *ModelCatalog) SetAfterSyncHook(fn func(ctx context.Context)) { mc.afterSyncHook = fn } -// ReloadFromDB reloads the in-memory pricing cache and model-parameters provider cache from the database. -// In enterprise this is called on non-leader pods when they receive a gossip sync notification. +// ReloadFromDB reloads pricing + model-parameters caches from the database. +// Gossip handler on non-leader pods. func (mc *ModelCatalog) ReloadFromDB(ctx context.Context) error { - if err := mc.loadPricingFromDatabase(ctx); err != nil { + if err := mc.datasheet.LoadFromDB(ctx); err != nil { return err } - mc.populateModelPoolFromPricingData() - _, err := mc.loadModelParametersFromDatabase(ctx) + _, err := mc.datasheet.LoadModelParamsFromDB(ctx) return err } -// UpdateSyncConfig updates the pricing URL and sync interval, restarts the background sync worker, -// then delegates to ForceReloadPricing for a full sync cycle. -func (mc *ModelCatalog) UpdateSyncConfig(ctx context.Context, config *Config) error { - // Acquire pricing mutex to update configuration atomically - mc.syncMu.Lock() +// ReloadPricing re-reads the pricing table into the in-memory cache. The +// management API uses this after a batched write so the new attributes are +// observable immediately. The 24-hour ticker still owns refreshing pricing +// fields from the upstream datasheet; this just refreshes the cache. +func (mc *ModelCatalog) ReloadPricing(ctx context.Context) error { + return mc.datasheet.LoadFromDB(ctx) +} - // Stop existing sync worker before updating configuration +// UpdateSyncConfig updates the pricing/params URLs and sync interval, +// restarts the background sync worker, then runs a full sync cycle. +func (mc *ModelCatalog) UpdateSyncConfig(ctx context.Context, config *Config) error { if mc.syncCancel != nil { mc.syncCancel() } @@ -272,68 +295,91 @@ func (mc *ModelCatalog) UpdateSyncConfig(ctx context.Context, config *Config) er mc.syncTicker.Stop() } - // Update pricing configuration - mc.pricingURL = DefaultPricingURL - if config.PricingURL != nil { - mc.pricingURL = *config.PricingURL + pricingURL := DefaultPricingURL + if config != nil && config.PricingURL != nil { + pricingURL = *config.PricingURL } - - mc.modelParametersURL = DefaultModelParametersURL - if config.ModelParametersURL != nil && *config.ModelParametersURL != "" { - mc.modelParametersURL = *config.ModelParametersURL + modelParametersURL := DefaultModelParametersURL + if config != nil && config.ModelParametersURL != nil && *config.ModelParametersURL != "" { + modelParametersURL = *config.ModelParametersURL } + mcpLibraryURL := DefaultMCPLibraryURL + if config != nil && config.MCPLibraryURL != nil && *config.MCPLibraryURL != "" { + mcpLibraryURL = *config.MCPLibraryURL + } + mcpLibrarySyncInterval := DefaultSyncInterval + if config != nil && config.MCPLibrarySyncInterval != nil && *config.MCPLibrarySyncInterval > 0 { + mcpLibrarySyncInterval = time.Duration(*config.MCPLibrarySyncInterval) * time.Second + } + mc.syncMu.Lock() + mc.mcpLibraryURL = mcpLibraryURL + mc.mcpLibrarySyncInterval = mcpLibrarySyncInterval + mc.syncMu.Unlock() - mc.syncInterval = DefaultSyncInterval - if config.PricingSyncInterval != nil { - mc.syncInterval = time.Duration(*config.PricingSyncInterval) * time.Second + syncInterval := DefaultSyncInterval + if config != nil && config.PricingSyncInterval != nil { + syncInterval = time.Duration(*config.PricingSyncInterval) * time.Second } + mc.datasheet.UpdateSyncConfig(datasheet.Config{ + URL: pricingURL, + ModelParametersURL: modelParametersURL, + SyncInterval: syncInterval, + }) - // Create new sync worker with updated configuration mc.syncCtx, mc.syncCancel = context.WithCancel(ctx) mc.startSyncWorker(mc.syncCtx) - mc.syncMu.Unlock() - - // Delegate to ForceReloadPricing for a complete sync cycle return mc.ForceReloadPricing(ctx) } +// ForceReloadPricing triggers an immediate URL→DB→memory sync for pricing +// and model parameters in parallel, fires the gossip hook, and resets the +// ticker so the next scheduled sync waits a full interval from now. +// +// Behavior change from pre-refactor: this no longer touches the live +// list-models cache. List-models refresh is now driven by key/provider +// edits, not by pricing reloads. func (mc *ModelCatalog) ForceReloadPricing(ctx context.Context) error { - timeout := DefaultPricingTimeout + timeout := datasheet.DefaultPricingTimeout if timeout > 0 { var cancel context.CancelFunc ctx, cancel = context.WithTimeout(ctx, timeout) defer cancel() } - // Run pricing sync and model parameters sync in parallel var wg sync.WaitGroup var pricingErr, paramsErr error - - wg.Add(1) + wg.Add(2) go func() { defer wg.Done() - if err := mc.syncPricing(ctx); err != nil { + if err := mc.runPricingSync(ctx); err != nil { pricingErr = fmt.Errorf("failed to sync pricing data: %w", err) return } - - // Rebuild model pool from updated pricing data - mc.populateModelPoolFromPricingData() - - if err := mc.loadPricingOverridesFromStore(ctx); err != nil { + if err := mc.datasheet.LoadOverridesFromStore(ctx); err != nil { pricingErr = fmt.Errorf("failed to load pricing overrides: %w", err) - return + } + }() + go func() { + defer wg.Done() + if err := mc.runParamsSync(ctx); err != nil { + paramsErr = fmt.Errorf("failed to sync model parameters: %w", err) } }() + // MCP library sync runs alongside but is non-fatal: a failure here must not + // block a pricing/params force-reload. It is logged and the last-sync + // timestamp is only advanced on success. wg.Add(1) go func() { defer wg.Done() - if err := mc.syncModelParameters(ctx); err != nil { - paramsErr = fmt.Errorf("failed to sync model parameters: %w", err) + if err := mc.syncMCPLibrary(ctx); err != nil { + mc.logger.Warn("MCP library sync during force-reload failed: %v", err) return } + mc.syncMu.Lock() + mc.lastMCPLibrarySyncedAt = time.Now() + mc.syncMu.Unlock() }() wg.Wait() @@ -348,182 +394,238 @@ func (mc *ModelCatalog) ForceReloadPricing(ctx context.Context) error { mc.afterSyncHook(ctx) } - mc.syncMu.Lock() - // Reset the ticker so the next scheduled sync waits a full interval from now if mc.syncTicker != nil { - mc.syncTicker.Reset(mc.syncInterval) + mc.syncTicker.Reset(mc.datasheet.SyncInterval()) } - mc.syncMu.Unlock() + return nil +} +func (mc *ModelCatalog) Cleanup() error { + if mc.syncCancel != nil { + mc.syncCancel() + } + if mc.syncTicker != nil { + mc.syncTicker.Stop() + } + close(mc.done) + mc.wg.Wait() return nil } -// getPricingURL returns a copy of the pricing URL under mutex protection -func (mc *ModelCatalog) getPricingURL() string { - mc.syncMu.RLock() - defer mc.syncMu.RUnlock() - return mc.pricingURL +// --- Sync ticker (orchestrates datasheet.Store sync methods) --- + +func (mc *ModelCatalog) startSyncWorker(ctx context.Context) { + // IMPORTANT scheduling model: + // + // The sync worker wakes on a fixed ticker (syncWorkerTickerPeriod = 1h). + // On each wake it checks time.Since(LastSyncedAt) >= SyncInterval. + // This means SyncInterval defines the *minimum elapsed time* between syncs, + // and the actual frequency = max(syncWorkerTickerPeriod, SyncInterval). + // Setting SyncInterval below the ticker period has no effect — the hourly + // ticker is the hard lower bound on check granularity. + mc.syncTicker = time.NewTicker(syncWorkerTickerPeriod) + mc.wg.Add(1) + go mc.syncWorker(ctx) } -func (mc *ModelCatalog) getModelParametersURL() string { - mc.syncMu.RLock() - defer mc.syncMu.RUnlock() - return mc.modelParametersURL +func (mc *ModelCatalog) syncWorker(ctx context.Context) { + // Capture the ticker once so the select loop doesn't race with + // UpdateSyncConfig overwriting mc.syncTicker while this goroutine + // is still draining after mc.syncCancel(). + ticker := mc.syncTicker + defer mc.wg.Done() + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + mc.syncTick(ctx) + case <-mc.done: + return + } + } } -// IsRequestTypeSupported checks if a model supports chat completion. -// It checks the supportedResponseTypes index. -func (mc *ModelCatalog) IsRequestTypeSupported(model string, provider schemas.ModelProvider, requestType schemas.RequestType) bool { - mc.mu.RLock() - defer mc.mu.RUnlock() - outputs, ok := mc.supportedResponseTypes[model] - return ok && slices.Contains(outputs, string(requestType)) +func (mc *ModelCatalog) syncTick(ctx context.Context) { + pricingDue := time.Since(mc.datasheet.LastSyncedAt()) >= mc.datasheet.SyncInterval() + mcpLibraryDue := mc.isMCPLibrarySyncDue() + if !pricingDue && !mcpLibraryDue { + return + } + mc.logger.Debug("starting model catalog background sync") + + // Pricing and MCP library use separate distributed locks so their + // independent cadences don't block each other. + var outerWg sync.WaitGroup + if pricingDue { + outerWg.Add(1) + go func() { + defer outerWg.Done() + if err := mc.withDistributedLock(ctx, "model_catalog_pricing_sync", 10, func() error { + var wg sync.WaitGroup + var pricingErr, paramsErr error + wg.Add(2) + go func() { + defer wg.Done() + if err := mc.runPricingSync(ctx); err != nil { + mc.logger.Error("background pricing sync failed: %v", err) + pricingErr = err + } + }() + go func() { + defer wg.Done() + if err := mc.runParamsSync(ctx); err != nil { + mc.logger.Error("background model parameters sync failed: %v", err) + paramsErr = err + } + }() + wg.Wait() + if pricingErr == nil && paramsErr == nil { + if mc.afterSyncHook != nil { + mc.afterSyncHook(ctx) + } + mc.datasheet.MarkSynced(time.Now()) + } + if pricingErr != nil { + return pricingErr + } + return paramsErr + }); err != nil { + mc.logger.Error("failed to run pricing sync: %v", err) + } + }() + } + if mcpLibraryDue { + outerWg.Add(1) + go func() { + defer outerWg.Done() + if err := mc.withDistributedLock(ctx, "model_catalog_mcp_library_sync", 10, func() error { + if err := mc.syncMCPLibrary(ctx); err != nil { + mc.logger.Error("background MCP library sync failed: %v", err) + return err + } + mc.syncMu.Lock() + mc.lastMCPLibrarySyncedAt = time.Now() + mc.syncMu.Unlock() + return nil + }); err != nil { + mc.logger.Error("failed to run MCP library sync: %v", err) + } + }() + } + outerWg.Wait() + mc.logger.Debug("model catalog background sync completed") } -// GetSupportedParameters returns the list of supported parameter names for a model. -// Returns nil if the model is not found in the catalog. -func (mc *ModelCatalog) GetSupportedParameters(model string) []string { - mc.mu.RLock() - params, ok := mc.supportedParams[model] - mc.mu.RUnlock() - if !ok { +// runPricingSync wraps the datasheet pricing sync with the gate check. +func (mc *ModelCatalog) runPricingSync(ctx context.Context) error { + if mc.shouldSyncGate != nil && !mc.shouldSyncGate(ctx) { return nil } - // Return a copy to prevent external modification - result := make([]string, len(params)) - copy(result, params) - return result + return mc.datasheet.SyncFromURL(ctx) } -// populateModelPool populates the model pool with all available models per provider (thread-safe). -// -// This function is the only path that resets modelPool / unfilteredModelPool / -// baseModelIndex from upstream pricing data. It is called both at init (where -// the pool is empty) and on every reload (gossip ReloadFromDB, manual -// ForceReloadPricing). To avoid drift on reload — where a naive wipe would -// drop everything contributed by per-provider list-models output and key -// allowed_models — the pre-wipe pool is snapshotted and unioned back in after -// the pricing rebuild. baseModelIndex is intentionally not preserved: aliases -// outside the pricing sheet have no canonical base-model entry, and -// getBaseModelNameUnsafe falls through to algorithmic stripping for them. -func (mc *ModelCatalog) populateModelPoolFromPricingData() { - // Acquire write lock for the entire rebuild operation - mc.mu.Lock() - defer mc.mu.Unlock() - - // Snapshot the pre-wipe pool so non-pricing contributions (list-models - // output, allowed_models) survive the rebuild. - previousModelPool := make(map[schemas.ModelProvider][]string, len(mc.modelPool)) - for provider, models := range mc.modelPool { - copied := make([]string, len(models)) - copy(copied, models) - previousModelPool[provider] = copied - } - previousUnfilteredModelPool := make(map[schemas.ModelProvider][]string, len(mc.unfilteredModelPool)) - for provider, models := range mc.unfilteredModelPool { - copied := make([]string, len(models)) - copy(copied, models) - previousUnfilteredModelPool[provider] = copied - } - - // Clear existing model pool and base model index - mc.modelPool = make(map[schemas.ModelProvider][]string) - mc.unfilteredModelPool = make(map[schemas.ModelProvider][]string) - mc.baseModelIndex = make(map[string]string) - - // Map to track unique models per provider - providerModels := make(map[schemas.ModelProvider]map[string]bool) - - // Iterate through all pricing data to collect models per provider - for _, pricing := range mc.pricingData { - // Normalize provider before adding to model pool - normalizedProvider := schemas.ModelProvider(normalizeProvider(pricing.Provider)) - - // Initialize map for this provider if not exists - if providerModels[normalizedProvider] == nil { - providerModels[normalizedProvider] = make(map[string]bool) - } - - // Add model to the provider's model set (using map for deduplication) - providerModels[normalizedProvider][pricing.Model] = true - - // Build base model index from pre-computed base_model field - if pricing.BaseModel != "" { - mc.baseModelIndex[pricing.Model] = pricing.BaseModel - } +// runParamsSync wraps the datasheet params sync with the gate check. +func (mc *ModelCatalog) runParamsSync(ctx context.Context) error { + if mc.shouldSyncGate != nil && !mc.shouldSyncGate(ctx) { + mc.logger.Debug("model parameters sync cancelled by custom gate") + return nil } + return mc.datasheet.SyncModelParamsFromURL(ctx) +} - // Convert sets to slices and assign to modelPool - for provider, modelSet := range providerModels { - models := make([]string, 0, len(modelSet)) - for model := range modelSet { - models = append(models, model) +// withDistributedLock acquires a named distributed lock and runs fn under +// it. retries=0 blocks until acquired; retries>0 uses LockWithRetry. The +// unlock uses a fresh context so cancelled work contexts don't leak the +// lock until TTL expiry. +func (mc *ModelCatalog) withDistributedLock(ctx context.Context, key string, retries int, fn func() error) error { + lock, err := mc.distributedLockManager.NewLock(key) + if err != nil { + return fmt.Errorf("failed to create lock %q: %w", key, err) + } + if retries > 0 { + if err := lock.LockWithRetry(ctx, retries); err != nil { + return fmt.Errorf("failed to acquire lock %q: %w", key, err) } - mc.modelPool[provider] = models - mc.unfilteredModelPool[provider] = models - } - - // Union the pre-wipe snapshot back in. Anything previously added by - // UpsertModelDataForProvider / UpsertUnfilteredModelDataForProvider — - // list-models output, allowed_models aliases — is restored. Pricing - // entries from the rebuild win on duplicates (already in place above); - // removals via DeleteModelDataForProvider are respected because that - // method strips the provider from the live map before this runs. - for provider, models := range previousModelPool { - for _, m := range models { - if !slices.Contains(mc.modelPool[provider], m) { - mc.modelPool[provider] = append(mc.modelPool[provider], m) - } + } else { + if err := lock.Lock(ctx); err != nil { + return fmt.Errorf("failed to acquire lock %q: %w", key, err) } } - for provider, models := range previousUnfilteredModelPool { - for _, m := range models { - if !slices.Contains(mc.unfilteredModelPool[provider], m) { - mc.unfilteredModelPool[provider] = append(mc.unfilteredModelPool[provider], m) - } + defer func() { + if err := lock.Unlock(context.Background()); err != nil { + mc.logger.Warn("failed to release distributed lock %q: %v", key, err) } - } + }() + return fn() +} - // Log the populated model pool for debugging - totalModels := 0 - for provider, models := range mc.modelPool { - totalModels += len(models) - mc.logger.Debug("populated %d models for provider %s", len(models), string(provider)) - } - mc.logger.Info("populated model pool with %d models across %d providers", totalModels, len(mc.modelPool)) +// hasPricingData reports whether the datasheet store currently has any +// pricing rows in memory. Used during Init to decide between blocking sync +// and background sync. +func (mc *ModelCatalog) hasPricingData() bool { + return len(mc.datasheet.DatasheetProviders()) > 0 } -// Cleanup cleans up the model catalog -func (mc *ModelCatalog) Cleanup() error { - if mc.syncCancel != nil { - mc.syncCancel() +func (mc *ModelCatalog) hasMCPLibraryData(ctx context.Context) (bool, error) { + if mc.configStore == nil { + return false, nil } - - mc.syncMu.Lock() - if mc.syncTicker != nil { - mc.syncTicker.Stop() + _, totalCount, err := mc.configStore.GetMCPLibraryPaginated(ctx, configstore.MCPLibraryQueryParams{Limit: 1}) + if err != nil { + return false, err } - mc.syncMu.Unlock() + return totalCount > 0, nil +} - close(mc.done) - mc.wg.Wait() +func (mc *ModelCatalog) isMCPLibrarySyncDue() bool { + if mc.configStore == nil { + return false + } + mc.syncMu.RLock() + lastSyncedAt := mc.lastMCPLibrarySyncedAt + syncInterval := mc.mcpLibrarySyncInterval + mc.syncMu.RUnlock() + if syncInterval <= 0 { + syncInterval = DefaultSyncInterval + } + return lastSyncedAt.IsZero() || time.Since(lastSyncedAt) >= syncInterval +} - return nil +// knownProviders returns the union of providers seen by any store. Used by +// GetProvidersForModel (models.go) to enumerate candidates. +func (mc *ModelCatalog) knownProviders() []schemas.ModelProvider { + seen := make(map[schemas.ModelProvider]struct{}) + out := make([]schemas.ModelProvider, 0) + for _, p := range mc.datasheet.DatasheetProviders() { + if _, ok := seen[p]; !ok { + seen[p] = struct{}{} + out = append(out, p) + } + } + for k := range mc.live.Snapshot() { + if _, ok := seen[k.Provider]; !ok { + seen[k.Provider] = struct{}{} + out = append(out, k.Provider) + } + } + for _, p := range mc.keyconf.Providers() { + if _, ok := seen[p]; !ok { + seen[p] = struct{}{} + out = append(out, p) + } + } + return out } -// NewTestCatalog creates a minimal ModelCatalog for testing purposes. -// It does not start background sync workers or connect to external services. +// NewTestCatalog constructs a minimal ModelCatalog for unit tests. Does not +// start background workers or hit external services. func NewTestCatalog(baseModelIndex map[string]string) *ModelCatalog { - if baseModelIndex == nil { - baseModelIndex = make(map[string]string) - } return &ModelCatalog{ - modelPool: make(map[schemas.ModelProvider][]string), - unfilteredModelPool: make(map[schemas.ModelProvider][]string), - baseModelIndex: baseModelIndex, - pricingData: make(map[string]configstoreTables.TableModelPricing), - supportedResponseTypes: make(map[string][]string), - supportedParams: make(map[string][]string), - done: make(chan struct{}), + datasheet: datasheet.NewTestStore(baseModelIndex), + live: live.New(nil), + keyconf: keyconfig.New(nil), + done: make(chan struct{}), } } diff --git a/framework/modelcatalog/main_test.go b/framework/modelcatalog/main_test.go deleted file mode 100644 index 3f1120ab09..0000000000 --- a/framework/modelcatalog/main_test.go +++ /dev/null @@ -1,209 +0,0 @@ -package modelcatalog - -import ( - "testing" - - "github.com/maximhq/bifrost/core/schemas" - "github.com/maximhq/bifrost/framework/configstore" - configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" - "github.com/stretchr/testify/assert" -) - -// newTestCatalog creates a minimal ModelCatalog for testing within the package. -func newTestCatalog(modelPool map[schemas.ModelProvider][]string, baseModelIndex map[string]string) *ModelCatalog { - if modelPool == nil { - modelPool = make(map[schemas.ModelProvider][]string) - } - if baseModelIndex == nil { - baseModelIndex = make(map[string]string) - } - return &ModelCatalog{ - modelPool: modelPool, - baseModelIndex: baseModelIndex, - pricingData: make(map[string]configstoreTables.TableModelPricing), - } -} - -// --- GetBaseModelName tests --- - -func TestGetBaseModelName_Simple(t *testing.T) { - mc := newTestCatalog(nil, nil) - // No catalog data, no prefix — returns as-is (no date suffix to strip either) - assert.Equal(t, "gpt-4o", mc.GetBaseModelName("gpt-4o")) -} - -func TestGetBaseModelName_Prefixed(t *testing.T) { - mc := newTestCatalog(nil, nil) - // Provider prefix stripped, no catalog — algorithmic fallback returns base - assert.Equal(t, "gpt-4o", mc.GetBaseModelName("openai/gpt-4o")) -} - -func TestGetBaseModelName_PrefixedAnthropic(t *testing.T) { - mc := newTestCatalog(nil, nil) - assert.Equal(t, "claude-3-5-sonnet", mc.GetBaseModelName("anthropic/claude-3-5-sonnet")) -} - -func TestGetBaseModelName_FromCatalog(t *testing.T) { - // Model has a pre-computed base_model in the catalog - mc := newTestCatalog(nil, map[string]string{ - "gpt-4o": "gpt-4o", - "gpt-4o-2024-08-06": "gpt-4o", - }) - assert.Equal(t, "gpt-4o", mc.GetBaseModelName("gpt-4o")) - assert.Equal(t, "gpt-4o", mc.GetBaseModelName("gpt-4o-2024-08-06")) -} - -func TestGetBaseModelName_ProviderPrefixWithCatalog(t *testing.T) { - // Model has provider prefix — strip prefix, then find in catalog - mc := newTestCatalog(nil, map[string]string{ - "gpt-4o": "gpt-4o", - }) - assert.Equal(t, "gpt-4o", mc.GetBaseModelName("openai/gpt-4o")) -} - -func TestGetBaseModelName_FallbackAlgorithmic(t *testing.T) { - // Model NOT in catalog — falls back to schemas.BaseModelName (date stripping) - mc := newTestCatalog(nil, nil) - // Anthropic-style date suffix - assert.Equal(t, "claude-sonnet-4", mc.GetBaseModelName("claude-sonnet-4-20250514")) - // OpenAI-style date suffix - assert.Equal(t, "gpt-4o", mc.GetBaseModelName("gpt-4o-2024-08-06")) -} - -func TestGetBaseModelName_FallbackAlgorithmicWithPrefix(t *testing.T) { - // Provider prefix + not in catalog — strip prefix, then algorithmic fallback - mc := newTestCatalog(nil, nil) - assert.Equal(t, "claude-sonnet-4", mc.GetBaseModelName("anthropic/claude-sonnet-4-20250514")) -} - -func TestGetBaseModelName_UnknownModel(t *testing.T) { - mc := newTestCatalog(nil, nil) - assert.Equal(t, "some-random-model", mc.GetBaseModelName("some-random-model")) -} - -func TestGetBaseModelName_CatalogTakesPrecedence(t *testing.T) { - // If catalog says the base_model is X, use it even if algorithmic would give Y - mc := newTestCatalog(nil, map[string]string{ - "my-custom-model-20250101": "my-custom-model-20250101", // catalog says keep the date - }) - assert.Equal(t, "my-custom-model-20250101", mc.GetBaseModelName("my-custom-model-20250101")) -} - -// --- IsSameModel tests --- - -func TestIsSameModel_DirectMatch(t *testing.T) { - mc := newTestCatalog(nil, nil) - assert.True(t, mc.IsSameModel("gpt-4o", "gpt-4o")) -} - -func TestIsSameModel_ProviderPrefix(t *testing.T) { - mc := newTestCatalog(nil, nil) - assert.True(t, mc.IsSameModel("openai/gpt-4o", "gpt-4o")) - assert.True(t, mc.IsSameModel("gpt-4o", "openai/gpt-4o")) -} - -func TestIsSameModel_BothPrefixed(t *testing.T) { - mc := newTestCatalog(nil, nil) - assert.True(t, mc.IsSameModel("openai/gpt-4o", "openai/gpt-4o")) -} - -func TestIsSameModel_DifferentProvidersSameBase(t *testing.T) { - mc := newTestCatalog(nil, nil) - // Both have the same base model after stripping different provider prefixes - assert.True(t, mc.IsSameModel("openai/gpt-4o", "azure/gpt-4o")) -} - -func TestIsSameModel_DifferentModels(t *testing.T) { - mc := newTestCatalog(nil, nil) - assert.False(t, mc.IsSameModel("gpt-4o", "claude-3-5-sonnet")) -} - -func TestIsSameModel_DifferentModelsBothPrefixed(t *testing.T) { - mc := newTestCatalog(nil, nil) - assert.False(t, mc.IsSameModel("openai/gpt-4o", "anthropic/claude-3-5-sonnet")) -} - -func TestIsSameModel_CatalogBacked(t *testing.T) { - // Two model strings that look different but the catalog says they have the same base_model - mc := newTestCatalog(nil, map[string]string{ - "claude-3-5-sonnet": "claude-3-5-sonnet", - "claude-3-5-sonnet-20241022": "claude-3-5-sonnet", - }) - assert.True(t, mc.IsSameModel("claude-3-5-sonnet", "claude-3-5-sonnet-20241022")) - assert.True(t, mc.IsSameModel("claude-3-5-sonnet-20241022", "claude-3-5-sonnet")) -} - -func TestIsSameModel_AlgorithmicFallback(t *testing.T) { - // Models not in catalog — use algorithmic date stripping - mc := newTestCatalog(nil, nil) - assert.True(t, mc.IsSameModel("custom-model-20250101", "custom-model")) -} - -func TestIsSameModel_EmptyStrings(t *testing.T) { - mc := newTestCatalog(nil, nil) - assert.True(t, mc.IsSameModel("", "")) - assert.False(t, mc.IsSameModel("gpt-4o", "")) - assert.False(t, mc.IsSameModel("", "gpt-4o")) -} - -func TestIsModelAllowedForProvider_PrefixedAllowedModelInCatalog(t *testing.T) { - mc := newTestCatalog( - map[schemas.ModelProvider][]string{ - schemas.OpenRouter: {"openai/gpt-4o"}, - }, - nil, - ) - - providerConfig := configstore.ProviderConfig{} - - assert.True(t, mc.IsModelAllowedForProvider(schemas.OpenRouter, "gpt-4o", &providerConfig, []string{"openai/gpt-4o"})) -} - -func TestIsModelAllowedForProvider_CustomProviderListModelsDisabled(t *testing.T) { - mc := newTestCatalog(nil, nil) - - // Custom provider with list-models disabled + ["*"] → should return true - providerConfig := configstore.ProviderConfig{ - CustomProviderConfig: &schemas.CustomProviderConfig{ - AllowedRequests: &schemas.AllowedRequests{ - ListModels: false, - }, - }, - } - assert.True(t, mc.IsModelAllowedForProvider("custom-provider", "any-model", &providerConfig, []string{"*"})) -} - -func TestIsModelAllowedForProvider_CustomProviderListModelsEnabled(t *testing.T) { - mc := newTestCatalog( - map[schemas.ModelProvider][]string{ - "custom-provider": {"model-a"}, - }, - nil, - ) - - // Custom provider with list-models enabled + ["*"] → should go through catalog - providerConfig := configstore.ProviderConfig{ - CustomProviderConfig: &schemas.CustomProviderConfig{ - AllowedRequests: &schemas.AllowedRequests{ - ListModels: true, - }, - }, - } - // model-a is in catalog → allowed - assert.True(t, mc.IsModelAllowedForProvider("custom-provider", "model-a", &providerConfig, []string{"*"})) - // model-b is NOT in catalog → denied - assert.False(t, mc.IsModelAllowedForProvider("custom-provider", "model-b", &providerConfig, []string{"*"})) -} - -func TestIsModelAllowedForProvider_NilProviderConfig(t *testing.T) { - mc := newTestCatalog( - map[schemas.ModelProvider][]string{ - "some-provider": {"model-x"}, - }, - nil, - ) - - // nil providerConfig + ["*"] → should go through catalog (not bypass) - assert.True(t, mc.IsModelAllowedForProvider("some-provider", "model-x", nil, []string{"*"})) - assert.False(t, mc.IsModelAllowedForProvider("some-provider", "model-y", nil, []string{"*"})) -} diff --git a/framework/modelcatalog/mcp_library_sync.go b/framework/modelcatalog/mcp_library_sync.go new file mode 100644 index 0000000000..7b6b2f73f4 --- /dev/null +++ b/framework/modelcatalog/mcp_library_sync.go @@ -0,0 +1,318 @@ +package modelcatalog + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "time" + + bifrost "github.com/maximhq/bifrost/core" + "github.com/maximhq/bifrost/core/schemas" + "github.com/maximhq/bifrost/framework/configstore" + configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" + "gorm.io/gorm" +) + +const ( + urlFetchMaxRetries = 3 // retries after the first attempt (4 attempts total) + urlFetchMaxBackoff = 10 * time.Second // cap for exponential backoff (steps start at 1s) + retryBackoffMin = time.Second // initial wait before the first retry + maxMCPLibraryBodyBytes = 50 << 20 // 50 MiB — hard cap on the catalog payload to prevent OOM +) + +// withRetries runs op up to maxRetries+1 times, waiting with exponential +// backoff (starting at retryBackoffMin, capped at maxBackoff) between attempts. +// It returns the first successful result or the last error. The context is +// honored during both the operation and the backoff waits. +func withRetries[T any](ctx context.Context, maxRetries int, maxBackoff time.Duration, op func() (T, error)) (T, error) { + var zero T + if maxRetries < 0 { + maxRetries = 0 + } + var lastErr error + for attempt := 0; attempt <= maxRetries; attempt++ { + select { + case <-ctx.Done(): + return zero, ctx.Err() + default: + } + + if attempt > 0 { + backoff := retryBackoffMin * time.Duration(1< 0 && backoff > maxBackoff { + backoff = maxBackoff + } + select { + case <-ctx.Done(): + return zero, ctx.Err() + case <-time.After(backoff): + } + } + v, err := op() + if err == nil { + return v, nil + } + lastErr = err + } + return zero, lastErr +} + +// MCPLibraryEntry is the JSON shape a single server has in the remote MCP +// library catalog (the payload fetched from DefaultMCPLibraryURL / custom URL). +// The catalog carries no slug; it is derived from Name at sync time via +// Slugify. The remaining fields map onto TableMCPLibrary minus the DB-managed +// fields (ID, Slug, CreatedAt, UpdatedAt). +type MCPLibraryEntry struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + Category string `json:"category,omitempty"` + ConnectionType schemas.MCPConnectionType `json:"connection_type"` + ConnectionURL string `json:"connection_url,omitempty"` + StdioConfig *schemas.MCPStdioConfig `json:"stdio_config,omitempty"` + AuthType schemas.MCPAuthType `json:"auth_type,omitempty"` + RequiredHeaderKeys []string `json:"required_header_keys,omitempty"` + IconURL string `json:"icon_url,omitempty"` + DocsURL string `json:"docs_url,omitempty"` + Publisher string `json:"publisher,omitempty"` + Tags []string `json:"tags,omitempty"` + Metadata map[string]any `json:"metadata,omitempty"` +} + +// MCPLibraryPayload is the top-level JSON envelope returned by the remote +// MCP library catalog endpoint. +type MCPLibraryPayload struct { + Servers []MCPLibraryEntry `json:"servers"` + LastUpdatedAt string `json:"lastUpdatedAt,omitempty"` +} + +// SyncMCPLibrary fetches the MCP server catalog from url, parses the JSON +// payload, and upserts each row into the mcp_library table keyed by slug. +// Returns the number of rows upserted. +// +// The function is intentionally stateless and operates directly on the +// ConfigStore so it can be called from both the force-sync handler and the +// background worker without needing a dedicated manager struct. +func SyncMCPLibrary(ctx context.Context, url string, store configstore.ConfigStore) (int, error) { + if url == "" { + url = DefaultMCPLibraryURL + } + + entries, err := withRetries(ctx, urlFetchMaxRetries, urlFetchMaxBackoff, func() ([]MCPLibraryEntry, error) { + return fetchMCPLibrary(ctx, url) + }) + if err != nil { + return 0, fmt.Errorf("failed to fetch MCP library from %s: %w", url, err) + } + + if len(entries) == 0 { + return 0, nil + } + + // Load the slugs the sync must not touch: org-internal ("custom") rows and + // soft-deleted ("tombstoned") rows. A remote payload entry whose slug is in + // this set is skipped silently so the rest of the payload still seeds. + protected, err := store.GetProtectedMCPLibrarySlugs(ctx) + if err != nil { + return 0, fmt.Errorf("failed to load protected MCP library slugs: %w", err) + } + protectedSet := make(map[string]bool, len(protected)) + for _, slug := range protected { + protectedSet[slug] = true + } + + // Upsert all entries in a single transaction. + count := 0 + err = store.ExecuteTransaction(ctx, func(tx *gorm.DB) error { + seen := make(map[string]bool, len(entries)) + for i := range entries { + e := &entries[i] + if e.Name == "" { + continue // skip malformed entries + } + // The catalog payload carries no slug; derive it from the name, + // matching the slug generation used for custom library entries. + slug := Slugify(e.Name) + if slug == "" { + continue // name had no slug-able content + } + if protectedSet[slug] { + continue // never overwrite custom or tombstoned rows + } + if seen[slug] { + continue // deduplicate within the payload + } + seen[slug] = true + + now := time.Now() + row := &configstoreTables.TableMCPLibrary{ + Slug: slug, + Name: e.Name, + Description: e.Description, + Category: e.Category, + ConnectionType: e.ConnectionType, + ConnectionURL: e.ConnectionURL, + StdioConfig: e.StdioConfig, + AuthType: e.AuthType, + RequiredHeaderKeys: e.RequiredHeaderKeys, + IconURL: e.IconURL, + DocsURL: e.DocsURL, + Publisher: e.Publisher, + Tags: e.Tags, + Metadata: e.Metadata, + Source: "remote", + CreatedAt: now, + UpdatedAt: now, + } + if err := store.UpsertMCPLibraryEntry(ctx, row, tx); err != nil { + return fmt.Errorf("failed to upsert MCP library entry %q: %w", slug, err) + } + count++ + } + return nil + }) + if err != nil { + return 0, fmt.Errorf("failed to sync MCP library to database: %w", err) + } + + return count, nil +} + +// syncMCPLibrary is the ModelCatalog method called by the background sync worker +// and ForceReloadPricing. It delegates to the stateless SyncMCPLibrary function +// using the catalog's configured URL and config store. +func (mc *ModelCatalog) syncMCPLibrary(ctx context.Context) error { + if mc.shouldSyncGate != nil && !mc.shouldSyncGate(ctx) { + mc.logger.Debug("MCP library sync cancelled by custom gate") + return nil + } + _, err := mc.syncMCPLibraryNow(ctx) + return err +} + +// ForceReloadMCPLibrary triggers an immediate MCP library sync from the current +// configured source and advances the MCP library sync timer on success. +func (mc *ModelCatalog) ForceReloadMCPLibrary(ctx context.Context) (int, error) { + if mc.shouldSyncGate != nil && !mc.shouldSyncGate(ctx) { + mc.logger.Debug("MCP library sync cancelled by custom gate") + return 0, nil + } + count, err := mc.syncMCPLibraryNow(ctx) + if err != nil { + return 0, err + } + mc.syncMu.Lock() + mc.lastMCPLibrarySyncedAt = time.Now() + mc.syncMu.Unlock() + return count, nil +} + +func (mc *ModelCatalog) syncMCPLibraryNow(ctx context.Context) (int, error) { + if mc.configStore == nil { + return 0, nil + } + url := mc.getMCPLibraryURL() + count, err := SyncMCPLibrary(ctx, url, mc.configStore) + if err != nil { + return 0, err + } + mc.logger.Info("MCP library sync completed: %d entries synced from %s", count, url) + return count, nil +} + +// getMCPLibraryURL returns a copy of the MCP library URL under mutex protection. +func (mc *ModelCatalog) getMCPLibraryURL() string { + mc.syncMu.RLock() + defer mc.syncMu.RUnlock() + return mc.mcpLibraryURL +} + +// fetchMCPLibrary downloads and parses the MCP library JSON from the given URL. +func fetchMCPLibrary(ctx context.Context, rawURL string) ([]MCPLibraryEntry, error) { + parsed, err := url.Parse(rawURL) + if err != nil { + return nil, fmt.Errorf("failed to parse MCP library URL: %w", err) + } + + var data []byte + if parsed.Scheme == "file" { + f, err := os.Open(parsed.Path) + if err != nil { + return nil, fmt.Errorf("failed to open MCP library file: %w", err) + } + defer f.Close() + data, err = io.ReadAll(io.LimitReader(f, maxMCPLibraryBodyBytes+1)) + if err != nil { + return nil, fmt.Errorf("failed to read MCP library file: %w", err) + } + if int64(len(data)) > maxMCPLibraryBodyBytes { + return nil, fmt.Errorf("MCP library file exceeds %d bytes", maxMCPLibraryBodyBytes) + } + } else { + if err := bifrost.ValidateExternalURL(rawURL, true); err != nil { + return nil, fmt.Errorf("MCP library URL validation failed: %w", err) + } + client := &http.Client{Timeout: DefaultMCPLibraryTimeout} + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, fmt.Errorf("failed to create HTTP request: %w", err) + } + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to download MCP library data: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to download MCP library data: HTTP %d", resp.StatusCode) + } + + data, err = io.ReadAll(io.LimitReader(resp.Body, maxMCPLibraryBodyBytes+1)) + if err != nil { + return nil, fmt.Errorf("failed to read MCP library response: %w", err) + } + if int64(len(data)) > maxMCPLibraryBodyBytes { + return nil, fmt.Errorf("MCP library response exceeds %d bytes", maxMCPLibraryBodyBytes) + } + } + + var payload MCPLibraryPayload + if err := json.Unmarshal(data, &payload); err != nil { + return nil, fmt.Errorf("failed to unmarshal MCP library data: %w", err) + } + + return payload.Servers, nil +} + +// Slugify derives a URL/identifier-safe slug from a display name: lowercase, +// non-alphanumeric runs collapsed to a single "-", and leading/trailing "-" +// trimmed. Used to key custom library entries off their name so the existing +// unique slug index detects duplicates. Returns "" for names with no +// alphanumeric content (the caller rejects an empty slug). +// +// NOTE: only ASCII letters and digits are retained — accented/unicode characters +// (e.g. "données") are stripped, which may produce unexpected slugs for non-ASCII +// names. This is intentional to keep slugs URL-safe without a transliteration +// dependency; callers should be aware of this limitation. +func Slugify(name string) string { + var b strings.Builder + prevDash := false + for _, r := range strings.ToLower(name) { + switch { + case (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9'): + b.WriteRune(r) + prevDash = false + default: + if !prevDash && b.Len() > 0 { + b.WriteByte('-') + prevDash = true + } + } + } + return strings.Trim(b.String(), "-") +} diff --git a/framework/modelcatalog/mcplibrarysync_test.go b/framework/modelcatalog/mcplibrarysync_test.go new file mode 100644 index 0000000000..0f9fbee8e1 --- /dev/null +++ b/framework/modelcatalog/mcplibrarysync_test.go @@ -0,0 +1,144 @@ +package modelcatalog + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/maximhq/bifrost/core/schemas" + "github.com/stretchr/testify/require" +) + +func TestFetchMCPLibraryFromFileURL(t *testing.T) { + path := filepath.Join(t.TempDir(), "servers.json") + payload := `{"servers":[{"name":"Filesystem","connection_type":"stdio","auth_type":"none"}]}` + require.NoError(t, os.WriteFile(path, []byte(payload), 0o600)) + + entries, err := fetchMCPLibrary(context.Background(), "file://"+path) + require.NoError(t, err) + require.Len(t, entries, 1) + require.Equal(t, "Filesystem", entries[0].Name) + require.Equal(t, schemas.MCPConnectionTypeSTDIO, entries[0].ConnectionType) +} + +func TestWithRetries_TableDriven(t *testing.T) { + t.Parallel() + + errTransient := errors.New("transient") + + tests := []struct { + name string + maxRetries int + maxBackoff time.Duration + // failCount is how many times op fails before succeeding. + // Set to -1 to always fail. + failCount int + ctxTimeout time.Duration // 0 means no timeout + wantAttempts int + wantVal string + wantErr error + }{ + { + name: "succeeds after N retries", + maxRetries: 3, + maxBackoff: time.Millisecond, + failCount: 2, + wantAttempts: 3, + wantVal: "ok", + }, + { + name: "succeeds on first attempt", + maxRetries: 3, + maxBackoff: time.Millisecond, + failCount: 0, + wantAttempts: 1, + wantVal: "ok", + }, + { + name: "exhausts all retries", + maxRetries: 2, + maxBackoff: time.Millisecond, + failCount: -1, + wantAttempts: 3, // 1 initial + 2 retries + wantErr: errTransient, + }, + { + // maxBackoff left high so the first retry's backoff wait (1s, + // from retryBackoffMin) outlasts the 20ms ctx timeout, forcing + // cancellation during the wait rather than after exhausting + // retries. + name: "context cancelled before success", + maxRetries: 5, + maxBackoff: time.Second, + failCount: -1, + ctxTimeout: 20 * time.Millisecond, + wantErr: context.DeadlineExceeded, + }, + { + name: "backoff does not exceed cap", + maxRetries: 6, + maxBackoff: 2 * time.Millisecond, + failCount: -1, + wantAttempts: 7, // 1 initial + 6 retries + wantErr: errTransient, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ctx := context.Background() + var cancel context.CancelFunc + if tt.ctxTimeout > 0 { + ctx, cancel = context.WithTimeout(ctx, tt.ctxTimeout) + defer cancel() + } + + attempts := 0 + start := time.Now() + + op := func() (string, error) { + attempts++ + if tt.failCount < 0 || attempts <= tt.failCount { + return "", errTransient + } + return "ok", nil + } + + val, err := withRetries(ctx, tt.maxRetries, tt.maxBackoff, op) + + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + require.Empty(t, val) + } else { + require.NoError(t, err) + require.Equal(t, tt.wantVal, val) + } + + if tt.wantAttempts > 0 { + require.Equal(t, tt.wantAttempts, attempts) + } + + // For the context-cancellation case, verify that attempts stopped + // early (well before maxRetries+1). + if tt.ctxTimeout > 0 { + require.Less(t, attempts, tt.maxRetries+1, + "expected context cancellation to stop retries early") + } + + // For the backoff-cap case, verify total elapsed time stays + // bounded. With maxBackoff=2ms and 6 retries, uncapped + // exponential would be 1+2+4+8+16+32 = 63ms. Capped at 2ms + // it's at most 6*2 = 12ms. We allow a generous margin. + if tt.name == "backoff does not exceed cap" { + elapsed := time.Since(start) + require.Less(t, elapsed, 200*time.Millisecond, + "total elapsed time suggests backoff was not capped") + } + }) + } +} diff --git a/framework/modelcatalog/models.go b/framework/modelcatalog/models.go index d91b337d98..b28454669c 100644 --- a/framework/modelcatalog/models.go +++ b/framework/modelcatalog/models.go @@ -7,228 +7,225 @@ import ( "github.com/maximhq/bifrost/core/schemas" "github.com/maximhq/bifrost/framework/configstore" - configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" ) -// GetModelCapabilityEntryForModel returns capability metadata for a model/provider pair. -// It prefers chat, then responses, then text-completion entries; if none exist, -// it falls back to the lexicographically first available mode for deterministic behavior. -func (mc *ModelCatalog) GetModelCapabilityEntryForModel(model string, provider schemas.ModelProvider) *PricingEntry { - mc.mu.RLock() - defer mc.mu.RUnlock() - - if entry := mc.getCapabilityEntryForExactModelUnsafe(model, provider); entry != nil { - return entry - } - - baseModel := mc.getBaseModelNameUnsafe(model) - if baseModel != model { - if entry := mc.getCapabilityEntryForExactModelUnsafe(baseModel, provider); entry != nil { - return entry +// GetModelsForProvider returns the effective allowed model set for the +// provider. Filtered live entries are authoritative when present (they were +// pre-gated by ListModelsPipeline against the key's allow/block/aliases); +// otherwise the datasheet view is filtered by the keyconfig aggregates. +func (mc *ModelCatalog) GetModelsForProvider(provider schemas.ModelProvider) []string { + blacklisted := mc.keyconf.BlacklistedFor(provider) + allowed := mc.keyconf.AllowedFor(provider) + + var out []string + if liveModels := mc.live.ModelsForProvider(provider); len(liveModels) > 0 { + out = liveModels + } else if datasheetModels := mc.datasheet.DatasheetModelsForProvider(provider); len(datasheetModels) > 0 && allowed != nil { + out = make([]string, 0, len(datasheetModels)) + for _, m := range datasheetModels { + if blacklisted.IsBlocked(m) { + continue + } + if allowed.IsAllowed(m) { + out = append(out, m) + } } + } else { + out = []string{} } - if entry := mc.getCapabilityEntryForModelFamilyUnsafe(baseModel, provider); entry != nil { - return entry + seen := make(map[string]struct{}, len(out)) + for _, m := range out { + seen[m] = struct{}{} } - - return nil -} - -// GetModelsForProvider returns all available models for a given provider (thread-safe) -func (mc *ModelCatalog) GetModelsForProvider(provider schemas.ModelProvider) []string { - mc.mu.RLock() - defer mc.mu.RUnlock() - - models, exists := mc.modelPool[provider] - if !exists { - return []string{} + for _, e := range mc.keyconf.EntriesFor(provider) { + if !e.Enabled { + continue + } + for alias := range e.Aliases { + if blacklisted.IsBlocked(alias) { + continue + } + if allowed == nil || !allowed.IsAllowed(alias) { + continue + } + if _, ok := seen[alias]; ok { + continue + } + seen[alias] = struct{}{} + out = append(out, alias) + } + for _, m := range e.Allowed { + if m == "*" || blacklisted.IsBlocked(m) { + continue + } + if _, ok := seen[m]; ok { + continue + } + seen[m] = struct{}{} + out = append(out, m) + } } - - // Return a copy to prevent external modification - result := make([]string, len(models)) - copy(result, models) - return result + return out } -// GetUnfilteredModelsForProvider returns all available models for a given provider (thread-safe) +// GetUnfilteredModelsForProvider returns the raw catalog view (no gate +// applied): union of live unfiltered entries and the datasheet view. func (mc *ModelCatalog) GetUnfilteredModelsForProvider(provider schemas.ModelProvider) []string { - mc.mu.RLock() - defer mc.mu.RUnlock() - - models, exists := mc.unfilteredModelPool[provider] - if !exists { - return []string{} + liveModels := mc.live.UnfilteredModelsForProvider(provider) + datasheetModels := mc.datasheet.DatasheetModelsForProvider(provider) + if len(liveModels) == 0 { + return datasheetModels } - - // Return a copy to prevent external modification - result := make([]string, len(models)) - copy(result, models) - return result + if len(datasheetModels) == 0 { + return liveModels + } + seen := make(map[string]struct{}, len(liveModels)+len(datasheetModels)) + out := make([]string, 0, len(liveModels)+len(datasheetModels)) + for _, m := range liveModels { + if _, ok := seen[m]; !ok { + seen[m] = struct{}{} + out = append(out, m) + } + } + for _, m := range datasheetModels { + if _, ok := seen[m]; !ok { + seen[m] = struct{}{} + out = append(out, m) + } + } + slices.Sort(out) + return out } -// GetDistinctBaseModelNames returns all unique base model names from the catalog (thread-safe). -// This is used for governance model selection when no specific provider is chosen. +// GetDistinctBaseModelNames returns all unique base model names from the +// datasheet. Used by governance for cross-provider model selection. func (mc *ModelCatalog) GetDistinctBaseModelNames() []string { - mc.mu.RLock() - defer mc.mu.RUnlock() - - seen := make(map[string]bool) - for _, baseName := range mc.baseModelIndex { - seen[baseName] = true - } - - result := make([]string, 0, len(seen)) - for name := range seen { - result = append(result, name) - } - return result + return mc.datasheet.DistinctBaseModelNames() } -// GetProvidersForModel returns all providers for a given model (thread-safe) +// GetProvidersForModel returns every provider that can serve the model. +// Composes across stores and applies the cross-provider special cases +// (openrouter / vertex / groq-gpt / bedrock-claude) preserved verbatim from +// the pre-refactor implementation. func (mc *ModelCatalog) GetProvidersForModel(model string) []schemas.ModelProvider { - mc.mu.RLock() - defer mc.mu.RUnlock() + baseModel := mc.datasheet.BaseModelName(model) providers := make([]schemas.ModelProvider, 0) - for provider, models := range mc.modelPool { - isModelMatch := false + seen := make(map[schemas.ModelProvider]struct{}) + for _, p := range mc.knownProviders() { + models := mc.GetModelsForProvider(p) + matched := false for _, m := range models { - if m == model || mc.getBaseModelNameUnsafe(m) == mc.getBaseModelNameUnsafe(model) { - isModelMatch = true + if m == model || mc.datasheet.BaseModelName(m) == baseModel { + matched = true break } } - if isModelMatch { - providers = append(providers, provider) + if matched { + if _, ok := seen[p]; !ok { + providers = append(providers, p) + seen[p] = struct{}{} + } } } - // Handler special provider cases - // 1. Handler openrouter models - if !slices.Contains(providers, schemas.OpenRouter) { - for _, provider := range providers { - if openRouterModels, ok := mc.modelPool[schemas.OpenRouter]; ok { - if slices.Contains(openRouterModels, string(provider)+"/"+model) { - providers = append(providers, schemas.OpenRouter) - } + // Cross-provider special cases + if _, ok := seen[schemas.OpenRouter]; !ok { + openRouterModels := mc.GetModelsForProvider(schemas.OpenRouter) + for _, p := range providers { + if slices.Contains(openRouterModels, string(p)+"/"+model) { + providers = append(providers, schemas.OpenRouter) + seen[schemas.OpenRouter] = struct{}{} + break } } } - - // 2. Handle vertex models - if !slices.Contains(providers, schemas.Vertex) { - for _, provider := range providers { - if vertexModels, ok := mc.modelPool[schemas.Vertex]; ok { - if slices.Contains(vertexModels, string(provider)+"/"+model) { - providers = append(providers, schemas.Vertex) - } + if _, ok := seen[schemas.Vertex]; !ok { + vertexModels := mc.GetModelsForProvider(schemas.Vertex) + for _, p := range providers { + if slices.Contains(vertexModels, string(p)+"/"+model) { + providers = append(providers, schemas.Vertex) + seen[schemas.Vertex] = struct{}{} + break } } } - - // 3. Handle openai models for groq - if !slices.Contains(providers, schemas.Groq) && strings.Contains(model, "gpt-") { - if groqModels, ok := mc.modelPool[schemas.Groq]; ok { - if slices.Contains(groqModels, "openai/"+model) { - providers = append(providers, schemas.Groq) + if _, ok := seen[schemas.Groq]; !ok && strings.Contains(model, "gpt-") { + if slices.Contains(mc.GetModelsForProvider(schemas.Groq), "openai/"+model) { + providers = append(providers, schemas.Groq) + } + } + if _, ok := seen[schemas.Bedrock]; !ok && strings.Contains(model, "claude") { + for _, bedrockModel := range mc.GetModelsForProvider(schemas.Bedrock) { + if strings.Contains(bedrockModel, model) { + providers = append(providers, schemas.Bedrock) + break } } } - // 4. Handle anthropic models for bedrock - if !slices.Contains(providers, schemas.Bedrock) && strings.Contains(model, "claude") { - if bedrockModels, ok := mc.modelPool[schemas.Bedrock]; ok { - for _, bedrockModel := range bedrockModels { - if strings.Contains(bedrockModel, model) { - providers = append(providers, schemas.Bedrock) - break - } - } + for _, p := range mc.keyconf.Providers() { + if _, ok := seen[p]; ok { + continue + } + if mc.keyconf.BlacklistedFor(p).IsBlocked(model) { + continue + } + allowed := mc.keyconf.AllowedFor(p) + matched := false + if _, hit := mc.keyconf.ResolveAlias(p, model); hit && allowed.IsAllowed(model) { + matched = true + } else if allowed.Contains(model) { + matched = true + } else if allowed.IsUnrestricted() && + len(mc.datasheet.DatasheetModelsForProvider(p)) == 0 && + len(mc.live.UnfilteredModelsForProvider(p)) == 0 { + matched = true + } + if matched { + providers = append(providers, p) + seen[p] = struct{}{} } } return providers } -// IsModelAllowedForProvider checks if a model is allowed for a specific provider -// based on the allowed models list and catalog data. It handles all cross-provider -// logic including provider-prefixed models and special routing rules. -// -// Parameters: -// - provider: The provider to check against -// - model: The model name (without provider prefix, e.g., "gpt-4o" or "claude-3-5-sonnet") -// - allowedModels: List of allowed model names (can be empty, can include provider prefixes) -// -// Behavior: -// - If allowedModels is ["*"]: Uses model catalog to check if provider supports the model -// (delegates to GetProvidersForModel which handles all cross-provider logic) -// - If allowedModels is empty ([]): Deny-by-default — returns false for any provider/model pair -// - If allowedModels is not empty: Checks if model matches any entry in the list -// Provider-specific validation: -// - Direct matches: "gpt-4o" in allowedModels for any provider -// - Prefixed matches: Only if the prefixed model exists in provider's catalog -// (e.g., "openai/gpt-4o" in allowedModels only matches if openrouter's catalog -// contains "openai/gpt-4o" AND the model part matches the request) -// -// Returns: -// - bool: true if the model is allowed for the provider, false otherwise +// IsModelAllowedForProvider checks whether the model is allowed for the +// provider given an explicit allowedModels list (used by VK governance +// checks, not by the static keyconfig allow set). // -// Examples: -// -// // Wildcard allowedModels - uses catalog to check provider support -// mc.IsModelAllowedForProvider("openrouter", "claude-3-5-sonnet", []string{"*"}) -// // Returns: true (catalog knows openrouter has "anthropic/claude-3-5-sonnet") -// -// // Empty allowedModels - deny all (deny-by-default) -// mc.IsModelAllowedForProvider("openrouter", "claude-3-5-sonnet", []string{}) -// // Returns: false (no models are permitted) -// -// // Explicit allowedModels with prefix - validates against catalog -// mc.IsModelAllowedForProvider("openrouter", "gpt-4o", []string{"openai/gpt-4o"}) -// // Returns: true (openrouter's catalog contains "openai/gpt-4o" AND model part is "gpt-4o") -// -// // Explicit allowedModels with prefix - wrong model -// mc.IsModelAllowedForProvider("openrouter", "claude-3-5-sonnet", []string{"openai/gpt-4o"}) -// // Returns: false (model part "gpt-4o" doesn't match request "claude-3-5-sonnet") -// -// // Explicit allowedModels without prefix -// mc.IsModelAllowedForProvider("openai", "gpt-4o", []string{"gpt-4o"}) -// // Returns: true (direct match) +// - allowedModels=["*"]: defer to GetProvidersForModel (with custom-provider +// fast path when list-models is disabled). +// - allowedModels=[]: deny-by-default. +// - explicit allowedModels: direct or provider-prefixed match against the +// provider's catalog. func (mc *ModelCatalog) IsModelAllowedForProvider(provider schemas.ModelProvider, model string, providerConfig *configstore.ProviderConfig, allowedModels schemas.WhiteList) bool { - // Case 1: ["*"] = allow all models; use catalog to determine support - // Empty allowedModels = deny all (fail-safe deny-by-default) + isCustomProvider := false + hasListModelsEndpointDisabled := false + if providerConfig != nil && providerConfig.CustomProviderConfig != nil { + isCustomProvider = true + hasListModelsEndpointDisabled = !providerConfig.CustomProviderConfig.IsOperationAllowed(schemas.ListModelsRequest) + } + if allowedModels.IsUnrestricted() { - // Providers whose models the catalog cannot enumerate (custom without list-models, - // or keyless self-hosted vLLM/Ollama/SGL) cannot be cross-checked against catalog - // membership, so a wildcard allow-list permits any model for them. - if mc.IsCatalogOpaqueProvider(provider, providerConfig) { + if isCustomProvider && hasListModelsEndpointDisabled { return true } - supportedProviders := mc.GetProvidersForModel(model) - return slices.Contains(supportedProviders, provider) + return slices.Contains(mc.GetProvidersForModel(model), provider) } if allowedModels.IsEmpty() { return false } - // Case 2: Explicit allowedModels = check if model matches any entry - // Get provider's catalog models for validation of prefixed entries providerCatalogModels := mc.GetModelsForProvider(provider) - for _, allowedModel := range allowedModels { - // Direct match: "gpt-4o" == "gpt-4o" if allowedModel == model { return true } - - // Provider-prefixed match: verify it exists in provider's catalog first - // This ensures we only allow provider-specific model combinations that are actually supported if strings.Contains(allowedModel, "/") { - // Check if this exact prefixed model exists in the provider's catalog - // e.g., for openrouter, check if "openai/gpt-4o" is in its catalog if slices.Contains(providerCatalogModels, allowedModel) { - // Extract the model part and compare with request _, modelPart := schemas.ParseModelString(allowedModel, "") if modelPart == model { return true @@ -236,211 +233,21 @@ func (mc *ModelCatalog) IsModelAllowedForProvider(provider schemas.ModelProvider } } } - return false } -// IsCatalogOpaqueProvider reports whether the catalog cannot enumerate the models a provider -// serves, so a wildcard ("*") allow-list must be honored as allow-all rather than cross-checked -// against catalog membership. True for custom providers and for native providers the catalog has -// no model list for (keyless self-hosted vLLM/Ollama/SGL, or providers without list-models -// support). Shared by OSS governance and enterprise load-balancing so the rule has one definition. -func (mc *ModelCatalog) IsCatalogOpaqueProvider(provider schemas.ModelProvider, providerConfig *configstore.ProviderConfig) bool { - if providerConfig != nil && providerConfig.CustomProviderConfig != nil { - // A custom provider is opaque only when it cannot list its models. If it supports the - // list-models endpoint, the catalog can enumerate its models, so it is NOT opaque. - return !providerConfig.CustomProviderConfig.IsOperationAllowed(schemas.ListModelsRequest) - } - if mc == nil { - return false - } - // Only an emptiness check is needed, so read modelPool directly under the - // read lock rather than calling GetModelsForProvider, which allocates and - // copies the full slice on this request-path check. - mc.mu.RLock() - defer mc.mu.RUnlock() - return len(mc.modelPool[provider]) == 0 -} - -// GetBaseModelName returns the canonical base model name for a given model string. -// It uses the pre-computed base_model from the pricing catalog when available, -// falling back to algorithmic date/version stripping for models not in the catalog. -// -// Examples: -// -// mc.GetBaseModelName("gpt-4o") // Returns: "gpt-4o" -// mc.GetBaseModelName("openai/gpt-4o") // Returns: "gpt-4o" -// mc.GetBaseModelName("gpt-4o-2024-08-06") // Returns: "gpt-4o" (algorithmic fallback) func (mc *ModelCatalog) GetBaseModelName(model string) string { - mc.mu.RLock() - defer mc.mu.RUnlock() - return mc.getBaseModelNameUnsafe(model) -} - -// getBaseModelNameUnsafe returns the canonical base model name for a given model string without locking. -// This is used to avoid locking overhead when getting the base model name for many models. -// Make sure the caller function is holding the read lock before calling this function. -// It is not safe to use this function when the model pool is being updated. -func (mc *ModelCatalog) getBaseModelNameUnsafe(model string) string { - // Step 1: Direct lookup in base model index - if base, ok := mc.baseModelIndex[model]; ok { - return base - } - - // Step 2: Strip provider prefix and try again - _, baseName := schemas.ParseModelString(model, "") - if baseName != model { - if base, ok := mc.baseModelIndex[baseName]; ok { - return base - } - } - - // Step 3: Fallback to algorithmic date/version stripping - // (for models not in the catalog, e.g., user-configured custom models) - return schemas.BaseModelName(baseName) + return mc.datasheet.BaseModelName(model) } -// IsSameModel checks if two model strings refer to the same underlying model. -// It compares the canonical base model names derived from the pricing catalog -// (or algorithmic fallback for models not in the catalog). -// -// Examples: -// -// mc.IsSameModel("gpt-4o", "gpt-4o") // true (direct match) -// mc.IsSameModel("openai/gpt-4o", "gpt-4o") // true (same base model) -// mc.IsSameModel("gpt-4o", "claude-3-5-sonnet") // false (different models) -// mc.IsSameModel("openai/gpt-4o", "anthropic/claude-3-5-sonnet") // false func (mc *ModelCatalog) IsSameModel(model1, model2 string) bool { - if model1 == model2 { - return true - } - return mc.GetBaseModelName(model1) == mc.GetBaseModelName(model2) + return mc.datasheet.IsSameModel(model1, model2) } -// DeleteModelDataForProvider deletes all model data from the pool for a given provider -func (mc *ModelCatalog) DeleteModelDataForProvider(provider schemas.ModelProvider) { - mc.mu.Lock() - defer mc.mu.Unlock() - - delete(mc.modelPool, provider) - delete(mc.unfilteredModelPool, provider) -} - -// UpsertModelDataForProvider upserts model data for a given provider -func (mc *ModelCatalog) UpsertModelDataForProvider(provider schemas.ModelProvider, modelData *schemas.BifrostListModelsResponse, allowedModels []schemas.Model) { - if modelData == nil { - return - } - mc.mu.Lock() - defer mc.mu.Unlock() - - // Populating models from pricing data for the given provider - // Provider models map - providerModels := []string{} - // Iterate through all pricing data to collect models per provider - for _, pricing := range mc.pricingData { - // Normalize provider before adding to model pool - normalizedProvider := schemas.ModelProvider(normalizeProvider(pricing.Provider)) - // We will only add models for the given provider - if normalizedProvider != provider { - continue - } - // Add model to the provider's model set (using map for deduplication) - if slices.Contains(providerModels, pricing.Model) { - continue - } - providerModels = append(providerModels, pricing.Model) - // Build base model index from pre-computed base_model field - if pricing.BaseModel != "" { - mc.baseModelIndex[pricing.Model] = pricing.BaseModel - } - } - // If modelData is empty, then we allow all models - if len(modelData.Data) == 0 && len(allowedModels) == 0 { - mc.modelPool[provider] = providerModels - return - } - // Here we make sure that we still keep the backup for model catalog intact - // So we start with a existing model pool and add the new models from incoming data - finalModelList := make([]string, 0) - seenModels := make(map[string]bool) - // Case where list models failed but we have allowed models from keys - if len(modelData.Data) == 0 && len(allowedModels) > 0 { - for _, allowedModel := range allowedModels { - parsedProvider, parsedModel := schemas.ParseModelString(allowedModel.ID, "") - if parsedProvider != provider { - continue - } - if !seenModels[parsedModel] { - seenModels[parsedModel] = true - finalModelList = append(finalModelList, parsedModel) - } - } - } - for _, model := range modelData.Data { - parsedProvider, parsedModel := schemas.ParseModelString(model.ID, "") - if parsedProvider != provider { - continue - } - if !seenModels[parsedModel] { - seenModels[parsedModel] = true - finalModelList = append(finalModelList, parsedModel) - } - } - - if len(allowedModels) == 0 { - for _, model := range providerModels { - if !seenModels[model] { - seenModels[model] = true - finalModelList = append(finalModelList, model) - } - } - } - mc.modelPool[provider] = finalModelList -} - -// UpsertUnfilteredModelDataForProvider upserts unfiltered model data for a given provider -func (mc *ModelCatalog) UpsertUnfilteredModelDataForProvider(provider schemas.ModelProvider, modelData *schemas.BifrostListModelsResponse) { - if modelData == nil { - return - } - mc.mu.Lock() - defer mc.mu.Unlock() - - // Populating models from pricing data for the given provider - providerModels := []string{} - seenModels := make(map[string]bool) - for _, pricing := range mc.pricingData { - normalizedProvider := schemas.ModelProvider(normalizeProvider(pricing.Provider)) - if normalizedProvider != provider { - continue - } - if !seenModels[pricing.Model] { - seenModels[pricing.Model] = true - providerModels = append(providerModels, pricing.Model) - } - } - for _, model := range modelData.Data { - parsedProvider, parsedModel := schemas.ParseModelString(model.ID, "") - if parsedProvider != provider { - continue - } - if !seenModels[parsedModel] { - seenModels[parsedModel] = true - providerModels = append(providerModels, parsedModel) - } - } - mc.unfilteredModelPool[provider] = providerModels -} - -// RefineModelForProvider refines the model for a given provider by performing a lookup -// in mc.modelPool and using schemas.ParseModelString to extract provider and model parts. -// e.g. "gpt-oss-120b" for groq provider -> "openai/gpt-oss-120b" -// -// Behavior: -// - When the provider's catalog (mc.modelPool) yields multiple matching models, returns an error -// - When exactly one match is found, returns the fully-qualified model (provider/model format) -// - When the provider is not handled or no refinement is needed, returns the original model unchanged +// RefineModelForProvider refines a model identifier for providers that need +// a leading "provider/" segment (Groq, Replicate). Returns the original +// model unchanged when no refinement applies, or an error when multiple +// catalog candidates match ambiguously. func (mc *ModelCatalog) RefineModelForProvider(provider schemas.ModelProvider, model string) (string, error) { switch provider { case schemas.Groq: @@ -454,179 +261,14 @@ func (mc *ModelCatalog) RefineModelForProvider(provider schemas.ModelProvider, m return model, nil } -// SetPricingOverrides replaces the full in-memory pricing override set. -func (mc *ModelCatalog) SetPricingOverrides(rows []configstoreTables.TablePricingOverride) error { - seen := make(map[string]int, len(rows)) - overrides := make([]PricingOverride, 0, len(rows)) - for i := range rows { - o, err := convertTablePricingOverrideToPricingOverride(&rows[i]) - if err != nil { - return err - } - if idx, exists := seen[o.ID]; exists { - overrides[idx] = o // last entry wins for duplicate IDs - } else { - seen[o.ID] = len(overrides) - overrides = append(overrides, o) - } - } - mc.overridesMu.Lock() - mc.rawOverrides = overrides - mc.customPricing = buildCustomPricingData(overrides) - mc.overridesMu.Unlock() - return nil -} - -// UpsertPricingOverrides inserts or replaces one or more pricing overrides in a single -// operation, rebuilding the lookup map only once at the end. -func (mc *ModelCatalog) UpsertPricingOverrides(rows ...*configstoreTables.TablePricingOverride) error { - // Deduplicate the input batch by ID (last entry wins) and build the - // incoming set for O(1) lookup when filtering existing rawOverrides. - seenIncoming := make(map[string]int, len(rows)) - overrides := make([]PricingOverride, 0, len(rows)) - for _, row := range rows { - o, err := convertTablePricingOverrideToPricingOverride(row) - if err != nil { - return err - } - if idx, exists := seenIncoming[o.ID]; exists { - overrides[idx] = o // last entry wins for duplicate IDs - } else { - seenIncoming[o.ID] = len(overrides) - overrides = append(overrides, o) - } - } - - mc.overridesMu.Lock() - defer mc.overridesMu.Unlock() - - updated := make([]PricingOverride, 0, len(mc.rawOverrides)+len(overrides)) - for _, o := range mc.rawOverrides { - if _, replacing := seenIncoming[o.ID]; !replacing { - updated = append(updated, o) - } - } - updated = append(updated, overrides...) - mc.rawOverrides = updated - mc.customPricing = buildCustomPricingData(updated) - return nil -} - -// DeletePricingOverride removes a pricing override by ID. -func (mc *ModelCatalog) DeletePricingOverride(id string) { - mc.overridesMu.Lock() - defer mc.overridesMu.Unlock() - - updated := make([]PricingOverride, 0, len(mc.rawOverrides)) - for _, o := range mc.rawOverrides { - if o.ID != id { - updated = append(updated, o) - } - } - mc.rawOverrides = updated - mc.customPricing = buildCustomPricingData(updated) -} - -// IsTextCompletionSupported checks if a model supports text completion for the given provider. -// Returns true if the model has pricing data for text completion ("text_completion"), -// false otherwise. This is used by the litellmcompat plugin to determine whether to -// convert text completion requests to chat completion requests. -func (mc *ModelCatalog) IsTextCompletionSupported(model string, provider schemas.ModelProvider) bool { - mc.mu.RLock() - defer mc.mu.RUnlock() - // Check for text completion mode in pricing data - key := makeKey(model, normalizeProvider(string(provider)), normalizeRequestType(schemas.TextCompletionRequest)) - _, ok := mc.pricingData[key] - return ok -} - -// HELPER FUNCTIONS - -func (mc *ModelCatalog) getCapabilityEntryForExactModelUnsafe(model string, provider schemas.ModelProvider) *PricingEntry { - preferredModes := []schemas.RequestType{ - schemas.ChatCompletionRequest, - schemas.ResponsesRequest, - schemas.TextCompletionRequest, - } - - for _, mode := range preferredModes { - key := makeKey(model, string(provider), normalizeRequestType(mode)) - pricing, ok := mc.pricingData[key] - if ok { - return convertTableModelPricingToPricingData(&pricing) - } - } - - prefix := model + "|" + string(provider) + "|" - matchingKeys := make([]string, 0) - for key := range mc.pricingData { - if strings.HasPrefix(key, prefix) { - matchingKeys = append(matchingKeys, key) - } - } - return mc.selectCapabilityEntryFromKeysUnsafe(matchingKeys) -} - -func (mc *ModelCatalog) getCapabilityEntryForModelFamilyUnsafe(baseModel string, provider schemas.ModelProvider) *PricingEntry { - if baseModel == "" { - return nil - } - - matchingKeys := make([]string, 0) - for key, pricing := range mc.pricingData { - if normalizeProvider(pricing.Provider) != string(provider) { - continue - } - if mc.getBaseModelNameUnsafe(pricing.Model) != baseModel { - continue - } - matchingKeys = append(matchingKeys, key) - } - return mc.selectCapabilityEntryFromKeysUnsafe(matchingKeys) -} - -func (mc *ModelCatalog) selectCapabilityEntryFromKeysUnsafe(matchingKeys []string) *PricingEntry { - if len(matchingKeys) == 0 { - return nil - } - - preferredModes := []string{ - normalizeRequestType(schemas.ChatCompletionRequest), - normalizeRequestType(schemas.ResponsesRequest), - normalizeRequestType(schemas.TextCompletionRequest), - } - - for _, mode := range preferredModes { - modeMatches := make([]string, 0) - for _, key := range matchingKeys { - parts := strings.SplitN(key, "|", 3) - if len(parts) != 3 || parts[2] != mode { - continue - } - modeMatches = append(modeMatches, key) - } - if len(modeMatches) == 0 { - continue - } - slices.Sort(modeMatches) - pricing := mc.pricingData[modeMatches[0]] - return convertTableModelPricingToPricingData(&pricing) - } - - slices.Sort(matchingKeys) - pricing := mc.pricingData[matchingKeys[0]] - return convertTableModelPricingToPricingData(&pricing) -} - // refineNestedProviderModel resolves provider-native model slugs such as -// "openai/gpt-5-nano" from a base model request like "gpt-5-nano". -// It only considers catalog entries whose leading segment is a known Bifrost provider, -// so Replicate owner/model identifiers like "meta/llama-3-8b" are left untouched. +// "openai/gpt-5-nano" from a base model request like "gpt-5-nano". Only +// considers catalog entries whose leading segment is a known Bifrost +// provider so Replicate owner/model identifiers like "meta/llama-3-8b" are +// left untouched. func (mc *ModelCatalog) refineNestedProviderModel(provider schemas.ModelProvider, model string) (string, error) { - mc.mu.RLock() - models, ok := mc.modelPool[provider] - mc.mu.RUnlock() - if !ok { + models := mc.GetModelsForProvider(provider) + if len(models) == 0 { return model, nil } @@ -637,7 +279,6 @@ func (mc *ModelCatalog) refineNestedProviderModel(provider schemas.ModelProvider if providerPart == "" || model != modelPart { continue } - candidate := string(providerPart) + "/" + modelPart if _, seen := seenCandidates[candidate]; seen { continue diff --git a/framework/modelcatalog/pool.go b/framework/modelcatalog/pool.go new file mode 100644 index 0000000000..0e51b7c3cb --- /dev/null +++ b/framework/modelcatalog/pool.go @@ -0,0 +1,135 @@ +// Editing the model pool. These methods are the push surface server.go +// orchestrates against — fetched list-models responses go into live, +// configstore key edits go into keyconfig, and the composed pool is what +// the read methods in models.go return. +package modelcatalog + +import ( + "strings" + + "github.com/maximhq/bifrost/core/schemas" +) + +func parseListModelString(model string, defaultProvider schemas.ModelProvider) (schemas.ModelProvider, string) { + provider, parsedModel := schemas.ParseModelString(model, defaultProvider) + if !strings.Contains(model, "/") { + return provider, parsedModel + } + if provider != defaultProvider || parsedModel != model { + return provider, parsedModel + } + + parts := strings.SplitN(model, "/", 2) + if len(parts) == 2 { + normalizedProvider := strings.ToLower(parts[0]) + if schemas.IsKnownProvider(normalizedProvider) { + return schemas.ModelProvider(normalizedProvider), parts[1] + } + } + + return provider, parsedModel +} + +// UpsertLive caches one (provider, keyID, unfiltered) list-models response. +func (mc *ModelCatalog) UpsertLive(provider schemas.ModelProvider, keyID string, unfiltered bool, models []string) { + mc.live.Upsert(provider, keyID, unfiltered, models) +} + +// UpsertLiveFromResponse extracts model IDs from a BifrostListModelsResponse +// (parsing "provider/model" prefixes, filtering by provider match, +// deduplicating) and pushes them into the live cache. A nil resp is a no-op +// so callers can't accidentally clear an existing cache entry by handing in +// a missing response. +func (mc *ModelCatalog) UpsertLiveFromResponse(provider schemas.ModelProvider, keyID string, unfiltered bool, resp *schemas.BifrostListModelsResponse) { + if resp == nil { + return + } + mc.live.Upsert(provider, keyID, unfiltered, extractModelIDs(resp, provider)) +} + +// InvalidateLive drops both filtered + unfiltered live entries for one key. +func (mc *ModelCatalog) InvalidateLive(provider schemas.ModelProvider, keyID string) { + mc.live.Invalidate(provider, keyID) +} + +// InvalidateLiveProvider drops all live entries for a provider. +func (mc *ModelCatalog) InvalidateLiveProvider(provider schemas.ModelProvider) { + mc.live.InvalidateProvider(provider) +} + +// SetKeyConfigForProvider replaces the keyconfig snapshot for one provider. +func (mc *ModelCatalog) SetKeyConfigForProvider(provider schemas.ModelProvider, keys []schemas.Key) { + mc.keyconf.SetProvider(provider, keys) +} + +// ReplaceKeyConfig atomically resets the keyconfig snapshot for all providers. +func (mc *ModelCatalog) ReplaceKeyConfig(snapshot map[schemas.ModelProvider][]schemas.Key) { + mc.keyconf.Replace(snapshot) +} + +// RemoveKeyConfigForProvider drops keyconfig state for the provider. +func (mc *ModelCatalog) RemoveKeyConfigForProvider(provider schemas.ModelProvider) { + mc.keyconf.RemoveProvider(provider) +} + +// KeyConfigEntries returns the per-key entries for one provider (used by +// orchestration to know which keys to fan list-models calls across). +func (mc *ModelCatalog) KeyConfigEntries(provider schemas.ModelProvider) []KeyConfigEntry { + return mc.keyconf.EntriesFor(provider) +} + +// ResolveAlias returns which key owns an alias on the provider and its +// AliasConfig. +func (mc *ModelCatalog) ResolveAlias(provider schemas.ModelProvider, model string) (AliasOwner, bool) { + return mc.keyconf.ResolveAlias(provider, model) +} + +// KeysAllowingModel returns the IDs of enabled keys that can serve the model. +func (mc *ModelCatalog) KeysAllowingModel(provider schemas.ModelProvider, model string) []string { + return mc.keyconf.KeysAllowingModel(provider, model) +} + +// AllowedModelsForProvider returns the aggregated whitelist for the +// provider (union of enabled keys' Models minus per-key blacklists, or +// ["*"] when any key is unrestricted). Used by the load balancer to know +// what each provider can serve without re-walking the configstore. +func (mc *ModelCatalog) AllowedModelsForProvider(provider schemas.ModelProvider) schemas.WhiteList { + return mc.keyconf.AllowedFor(provider) +} + +// BlacklistedModelsForProvider returns the intersection of enabled keys' +// BlacklistedModels for the provider — a model is included only when every +// enabled key blacklists it. +func (mc *ModelCatalog) BlacklistedModelsForProvider(provider schemas.ModelProvider) schemas.BlackList { + return mc.keyconf.BlacklistedFor(provider) +} + +// ConfiguredProviders returns every provider with at least one entry in +// keyconfig. Used by the load balancer's provider selection where the +// configured-provider set is the routing-eligible universe. +func (mc *ModelCatalog) ConfiguredProviders() []schemas.ModelProvider { + return mc.keyconf.Providers() +} + +// extractModelIDs flattens a list-models response into bare model +// identifiers, filtering entries whose ID prefix doesn't match the +// requested provider. +func extractModelIDs(resp *schemas.BifrostListModelsResponse, provider schemas.ModelProvider) []string { + if resp == nil { + return nil + } + seen := make(map[string]struct{}, len(resp.Data)) + out := make([]string, 0, len(resp.Data)) + for _, m := range resp.Data { + parsedProvider, parsedModel := parseListModelString(m.ID, "") + if parsedProvider != "" && parsedProvider != provider { + continue + } + if _, ok := seen[parsedModel]; ok { + continue + } + seen[parsedModel] = struct{}{} + out = append(out, parsedModel) + } + return out +} diff --git a/framework/modelcatalog/pool_test.go b/framework/modelcatalog/pool_test.go new file mode 100644 index 0000000000..88cf6d5f43 --- /dev/null +++ b/framework/modelcatalog/pool_test.go @@ -0,0 +1,184 @@ +package modelcatalog + +import ( + "slices" + "testing" + + "github.com/maximhq/bifrost/core/schemas" +) + +// TestUpsertLiveFromResponse_NilRespIsNoop guards the API surface: handing a +// nil response into UpsertLiveFromResponse must not clear an existing cache +// entry by storing an empty slice. Without the early-return, extractModelIDs +// would return nil and the live store would publish an empty model list, +// silently removing the provider's previously-fetched models from routing. +func TestUpsertLiveFromResponse_NilRespIsNoop(t *testing.T) { + mc := NewTestCatalog(nil) + mc.UpsertLive(schemas.OpenAI, "k1", false, []string{"gpt-4o", "o1"}) + + mc.UpsertLiveFromResponse(schemas.OpenAI, "k1", false, nil) + + got := mc.GetModelsForProvider(schemas.OpenAI) + slices.Sort(got) + want := []string{"gpt-4o", "o1"} + if !slices.Equal(got, want) { + t.Errorf("after UpsertLiveFromResponse(nil), GetModelsForProvider = %v, want %v (entry must survive nil resp)", got, want) + } +} + +// TestUpsertLiveFromResponse_PopulatesFromResponse covers the happy path: +// extractModelIDs strips the owning provider prefix and the resulting bare +// names land in the live cache for GetModelsForProvider. +func TestUpsertLiveFromResponse_PopulatesFromResponse(t *testing.T) { + mc := NewTestCatalog(nil) + resp := &schemas.BifrostListModelsResponse{ + Data: []schemas.Model{ + {ID: "openai/gpt-4o"}, + {ID: "openai/o1"}, + }, + } + mc.UpsertLiveFromResponse(schemas.OpenAI, "k1", false, resp) + + got := mc.GetModelsForProvider(schemas.OpenAI) + slices.Sort(got) + want := []string{"gpt-4o", "o1"} + if !slices.Equal(got, want) { + t.Errorf("GetModelsForProvider = %v, want %v", got, want) + } +} + +// TestExtractModelIDs_StripsOwningProviderPrefix verifies the canonical +// shape returned by every provider's ListModels — an ID prefixed with its +// own provider key — gets reduced to a bare model name. +func TestExtractModelIDs_StripsOwningProviderPrefix(t *testing.T) { + resp := &schemas.BifrostListModelsResponse{ + Data: []schemas.Model{ + {ID: "openai/gpt-4o"}, + {ID: "openai/o1"}, + }, + } + got := extractModelIDs(resp, schemas.OpenAI) + slices.Sort(got) + want := []string{"gpt-4o", "o1"} + if !slices.Equal(got, want) { + t.Errorf("extractModelIDs = %v, want %v", got, want) + } +} + +func TestExtractModelIDs_StripsCaseVariantOwningProviderPrefix(t *testing.T) { + resp := &schemas.BifrostListModelsResponse{ + Data: []schemas.Model{ + {ID: "OpenAI/gpt-4o"}, + {ID: "openai/o1"}, + }, + } + got := extractModelIDs(resp, schemas.OpenAI) + slices.Sort(got) + want := []string{"gpt-4o", "o1"} + if !slices.Equal(got, want) { + t.Errorf("extractModelIDs = %v, want %v", got, want) + } +} + +// TestExtractModelIDs_KeepsNestedProviderForGateway covers the +// gateway-provider shape (OpenRouter returns IDs like "openrouter/openai/gpt-4") +// — ParseModelString splits on the first slash, so the parsed prefix matches +// the owning provider and the remainder ("openai/gpt-4") is kept as-is. +func TestExtractModelIDs_KeepsNestedProviderForGateway(t *testing.T) { + resp := &schemas.BifrostListModelsResponse{ + Data: []schemas.Model{ + {ID: "openrouter/openai/gpt-4"}, + {ID: "openrouter/anthropic/claude-sonnet-4"}, + }, + } + got := extractModelIDs(resp, schemas.OpenRouter) + slices.Sort(got) + want := []string{"anthropic/claude-sonnet-4", "openai/gpt-4"} + if !slices.Equal(got, want) { + t.Errorf("extractModelIDs = %v, want %v", got, want) + } +} + +// TestExtractModelIDs_DropsForeignPrefix asserts the defensive filter: an +// ID prefixed with a different provider than the one being upserted is +// excluded. This shouldn't fire in practice (providers self-prefix their +// own list-models output before it reaches here), but the guard exists for +// malformed inputs and the test pins the behavior so refactors don't +// silently invert it. +func TestExtractModelIDs_DropsForeignPrefix(t *testing.T) { + resp := &schemas.BifrostListModelsResponse{ + Data: []schemas.Model{ + {ID: "openai/gpt-4o"}, + {ID: "anthropic/claude-sonnet"}, // foreign — should be dropped + }, + } + got := extractModelIDs(resp, schemas.OpenAI) + slices.Sort(got) + want := []string{"gpt-4o"} + if !slices.Equal(got, want) { + t.Errorf("extractModelIDs = %v, want %v (anthropic-prefixed entry must be dropped when caller asks for openai)", got, want) + } +} + +// TestExtractModelIDs_NilResp returns nil — the public wrapper relies on +// this to short-circuit cleanly when a list-models call returns no body. +func TestExtractModelIDs_NilResp(t *testing.T) { + if got := extractModelIDs(nil, schemas.OpenAI); got != nil { + t.Errorf("extractModelIDs(nil) = %v, want nil", got) + } +} + +// TestExtractModelIDs_Dedup keeps only one entry when the same bare model +// name appears twice in the response (one prefixed, one bare). +func TestExtractModelIDs_Dedup(t *testing.T) { + resp := &schemas.BifrostListModelsResponse{ + Data: []schemas.Model{ + {ID: "openai/gpt-4o"}, + {ID: "gpt-4o"}, + {ID: "openai/gpt-4o"}, + }, + } + got := extractModelIDs(resp, schemas.OpenAI) + if len(got) != 1 || got[0] != "gpt-4o" { + t.Errorf("extractModelIDs = %v, want [gpt-4o] (deduped)", got) + } +} + +// TestInvalidateLive_DropsBothFiltersForKey verifies the InvalidateLive +// forwarder reaches the live store and clears filtered + unfiltered entries +// for one (provider, keyID) pair in a single call. +func TestInvalidateLive_DropsBothFiltersForKey(t *testing.T) { + mc := NewTestCatalog(nil) + mc.UpsertLive(schemas.OpenAI, "k1", false, []string{"gpt-4o"}) + mc.UpsertLive(schemas.OpenAI, "k1", true, []string{"gpt-4o", "o1"}) + mc.UpsertLive(schemas.OpenAI, "k2", false, []string{"o1"}) + + mc.InvalidateLive(schemas.OpenAI, "k1") + + // k1 entries are gone; k2 survives. + if got := mc.GetModelsForProvider(schemas.OpenAI); !slices.Equal(got, []string{"o1"}) { + t.Errorf("filtered union after InvalidateLive(k1) = %v, want [o1] (k1 filtered dropped, k2 survives)", got) + } + if got := mc.GetUnfilteredModelsForProvider(schemas.OpenAI); len(got) != 0 { + t.Errorf("unfiltered union after InvalidateLive(k1) = %v, want [] (k1 unfiltered dropped; k2 has no unfiltered entry)", got) + } +} + +// TestInvalidateLiveProvider_DropsAcrossKeys verifies the provider-wide +// forwarder clears every (keyID, mode) combination for the provider. +func TestInvalidateLiveProvider_DropsAcrossKeys(t *testing.T) { + mc := NewTestCatalog(nil) + mc.UpsertLive(schemas.OpenAI, "k1", false, []string{"gpt-4o"}) + mc.UpsertLive(schemas.OpenAI, "k2", false, []string{"o1"}) + mc.UpsertLive(schemas.Anthropic, "k1", false, []string{"claude-sonnet"}) + + mc.InvalidateLiveProvider(schemas.OpenAI) + + if got := mc.GetModelsForProvider(schemas.OpenAI); len(got) != 0 { + t.Errorf("OpenAI after InvalidateLiveProvider = %v, want [] (every key dropped)", got) + } + // Other providers untouched. + if got := mc.GetModelsForProvider(schemas.Anthropic); !slices.Equal(got, []string{"claude-sonnet"}) { + t.Errorf("Anthropic after InvalidateLiveProvider(OpenAI) = %v, want [claude-sonnet]", got) + } +} diff --git a/framework/modelcatalog/pricing.go b/framework/modelcatalog/pricing.go index d7a2d507e2..f4e33ec415 100644 --- a/framework/modelcatalog/pricing.go +++ b/framework/modelcatalog/pricing.go @@ -2,1480 +2,60 @@ package modelcatalog import ( "context" - "fmt" - "strconv" - "strings" - "github.com/bytedance/sonic" "github.com/maximhq/bifrost/core/schemas" configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" + "github.com/maximhq/bifrost/framework/modelcatalog/datasheet" ) -// Default sync interval and config key -const ( - TokenTierAbove272K = 272000 - TokenTierAbove200K = 200000 - TokenTierAbove128K = 128000 -) - -// PricingEntry represents a single model's pricing information. -// Field names and JSON tags match the datasheet schema exactly. -// AdditionalAttributes carries editorial metadata stored on the pricing row, never populated from the URL datasheet — only from DB reads via the management API. -type PricingEntry struct { - BaseModel string `json:"base_model,omitempty"` - Provider string `json:"provider"` - Mode string `json:"mode"` - - ContextLength *int `json:"context_length,omitempty"` - MaxInputTokens *int `json:"max_input_tokens,omitempty"` - MaxOutputTokens *int `json:"max_output_tokens,omitempty"` - Architecture *schemas.Architecture `json:"architecture,omitempty"` - - // AdditionalAttributes carries editorial metadata stored on the pricing - // row (e.g. description). Populated from the DB read path only; the - // json:"-" tag prevents URL datasheet payloads from ever feeding into - // this field via json.Unmarshal. - AdditionalAttributes map[string]string `json:"-"` - - PricingOptions +// GetModelCapabilityEntryForModel returns capability metadata for a +// (model, provider) pair. Prefers chat, then responses, then text-completion +// entries; falls back to the lexicographically first available mode for +// deterministic behavior. +func (mc *ModelCatalog) GetModelCapabilityEntryForModel(model string, provider schemas.ModelProvider) *PricingEntry { + return mc.datasheet.GetCapabilityEntry(model, provider) } -// UnmarshalJSON implements json.Unmarshaler for PricingEntry. -// It handles the special case where search_context_cost_per_query may arrive as either -// a plain float64 or a tiered object {"search_context_size_low":…, -// "search_context_size_medium":…, "search_context_size_high":…}. -func (p *PricingEntry) UnmarshalJSON(data []byte) error { - // Type alias breaks the UnmarshalJSON recursion while keeping all other fields. - type PricingEntryAlias PricingEntry - var raw struct { - PricingEntryAlias - SearchContextCostPerQuery *struct { - Low *float64 `json:"search_context_size_low"` - Medium *float64 `json:"search_context_size_medium"` - High *float64 `json:"search_context_size_high"` - } `json:"search_context_cost_per_query,omitempty"` - } - if err := sonic.Unmarshal(data, &raw); err != nil { - return err - } - *p = PricingEntry(raw.PricingEntryAlias) - - // search_context_cost_per_query arrives as a tiered object – all three values are - // equal for non-Perplexity providers; we prefer medium, then low, then high. - // Perplexity always returns a pre-computed total_cost so the per-query rate is - // never consumed for that provider. - if q := raw.SearchContextCostPerQuery; q != nil { - switch { - case q.Medium != nil: - p.SearchContextCostPerQuery = q.Medium - case q.Low != nil: - p.SearchContextCostPerQuery = q.Low - case q.High != nil: - p.SearchContextCostPerQuery = q.High - } - } - return nil +// IsRequestTypeSupported preserves the historical (model, provider, +// requestType) signature; provider is ignored (the underlying datasheet +// index is keyed by model only). +func (mc *ModelCatalog) IsRequestTypeSupported(model string, provider schemas.ModelProvider, requestType schemas.RequestType) bool { + return mc.datasheet.IsRequestTypeSupported(model, requestType) } -type PricingOptions struct { - // Costs - Text - InputCostPerToken *float64 `json:"input_cost_per_token,omitempty"` - OutputCostPerToken *float64 `json:"output_cost_per_token,omitempty"` - InputCostPerTokenBatches *float64 `json:"input_cost_per_token_batches,omitempty"` - OutputCostPerTokenBatches *float64 `json:"output_cost_per_token_batches,omitempty"` - InputCostPerTokenPriority *float64 `json:"input_cost_per_token_priority,omitempty"` - OutputCostPerTokenPriority *float64 `json:"output_cost_per_token_priority,omitempty"` - InputCostPerTokenFlex *float64 `json:"input_cost_per_token_flex,omitempty"` - OutputCostPerTokenFlex *float64 `json:"output_cost_per_token_flex,omitempty"` - InputCostPerCharacter *float64 `json:"input_cost_per_character,omitempty"` - // Costs - 128k Tier - InputCostPerTokenAbove128kTokens *float64 `json:"input_cost_per_token_above_128k_tokens,omitempty"` - InputCostPerImageAbove128kTokens *float64 `json:"input_cost_per_image_above_128k_tokens,omitempty"` - InputCostPerVideoPerSecondAbove128kTokens *float64 `json:"input_cost_per_video_per_second_above_128k_tokens,omitempty"` - InputCostPerAudioPerSecondAbove128kTokens *float64 `json:"input_cost_per_audio_per_second_above_128k_tokens,omitempty"` - OutputCostPerTokenAbove128kTokens *float64 `json:"output_cost_per_token_above_128k_tokens,omitempty"` - // Costs - 200k Tier - InputCostPerTokenAbove200kTokens *float64 `json:"input_cost_per_token_above_200k_tokens,omitempty"` - InputCostPerTokenAbove200kTokensPriority *float64 `json:"input_cost_per_token_above_200k_tokens_priority,omitempty"` - OutputCostPerTokenAbove200kTokens *float64 `json:"output_cost_per_token_above_200k_tokens,omitempty"` - OutputCostPerTokenAbove200kTokensPriority *float64 `json:"output_cost_per_token_above_200k_tokens_priority,omitempty"` - // Costs - 272k Tier - InputCostPerTokenAbove272kTokens *float64 `json:"input_cost_per_token_above_272k_tokens,omitempty"` - InputCostPerTokenAbove272kTokensPriority *float64 `json:"input_cost_per_token_above_272k_tokens_priority,omitempty"` - OutputCostPerTokenAbove272kTokens *float64 `json:"output_cost_per_token_above_272k_tokens,omitempty"` - OutputCostPerTokenAbove272kTokensPriority *float64 `json:"output_cost_per_token_above_272k_tokens_priority,omitempty"` - - // Costs - Cache - CacheCreationInputTokenCost *float64 `json:"cache_creation_input_token_cost,omitempty"` - CacheReadInputTokenCost *float64 `json:"cache_read_input_token_cost,omitempty"` - CacheCreationInputTokenCostAbove200kTokens *float64 `json:"cache_creation_input_token_cost_above_200k_tokens,omitempty"` - CacheReadInputTokenCostAbove200kTokens *float64 `json:"cache_read_input_token_cost_above_200k_tokens,omitempty"` - CacheReadInputTokenCostAbove200kTokensPriority *float64 `json:"cache_read_input_token_cost_above_200k_tokens_priority,omitempty"` - CacheCreationInputTokenCostAbove1hr *float64 `json:"cache_creation_input_token_cost_above_1hr,omitempty"` - CacheCreationInputTokenCostAbove1hrAbove200kTokens *float64 `json:"cache_creation_input_token_cost_above_1hr_above_200k_tokens,omitempty"` - CacheCreationInputAudioTokenCost *float64 `json:"cache_creation_input_audio_token_cost,omitempty"` - CacheReadInputTokenCostPriority *float64 `json:"cache_read_input_token_cost_priority,omitempty"` - CacheReadInputTokenCostFlex *float64 `json:"cache_read_input_token_cost_flex,omitempty"` - CacheReadInputImageTokenCost *float64 `json:"cache_read_input_image_token_cost,omitempty"` - CacheReadInputTokenCostAbove272kTokens *float64 `json:"cache_read_input_token_cost_above_272k_tokens,omitempty"` - CacheReadInputTokenCostAbove272kTokensPriority *float64 `json:"cache_read_input_token_cost_above_272k_tokens_priority,omitempty"` - - // Costs - Image - InputCostPerImage *float64 `json:"input_cost_per_image,omitempty"` - InputCostPerPixel *float64 `json:"input_cost_per_pixel,omitempty"` - OutputCostPerImage *float64 `json:"output_cost_per_image,omitempty"` - OutputCostPerPixel *float64 `json:"output_cost_per_pixel,omitempty"` - OutputCostPerImagePremiumImage *float64 `json:"output_cost_per_image_premium_image,omitempty"` - OutputCostPerImageAbove512x512Pixels *float64 `json:"output_cost_per_image_above_512_and_512_pixels,omitempty"` - OutputCostPerImageAbove512x512PixelsPremium *float64 `json:"output_cost_per_image_above_512_and_512_pixels_and_premium_image,omitempty"` - OutputCostPerImageAbove1024x1024Pixels *float64 `json:"output_cost_per_image_above_1024_and_1024_pixels,omitempty"` - OutputCostPerImageAbove1024x1024PixelsPremium *float64 `json:"output_cost_per_image_above_1024_and_1024_pixels_and_premium_image,omitempty"` - OutputCostPerImageAbove2048x2048Pixels *float64 `json:"output_cost_per_image_above_2048_and_2048_pixels,omitempty"` - OutputCostPerImageAbove4096x4096Pixels *float64 `json:"output_cost_per_image_above_4096_and_4096_pixels,omitempty"` - OutputCostPerImageLowQuality *float64 `json:"output_cost_per_image_low_quality,omitempty"` - OutputCostPerImageMediumQuality *float64 `json:"output_cost_per_image_medium_quality,omitempty"` - OutputCostPerImageHighQuality *float64 `json:"output_cost_per_image_high_quality,omitempty"` - OutputCostPerImageAutoQuality *float64 `json:"output_cost_per_image_auto_quality,omitempty"` - InputCostPerImageToken *float64 `json:"input_cost_per_image_token,omitempty"` - OutputCostPerImageToken *float64 `json:"output_cost_per_image_token,omitempty"` - - // Costs - Audio/Video - InputCostPerAudioToken *float64 `json:"input_cost_per_audio_token,omitempty"` - InputCostPerAudioPerSecond *float64 `json:"input_cost_per_audio_per_second,omitempty"` - InputCostPerSecond *float64 `json:"input_cost_per_second,omitempty"` - InputCostPerVideoPerSecond *float64 `json:"input_cost_per_video_per_second,omitempty"` - OutputCostPerAudioToken *float64 `json:"output_cost_per_audio_token,omitempty"` - OutputCostPerVideoPerSecond *float64 `json:"output_cost_per_video_per_second,omitempty"` - OutputCostPerSecond *float64 `json:"output_cost_per_second,omitempty"` - - // Costs - Other - // - // SearchContextCostPerQuery is stored as a single float64, but the pricing datasheet - // represents it as a tiered object with three keys: search_context_size_low, - // search_context_size_medium, and search_context_size_high. For every provider except - // Perplexity the three tier values are identical, so we collapse the object to its - // medium tier value (falling back to low then high). Perplexity always returns a - // pre-computed total_cost in its usage response, so the per-query rate is never - // consumed for that provider; the collapsed value is therefore correct in all cases. - // See UnmarshalJSON below for the custom decoding logic. - SearchContextCostPerQuery *float64 `json:"search_context_cost_per_query,omitempty"` - CodeInterpreterCostPerSession *float64 `json:"code_interpreter_cost_per_session,omitempty"` - - // Costs - OCR - OCRCostPerPage *float64 `json:"ocr_cost_per_page,omitempty"` - AnnotationCostPerPage *float64 `json:"annotation_cost_per_page,omitempty"` +func (mc *ModelCatalog) GetSupportedParameters(model string) []string { + return mc.datasheet.GetSupportedParameters(model) } -// serviceTier captures the OpenAI service_tier value from a response. -// Add new tier flags here as OpenAI introduces them. -type serviceTier struct { - isPriority bool // true when service_tier == "priority" - isFlex bool // true when service_tier == "flex" +func (mc *ModelCatalog) IsTextCompletionSupported(model string, provider schemas.ModelProvider) bool { + return mc.datasheet.IsTextCompletionSupported(model, provider) } -// costInput holds the extracted usage data from a BifrostResponse, -// normalized for the pricing engine. -type costInput struct { - usage *schemas.BifrostLLMUsage - audioTextInputChars int - audioSeconds *int - audioTokenDetails *schemas.TranscriptionUsageInputTokenDetails - imageUsage *schemas.ImageUsage - imageSize string // e.g. "1024x1024", used for per-pixel pricing - imageQuality string // "low", "medium", "high", "auto" (gpt-image-1.5); empty = use base rate - videoSeconds *int - ocrProcessedPages *int - ocrIsAnnotated *bool - // containerIdentifierString, when non-empty, replaces the actual requested/resolved - // model names during pricing lookup. Used for request types whose cost is not - // tied to a specific model. Currently only used for container creates. - containerIdentifierString string - tier serviceTier -} - -// GetPricingEntryForModel returns the pricing data +// GetPricingEntryForModel returns any pricing entry for the model across +// known modes. Used by the inference handler to enrich list-models responses. func (mc *ModelCatalog) GetPricingEntryForModel(model string, provider schemas.ModelProvider) *PricingEntry { - mc.mu.RLock() - defer mc.mu.RUnlock() - // Check all modes - for _, mode := range []schemas.RequestType{ - schemas.TextCompletionRequest, - schemas.ChatCompletionRequest, - schemas.ResponsesRequest, - schemas.EmbeddingRequest, - schemas.RerankRequest, - schemas.SpeechRequest, - schemas.TranscriptionRequest, - schemas.ImageGenerationRequest, - schemas.ImageEditRequest, - schemas.ImageVariationRequest, - schemas.VideoGenerationRequest, - schemas.OCRRequest, - } { - key := makeKey(model, string(provider), normalizeRequestType(mode)) - pricing, ok := mc.pricingData[key] - if ok { - return convertTableModelPricingToPricingData(&pricing) - } - } - return nil + return mc.datasheet.GetPricingEntryForModel(model, provider) } -// CalculateCost calculates the cost of a Bifrost response. -// It handles all request types, cache debug billing, and tiered pricing. -// If scopes is nil, an empty PricingLookupScopes is used; global and provider-scoped -// overrides may still apply since the provider is derived from the response. +// CalculateCost computes the dollar cost for a Bifrost response. func (mc *ModelCatalog) CalculateCost(result *schemas.BifrostResponse, scopes *PricingLookupScopes) float64 { - if result == nil { - return 0 - } - - var s PricingLookupScopes - if scopes != nil { - s = *scopes - } - - // Handle semantic cache billing - cacheDebug := result.GetExtraFields().CacheDebug - if cacheDebug != nil { - return mc.calculateCostWithCache(result, cacheDebug, s) - } - - return mc.calculateBaseCost(result, s) -} - -// calculateCostWithCache handles cost calculation when semantic cache debug info is present. -func (mc *ModelCatalog) calculateCostWithCache(result *schemas.BifrostResponse, cacheDebug *schemas.BifrostCacheDebug, scopes PricingLookupScopes) float64 { - if cacheDebug.CacheHit { - // Direct cache hit — no LLM call, no cost - if cacheDebug.HitType != nil && *cacheDebug.HitType == "direct" { - return 0 - } - // Semantic cache hit — only the embedding lookup cost - if cacheDebug.ProviderUsed != nil && cacheDebug.ModelUsed != nil && cacheDebug.InputTokens != nil { - return mc.computeCacheEmbeddingCost(cacheDebug, scopes) - } - return 0 - } - - // Cache miss — full LLM cost + embedding lookup cost - baseCost := mc.calculateBaseCost(result, scopes) - embeddingCost := mc.computeCacheEmbeddingCost(cacheDebug, scopes) - return baseCost + embeddingCost -} - -// computeCacheEmbeddingCost calculates the embedding cost for a semantic cache lookup. -func (mc *ModelCatalog) computeCacheEmbeddingCost(cacheDebug *schemas.BifrostCacheDebug, scopes PricingLookupScopes) float64 { - if cacheDebug == nil || cacheDebug.ProviderUsed == nil || cacheDebug.ModelUsed == nil || cacheDebug.InputTokens == nil { - return 0 - } - if scopes.Provider == "" { - scopes.Provider = *cacheDebug.ProviderUsed - } - pricing := mc.resolvePricing(*cacheDebug.ProviderUsed, *cacheDebug.ModelUsed, "", schemas.EmbeddingRequest, scopes) - if pricing == nil { - return 0 - } - return float64(*cacheDebug.InputTokens) * tieredInputRate(pricing, *cacheDebug.InputTokens, serviceTier{}) -} - -// computeContainerCreationCost returns the cost for creating a container from an already-resolved pricing entry. -func computeContainerCreationCost(pricing *configstoreTables.TableModelPricing) float64 { - if pricing == nil || pricing.CodeInterpreterCostPerSession == nil { - return 0 - } - return *pricing.CodeInterpreterCostPerSession -} - -// calculateBaseCost extracts usage from the response and routes to the appropriate compute function. -func (mc *ModelCatalog) calculateBaseCost(result *schemas.BifrostResponse, scopes PricingLookupScopes) float64 { - extraFields := result.GetExtraFields() - if extraFields == nil { - return 0 - } - - provider := string(extraFields.Provider) - originalModelRequested := extraFields.OriginalModelRequested - resolvedModelUsed := extraFields.ResolvedModelUsed - requestType := extraFields.RequestType - - // Extract usage data from the response (passthrough and native paths unified) - input := extractCostInput(result) - - // If provider already computed cost, use it - if input.usage != nil && input.usage.Cost != nil && input.usage.Cost.TotalCost > 0 { - return input.usage.Cost.TotalCost - } - - // If no usage data at all, nothing to price - if input.usage == nil && input.audioSeconds == nil && input.audioTokenDetails == nil && input.imageUsage == nil && input.videoSeconds == nil && input.audioTextInputChars == 0 && input.ocrProcessedPages == nil && input.containerIdentifierString == "" { - return 0 - } - - if result.PassthroughResponse != nil { - // Infer request type from usage fields + path; passthrough bypasses stream normalization. - requestType = inferPassthroughRequestType(extraFields.Provider, extraFields.PassthroughPath, result.PassthroughResponse.PassthroughUsage) - } else { - // Normalize stream request types to their base type for pricing lookup - requestType = normalizeStreamRequestType(requestType) - } - - // When a pricing model override is set, use it in place of the actual requested/resolved - // model names during pricing lookup (e.g. container creates always look up "container"). - lookupModel, lookupResolved := originalModelRequested, resolvedModelUsed - if input.containerIdentifierString != "" { - lookupModel = input.containerIdentifierString - lookupResolved = input.containerIdentifierString - } - - // Resolve pricing entry with deployment fallback - pricing := mc.resolvePricing(provider, lookupModel, lookupResolved, requestType, scopes) - if pricing == nil { - return 0 - } - - // Route to the appropriate compute function - switch requestType { - case schemas.ChatCompletionRequest, schemas.TextCompletionRequest, schemas.ResponsesRequest, schemas.RealtimeRequest, schemas.CompactionRequest: - return computeTextCost(pricing, input.usage, input.tier) - case schemas.EmbeddingRequest: - return computeEmbeddingCost(pricing, input.usage, input.tier) - case schemas.RerankRequest: - return computeRerankCost(pricing, input.usage, input.tier) - case schemas.SpeechRequest: - return computeSpeechCost(pricing, input.usage, input.audioSeconds, input.audioTextInputChars, input.tier) - case schemas.TranscriptionRequest: - return computeTranscriptionCost(pricing, input.usage, input.audioSeconds, input.audioTokenDetails, input.tier) - case schemas.ImageGenerationRequest, schemas.ImageEditRequest, schemas.ImageVariationRequest: - return computeImageCost(pricing, input.imageUsage, input.imageSize, input.imageQuality, input.tier) - case schemas.VideoGenerationRequest, schemas.VideoRemixRequest: - return computeVideoCost(pricing, input.usage, input.videoSeconds, input.tier) - case schemas.OCRRequest: - return computeOCRCost(pricing, input.ocrProcessedPages, input.ocrIsAnnotated) - case schemas.ContainerCreateRequest: - return computeContainerCreationCost(pricing) - default: - return 0 - } -} - -// --------------------------------------------------------------------------- -// Usage extraction -// --------------------------------------------------------------------------- - -func extractCostInput(result *schemas.BifrostResponse) costInput { - var input costInput - - switch { - case result.PassthroughResponse != nil && result.PassthroughResponse.PassthroughUsage != nil: - return passthroughUsageToCostInput(result.PassthroughResponse.PassthroughUsage) - - case result.TextCompletionResponse != nil && result.TextCompletionResponse.Usage != nil: - input.usage = result.TextCompletionResponse.Usage - - case result.ChatResponse != nil && result.ChatResponse.Usage != nil: - input.usage = result.ChatResponse.Usage - input.tier = tierFromString(result.ChatResponse.ServiceTier) - - case result.ResponsesResponse != nil && result.ResponsesResponse.Usage != nil: - input.usage = responsesUsageToBifrostUsage(result.ResponsesResponse.Usage) - input.tier = tierFromString(result.ResponsesResponse.ServiceTier) - - case result.CompactionResponse != nil && result.CompactionResponse.Usage != nil: - input.usage = responsesUsageToBifrostUsage(result.CompactionResponse.Usage) - - case result.ResponsesStreamResponse != nil && result.ResponsesStreamResponse.Response != nil && result.ResponsesStreamResponse.Response.Usage != nil: - input.usage = responsesUsageToBifrostUsage(result.ResponsesStreamResponse.Response.Usage) - input.tier = tierFromString(result.ResponsesStreamResponse.Response.ServiceTier) - - case result.EmbeddingResponse != nil && result.EmbeddingResponse.Usage != nil: - input.usage = result.EmbeddingResponse.Usage - - case result.RerankResponse != nil && result.RerankResponse.Usage != nil: - input.usage = result.RerankResponse.Usage - - case result.SpeechResponse != nil && result.SpeechResponse.Usage != nil: - input.usage = speechUsageToBifrostUsage(result.SpeechResponse.Usage) - input.audioTextInputChars = result.SpeechResponse.Usage.InputChars - - case result.SpeechStreamResponse != nil && result.SpeechStreamResponse.Usage != nil: - input.usage = speechUsageToBifrostUsage(result.SpeechStreamResponse.Usage) - input.audioTextInputChars = result.SpeechStreamResponse.Usage.InputChars - - case result.TranscriptionResponse != nil && result.TranscriptionResponse.Usage != nil: - input.usage, input.audioSeconds, input.audioTokenDetails = extractTranscriptionUsage(result.TranscriptionResponse.Usage) - - case result.TranscriptionStreamResponse != nil && result.TranscriptionStreamResponse.Usage != nil: - input.usage, input.audioSeconds, input.audioTokenDetails = extractTranscriptionUsage(result.TranscriptionStreamResponse.Usage) - - case result.ImageGenerationResponse != nil: - if result.ImageGenerationResponse.Usage != nil { - input.imageUsage = result.ImageGenerationResponse.Usage - } else { - // No usage data but response exists — default to empty so per-image pricing can apply - input.imageUsage = &schemas.ImageUsage{} - } - populateOutputImageCount(input.imageUsage, len(result.ImageGenerationResponse.Data)) - if result.ImageGenerationResponse.ImageGenerationResponseParameters != nil { - input.imageSize = result.ImageGenerationResponse.ImageGenerationResponseParameters.Size - input.imageQuality = result.ImageGenerationResponse.ImageGenerationResponseParameters.Quality - } - - case result.ImageGenerationStreamResponse != nil: - if result.ImageGenerationStreamResponse.Usage != nil { - input.imageUsage = result.ImageGenerationStreamResponse.Usage - } else { - input.imageUsage = &schemas.ImageUsage{} - } - input.imageSize = result.ImageGenerationStreamResponse.Size - input.imageQuality = result.ImageGenerationStreamResponse.Quality - - case result.VideoGenerationResponse != nil && result.VideoGenerationResponse.Seconds != nil: - seconds, err := strconv.Atoi(*result.VideoGenerationResponse.Seconds) - if err == nil { - input.videoSeconds = &seconds - } - - case result.OCRResponse != nil: - pages := len(result.OCRResponse.Pages) - if result.OCRResponse.UsageInfo != nil && result.OCRResponse.UsageInfo.PagesProcessed > 0 { - pages = result.OCRResponse.UsageInfo.PagesProcessed - } - input.ocrProcessedPages = &pages - isAnnotated := result.OCRResponse.DocumentAnnotation != nil && *result.OCRResponse.DocumentAnnotation != "" - input.ocrIsAnnotated = &isAnnotated - - case result.ContainerCreateResponse != nil: - if memLimit := result.ContainerCreateResponse.MemoryLimit; memLimit != "" { - input.containerIdentifierString = "container-" + memLimit - } else { - input.containerIdentifierString = "container" - } - } - - return input -} - -func responsesUsageToBifrostUsage(u *schemas.ResponsesResponseUsage) *schemas.BifrostLLMUsage { - usage := &schemas.BifrostLLMUsage{ - PromptTokens: u.InputTokens, - CompletionTokens: u.OutputTokens, - TotalTokens: u.TotalTokens, - Cost: u.Cost, - } - // Map token details for cache and search query pricing - if u.InputTokensDetails != nil { - usage.PromptTokensDetails = &schemas.ChatPromptTokensDetails{ - TextTokens: u.InputTokensDetails.TextTokens, - AudioTokens: u.InputTokensDetails.AudioTokens, - ImageTokens: u.InputTokensDetails.ImageTokens, - CachedReadTokens: u.InputTokensDetails.CachedReadTokens, - CachedWriteTokens: u.InputTokensDetails.CachedWriteTokens, - CachedWriteTokenDetails: u.InputTokensDetails.CachedWriteTokenDetails, - } - } - if u.OutputTokensDetails != nil { - usage.CompletionTokensDetails = &schemas.ChatCompletionTokensDetails{ - ReasoningTokens: u.OutputTokensDetails.ReasoningTokens, - AudioTokens: u.OutputTokensDetails.AudioTokens, - } - if u.OutputTokensDetails.NumSearchQueries != nil { - usage.CompletionTokensDetails.NumSearchQueries = u.OutputTokensDetails.NumSearchQueries - } - } - return usage -} - -func speechUsageToBifrostUsage(u *schemas.SpeechUsage) *schemas.BifrostLLMUsage { - return &schemas.BifrostLLMUsage{ - PromptTokens: u.InputTokens, - CompletionTokens: u.OutputTokens, - TotalTokens: u.TotalTokens, - } -} - -func extractTranscriptionUsage(u *schemas.TranscriptionUsage) (*schemas.BifrostLLMUsage, *int, *schemas.TranscriptionUsageInputTokenDetails) { - usage := &schemas.BifrostLLMUsage{} - if u.InputTokens != nil { - usage.PromptTokens = *u.InputTokens - } - if u.OutputTokens != nil { - usage.CompletionTokens = *u.OutputTokens - } - if u.TotalTokens != nil { - usage.TotalTokens = *u.TotalTokens - } else { - usage.TotalTokens = usage.PromptTokens + usage.CompletionTokens - } - - var audioTokenDetails *schemas.TranscriptionUsageInputTokenDetails - if u.InputTokenDetails != nil { - audioTokenDetails = &schemas.TranscriptionUsageInputTokenDetails{ - AudioTokens: u.InputTokenDetails.AudioTokens, - TextTokens: u.InputTokenDetails.TextTokens, - } - } - - return usage, u.Seconds, audioTokenDetails -} - -// --------------------------------------------------------------------------- -// Per-request-type cost computation -// --------------------------------------------------------------------------- - -// computeTextCost handles chat, text completion, and responses requests. -func computeTextCost(pricing *configstoreTables.TableModelPricing, usage *schemas.BifrostLLMUsage, tier serviceTier) float64 { - if usage == nil { - return 0 - } - - totalTokens := usage.TotalTokens - promptTokens := usage.PromptTokens - completionTokens := usage.CompletionTokens - - // Extract cached token counts - cachedReadTokens := 0 - cachedWriteTokens := 0 - cachedWriteTokensAbove1hr := 0 - if usage.PromptTokensDetails != nil { - cachedReadTokens = usage.PromptTokensDetails.CachedReadTokens - cachedWriteTokens = usage.PromptTokensDetails.CachedWriteTokens - if usage.PromptTokensDetails.CachedWriteTokenDetails != nil { - cachedWriteTokensAbove1hr = usage.PromptTokensDetails.CachedWriteTokenDetails.CachedWriteTokens1h - } - } - - inputRate := tieredInputRate(pricing, totalTokens, tier) - outputRate := tieredOutputRate(pricing, totalTokens, tier) - cacheReadInputRate := tieredCacheReadInputTokenRate(pricing, totalTokens, tier) - cacheCreationInputRate := tieredCacheCreationInputTokenRate(pricing, totalTokens, tier) - cacheCreationInputAbove1hrInputRate := tieredCacheCreationInputAbove1hrTokenRate(pricing, totalTokens, tier) - - // Clamp cached token counts to avoid negative billing on malformed provider payloads - if cachedReadTokens > promptTokens { - cachedReadTokens = promptTokens - } - if cachedWriteTokens > promptTokens-cachedReadTokens { - cachedWriteTokens = promptTokens - cachedReadTokens - } - // Should not happen, but just in case - if cachedWriteTokensAbove1hr > cachedWriteTokens { - cachedWriteTokensAbove1hr = cachedWriteTokens - } - - // Input cost: non-cached tokens at regular rate - nonCachedPrompt := promptTokens - cachedReadTokens - cachedWriteTokens - inputCost := float64(nonCachedPrompt) * inputRate - - // Add cached prompt tokens at cache read rate - if cachedReadTokens > 0 { - inputCost += float64(cachedReadTokens) * cacheReadInputRate - } - - // Add cached write tokens at cache creation rate - if cachedWriteTokens > 0 { - if cachedWriteTokensAbove1hr > 0 { - inputCost += float64(cachedWriteTokensAbove1hr) * cacheCreationInputAbove1hrInputRate - } - inputCost += float64(cachedWriteTokens-cachedWriteTokensAbove1hr) * cacheCreationInputRate - } - - outputCost := float64(completionTokens) * outputRate - - // Audio token cost: when token details include audio tokens, price them - // at the dedicated audio rate and subtract from the text token costs above. - // Realtime and audio-enabled chat models report audio tokens in details. - audioCost := 0.0 - inputAudioTokens := 0 - outputAudioTokens := 0 - if usage.PromptTokensDetails != nil { - inputAudioTokens = usage.PromptTokensDetails.AudioTokens - } - if usage.CompletionTokensDetails != nil { - outputAudioTokens = usage.CompletionTokensDetails.AudioTokens - } - if inputAudioTokens < 0 { - inputAudioTokens = 0 - } else if inputAudioTokens > promptTokens { - inputAudioTokens = promptTokens - } - if outputAudioTokens < 0 { - outputAudioTokens = 0 - } else if outputAudioTokens > completionTokens { - outputAudioTokens = completionTokens - } - if inputAudioTokens > 0 && pricing.InputCostPerAudioToken != nil { - // Subtract audio tokens charged at text rate, add at audio rate. - audioCost += float64(inputAudioTokens) * (*pricing.InputCostPerAudioToken - inputRate) - } - if outputAudioTokens > 0 && pricing.OutputCostPerAudioToken != nil { - audioCost += float64(outputAudioTokens) * (*pricing.OutputCostPerAudioToken - outputRate) - } - - // Search query cost - searchCost := 0.0 - if pricing.SearchContextCostPerQuery != nil && usage.CompletionTokensDetails != nil && usage.CompletionTokensDetails.NumSearchQueries != nil { - searchCost = float64(*usage.CompletionTokensDetails.NumSearchQueries) * *pricing.SearchContextCostPerQuery - } - - return inputCost + outputCost + audioCost + searchCost -} - -// computeEmbeddingCost handles embedding requests (input-only). -func computeEmbeddingCost(pricing *configstoreTables.TableModelPricing, usage *schemas.BifrostLLMUsage, tier serviceTier) float64 { - if usage == nil { - return 0 - } - return float64(usage.PromptTokens) * tieredInputRate(pricing, usage.TotalTokens, tier) -} - -// computeRerankCost handles rerank requests. -func computeRerankCost(pricing *configstoreTables.TableModelPricing, usage *schemas.BifrostLLMUsage, tier serviceTier) float64 { - if usage == nil { - return 0 - } - inputCost := float64(usage.PromptTokens) * tieredInputRate(pricing, usage.TotalTokens, tier) - outputCost := float64(usage.CompletionTokens) * tieredOutputRate(pricing, usage.TotalTokens, tier) - - searchCost := 0.0 - if pricing.SearchContextCostPerQuery != nil && usage.CompletionTokensDetails != nil && usage.CompletionTokensDetails.NumSearchQueries != nil { - searchCost = float64(*usage.CompletionTokensDetails.NumSearchQueries) * *pricing.SearchContextCostPerQuery - } - - return inputCost + outputCost + searchCost -} - -// computeSpeechCost handles speech (TTS) requests. -// Input is text (PromptTokens), output is audio (CompletionTokens). -// -// Per-character pricing (InputCostPerCharacter) is used as first-class support for TTS/audio -// models — providers such as OpenAI TTS, ElevenLabs, and AWS Polly bill per character of -// input text rather than per token. PromptTokens from usage is treated as the character count -// since TTS providers report their billable unit in that field. -// Output falls back to per-second duration when no audio token rate is configured. -func computeSpeechCost(pricing *configstoreTables.TableModelPricing, usage *schemas.BifrostLLMUsage, audioSeconds *int, audioTextInputChars int, tier serviceTier) float64 { - totalTokens := safeTotalTokens(usage) - - // Input: per-character rate takes precedence for TTS/audio models - inputCost := 0.0 - if audioTextInputChars > 0 { - if pricing.InputCostPerCharacter != nil { - inputCost = float64(audioTextInputChars) * *pricing.InputCostPerCharacter - } else { - inputCost = float64(audioTextInputChars) * tieredInputRate(pricing, totalTokens, tier) - } - } else if usage != nil && usage.PromptTokens > 0 { - inputCost = float64(usage.PromptTokens) * tieredInputRate(pricing, totalTokens, tier) - } - - // Output: audio tokens first, then per-second fallback - outputCost := computeAudioOutputCost(pricing, usage, audioSeconds, totalTokens, tier) - - return inputCost + outputCost -} - -// computeTranscriptionCost handles transcription (STT) requests. -// Input is audio, output is text (CompletionTokens). -// Input and output are calculated independently — tokens first, then per-second fallback. -func computeTranscriptionCost(pricing *configstoreTables.TableModelPricing, usage *schemas.BifrostLLMUsage, audioSeconds *int, audioTokenDetails *schemas.TranscriptionUsageInputTokenDetails, tier serviceTier) float64 { - totalTokens := safeTotalTokens(usage) - - // Input: audio tokens/details first, then per-second fallback - inputCost := computeAudioInputCost(pricing, usage, audioSeconds, audioTokenDetails, totalTokens, tier) - - // Output: text tokens - outputCost := 0.0 - if usage != nil && usage.CompletionTokens > 0 { - outputCost = float64(usage.CompletionTokens) * tieredOutputRate(pricing, totalTokens, tier) - } - - return inputCost + outputCost -} - -// computeAudioInputCost calculates input cost for audio: audio token details first, -// then generic input tokens, then per-second duration fallback. -func computeAudioInputCost(pricing *configstoreTables.TableModelPricing, usage *schemas.BifrostLLMUsage, audioSeconds *int, audioTokenDetails *schemas.TranscriptionUsageInputTokenDetails, totalTokens int, tier serviceTier) float64 { - // Audio token detail pricing (audio + text token breakdown) - if audioTokenDetails != nil && (audioTokenDetails.AudioTokens > 0 || audioTokenDetails.TextTokens > 0) { - return float64(audioTokenDetails.AudioTokens)*tieredAudioTokenInputRate(pricing, totalTokens, tier) + - float64(audioTokenDetails.TextTokens)*tieredInputRate(pricing, totalTokens, tier) - } - - // Generic input tokens - if usage != nil && usage.PromptTokens > 0 { - return float64(usage.PromptTokens) * tieredInputRate(pricing, totalTokens, tier) - } - - // Per-second duration fallback - if audioSeconds != nil && *audioSeconds > 0 { - if rate := tieredAudioInputPerSecondRate(pricing, totalTokens); rate > 0 { - return float64(*audioSeconds) * rate - } - } - - return 0 -} - -// computeAudioOutputCost calculates output cost for audio: audio tokens first, -// then generic output tokens, then per-second duration fallback. -func computeAudioOutputCost(pricing *configstoreTables.TableModelPricing, usage *schemas.BifrostLLMUsage, audioSeconds *int, totalTokens int, tier serviceTier) float64 { - // Audio-specific output tokens - if usage != nil && usage.CompletionTokens > 0 { - return float64(usage.CompletionTokens) * tieredAudioTokenOutputRate(pricing, totalTokens, tier) - } - - // Per-second duration fallback - if audioSeconds != nil && *audioSeconds > 0 { - if pricing.OutputCostPerSecond != nil { - return float64(*audioSeconds) * *pricing.OutputCostPerSecond - } - } - - return 0 -} - -// computeImageCost handles image generation requests. -// Input and output are calculated independently — each tries token-based pricing first, -// then per-pixel pricing, falling back to per-image count pricing. -// imageQuality must be one of "low", "medium", "high", "auto" to use quality-specific rates; other values use base rates. -func computeImageCost(pricing *configstoreTables.TableModelPricing, imageUsage *schemas.ImageUsage, imageSize string, imageQuality string, tier serviceTier) float64 { - if imageUsage == nil { - return 0 - } - - totalTokens := imageUsage.TotalTokens - pixels := parseImagePixels(imageSize) - inputCost := computeImageInputCost(pricing, imageUsage, totalTokens, pixels, tier) - outputCost := computeImageOutputCost(pricing, imageUsage, totalTokens, pixels, imageQuality, tier) - - return inputCost + outputCost -} - -// computeImageInputCost calculates input cost: tokens first, then per-pixel, then per-image count fallback. -func computeImageInputCost(pricing *configstoreTables.TableModelPricing, imageUsage *schemas.ImageUsage, totalTokens int, pixels int, tier serviceTier) float64 { - // Try token-based pricing first - var inputTextTokens, inputImageTokens int - if imageUsage.InputTokensDetails != nil { - inputImageTokens = imageUsage.InputTokensDetails.ImageTokens - inputTextTokens = imageUsage.InputTokensDetails.TextTokens - } else { - inputTextTokens = imageUsage.InputTokens - } - - if inputTextTokens > 0 || inputImageTokens > 0 { - return float64(inputTextTokens)*tieredInputRate(pricing, totalTokens, tier) + - float64(inputImageTokens)*tieredImageInputRate(pricing, totalTokens, tier) - } - - // Per-pixel pricing fallback - if pricing.InputCostPerPixel != nil && pixels > 0 && imageUsage.NumInputImages > 0 { - return float64(pixels*imageUsage.NumInputImages) * *pricing.InputCostPerPixel - } - - // Fall back to per-image count pricing - if pricing.InputCostPerImage != nil && imageUsage.NumInputImages > 0 { - return float64(imageUsage.NumInputImages) * *pricing.InputCostPerImage - } - - return 0 -} - -// computeImageOutputCost calculates output cost: tokens first, then per-pixel, then per-image count fallback. -// imageQuality: "low", "medium", "high", "auto" use quality-specific rates when available; other values use base/size-tier rates. -func computeImageOutputCost(pricing *configstoreTables.TableModelPricing, imageUsage *schemas.ImageUsage, totalTokens int, pixels int, imageQuality string, tier serviceTier) float64 { - // Try token-based pricing first - var outputTextTokens, outputImageTokens int - if imageUsage.OutputTokensDetails != nil { - outputImageTokens = imageUsage.OutputTokensDetails.ImageTokens - outputTextTokens = imageUsage.OutputTokensDetails.TextTokens - } else { - outputImageTokens = imageUsage.OutputTokens - } - - if outputTextTokens > 0 || outputImageTokens > 0 { - return float64(outputTextTokens)*tieredOutputRate(pricing, totalTokens, tier) + - float64(outputImageTokens)*tieredImageOutputRate(pricing, totalTokens, tier) - } - - // Per-pixel pricing fallback - if pricing.OutputCostPerPixel != nil && pixels > 0 { - numOutputImages := 1 - if imageUsage.OutputTokensDetails != nil && imageUsage.OutputTokensDetails.NImages > 0 { - numOutputImages = imageUsage.OutputTokensDetails.NImages - } - return float64(pixels*numOutputImages) * *pricing.OutputCostPerPixel - } - - // Fall back to per-image count pricing with size-tier selection - // TODO: handle premium image flag when it becomes available in imageUsage - numOutputImages := 1 - if imageUsage.OutputTokensDetails != nil && imageUsage.OutputTokensDetails.NImages > 0 { - numOutputImages = imageUsage.OutputTokensDetails.NImages - } - var perImageRate *float64 - q := imageQuality - if q == "" { - q = "auto" - } - switch q { - case "low": - if pricing.OutputCostPerImageLowQuality != nil { - perImageRate = pricing.OutputCostPerImageLowQuality - } - case "medium": - if pricing.OutputCostPerImageMediumQuality != nil { - perImageRate = pricing.OutputCostPerImageMediumQuality - } - case "high": - if pricing.OutputCostPerImageHighQuality != nil { - perImageRate = pricing.OutputCostPerImageHighQuality - } - case "auto": - if pricing.OutputCostPerImageAutoQuality != nil { - perImageRate = pricing.OutputCostPerImageAutoQuality - } - } - if perImageRate == nil { - const pixels512x512 = 512 * 512 - const pixels1024x1024 = 1024 * 1024 - const pixels2048x2048 = 2048 * 2048 - const pixels4096x4096 = 4096 * 4096 - switch { - case pixels >= pixels4096x4096 && pricing.OutputCostPerImageAbove4096x4096Pixels != nil: - perImageRate = pricing.OutputCostPerImageAbove4096x4096Pixels - case pixels >= pixels2048x2048 && pricing.OutputCostPerImageAbove2048x2048Pixels != nil: - perImageRate = pricing.OutputCostPerImageAbove2048x2048Pixels - case pixels >= pixels1024x1024 && pricing.OutputCostPerImageAbove1024x1024Pixels != nil: - perImageRate = pricing.OutputCostPerImageAbove1024x1024Pixels - case pixels >= pixels512x512 && pricing.OutputCostPerImageAbove512x512Pixels != nil: - perImageRate = pricing.OutputCostPerImageAbove512x512Pixels - default: - perImageRate = pricing.OutputCostPerImage - } - } - if perImageRate != nil { - return float64(numOutputImages) * *perImageRate - } - - return 0 -} - -// computeVideoCost handles video generation requests. -// Input and output are calculated independently — tokens first, then per-second fallback. -func computeVideoCost(pricing *configstoreTables.TableModelPricing, usage *schemas.BifrostLLMUsage, videoSeconds *int, tier serviceTier) float64 { - totalTokens := safeTotalTokens(usage) - - // Input: text prompt tokens first, then per-second fallback - inputCost := 0.0 - if usage != nil && usage.PromptTokens > 0 { - inputCost = float64(usage.PromptTokens) * tieredInputRate(pricing, totalTokens, tier) - } else if videoSeconds != nil && *videoSeconds > 0 { - if rate := tieredVideoInputPerSecondRate(pricing, totalTokens); rate > 0 { - inputCost = float64(*videoSeconds) * rate - } - } - - // Output: completion tokens first, then per-second fallback - outputCost := 0.0 - if usage != nil && usage.CompletionTokens > 0 { - outputCost = float64(usage.CompletionTokens) * tieredOutputRate(pricing, totalTokens, tier) - } else if videoSeconds != nil && *videoSeconds > 0 { - if pricing.OutputCostPerVideoPerSecond != nil { - outputCost = float64(*videoSeconds) * *pricing.OutputCostPerVideoPerSecond - } else if pricing.OutputCostPerSecond != nil { - outputCost = float64(*videoSeconds) * *pricing.OutputCostPerSecond - } - } - - return inputCost + outputCost -} - -// computeOCRCost handles OCR requests, billing per page processed. -// ocr_cost_per_page covers base processing; annotation_cost_per_page is added when set. -func computeOCRCost(pricing *configstoreTables.TableModelPricing, ocrProcessedPages *int, ocrIsAnnotated *bool) float64 { - if ocrProcessedPages == nil { - return 0 - } - pages := float64(*ocrProcessedPages) - cost := 0.0 - if pricing.OCRCostPerPage != nil { - cost += pages * *pricing.OCRCostPerPage - } - if ocrIsAnnotated != nil && *ocrIsAnnotated && pricing.AnnotationCostPerPage != nil { - cost += pages * *pricing.AnnotationCostPerPage - } - return cost -} - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -// tierFromString constructs a serviceTier from an OpenAI service_tier response value. -func tierFromString(s *schemas.BifrostServiceTier) serviceTier { - if s == nil { - return serviceTier{} - } - switch *s { - case schemas.BifrostServiceTierPriority: - return serviceTier{isPriority: true} - case schemas.BifrostServiceTierFlex: - return serviceTier{isFlex: true} - default: - return serviceTier{} - } -} - -// tieredInputRate returns the effective per-token input rate based on total token count. -// Flex applies a flat rate. Priority-specific tier rates are preferred where available. -func tieredInputRate(pricing *configstoreTables.TableModelPricing, totalTokens int, tier serviceTier) float64 { - if tier.isFlex && pricing.InputCostPerTokenFlex != nil { - return *pricing.InputCostPerTokenFlex - } - if totalTokens > TokenTierAbove272K { - if tier.isPriority && pricing.InputCostPerTokenAbove272kTokensPriority != nil { - return *pricing.InputCostPerTokenAbove272kTokensPriority - } - if pricing.InputCostPerTokenAbove272kTokens != nil { - return *pricing.InputCostPerTokenAbove272kTokens - } - } - if totalTokens > TokenTierAbove200K { - if tier.isPriority && pricing.InputCostPerTokenAbove200kTokensPriority != nil { - return *pricing.InputCostPerTokenAbove200kTokensPriority - } - if pricing.InputCostPerTokenAbove200kTokens != nil { - return *pricing.InputCostPerTokenAbove200kTokens - } - } - if totalTokens > TokenTierAbove128K && pricing.InputCostPerTokenAbove128kTokens != nil { - return *pricing.InputCostPerTokenAbove128kTokens - } - if tier.isPriority && pricing.InputCostPerTokenPriority != nil { - return *pricing.InputCostPerTokenPriority - } - if pricing.InputCostPerToken != nil { - return *pricing.InputCostPerToken - } - return 0 -} - -// tieredOutputRate returns the effective per-token output rate based on total token count. -// Flex applies a flat rate. Priority-specific tier rates are preferred where available. -func tieredOutputRate(pricing *configstoreTables.TableModelPricing, totalTokens int, tier serviceTier) float64 { - if tier.isFlex && pricing.OutputCostPerTokenFlex != nil { - return *pricing.OutputCostPerTokenFlex - } - if totalTokens > TokenTierAbove272K { - if tier.isPriority && pricing.OutputCostPerTokenAbove272kTokensPriority != nil { - return *pricing.OutputCostPerTokenAbove272kTokensPriority - } - if pricing.OutputCostPerTokenAbove272kTokens != nil { - return *pricing.OutputCostPerTokenAbove272kTokens - } - } - if totalTokens > TokenTierAbove200K { - if tier.isPriority && pricing.OutputCostPerTokenAbove200kTokensPriority != nil { - return *pricing.OutputCostPerTokenAbove200kTokensPriority - } - if pricing.OutputCostPerTokenAbove200kTokens != nil { - return *pricing.OutputCostPerTokenAbove200kTokens - } - } - if totalTokens > TokenTierAbove128K && pricing.OutputCostPerTokenAbove128kTokens != nil { - return *pricing.OutputCostPerTokenAbove128kTokens - } - - if tier.isPriority && pricing.OutputCostPerTokenPriority != nil { - return *pricing.OutputCostPerTokenPriority - } - - if pricing.OutputCostPerToken != nil { - return *pricing.OutputCostPerToken - } - - return 0 -} - -// tieredImageInputRate returns the effective rate for image tokens on the input side. -// Falls back to the general tieredInputRate when no image-specific rate is configured. -func tieredImageInputRate(pricing *configstoreTables.TableModelPricing, totalTokens int, tier serviceTier) float64 { - if totalTokens > TokenTierAbove128K && pricing.InputCostPerImageAbove128kTokens != nil { - return *pricing.InputCostPerImageAbove128kTokens - } - if pricing.InputCostPerImageToken != nil { - return *pricing.InputCostPerImageToken - } - return tieredInputRate(pricing, totalTokens, tier) -} - -// tieredImageOutputRate returns the effective rate for image tokens on the output side. -// Falls back to the general tieredOutputRate when no image-specific rate is configured. -func tieredImageOutputRate(pricing *configstoreTables.TableModelPricing, totalTokens int, tier serviceTier) float64 { - if pricing.OutputCostPerImageToken != nil { - return *pricing.OutputCostPerImageToken - } - return tieredOutputRate(pricing, totalTokens, tier) -} - -// tieredAudioInputPerSecondRate returns the effective per-second rate for audio input. -func tieredAudioInputPerSecondRate(pricing *configstoreTables.TableModelPricing, totalTokens int) float64 { - if totalTokens > TokenTierAbove128K && pricing.InputCostPerAudioPerSecondAbove128kTokens != nil { - return *pricing.InputCostPerAudioPerSecondAbove128kTokens - } - if pricing.InputCostPerAudioPerSecond != nil { - return *pricing.InputCostPerAudioPerSecond - } - if pricing.InputCostPerSecond != nil { - return *pricing.InputCostPerSecond - } - return 0 -} - -// tieredVideoInputPerSecondRate returns the effective per-second rate for video input. -func tieredVideoInputPerSecondRate(pricing *configstoreTables.TableModelPricing, totalTokens int) float64 { - if totalTokens > TokenTierAbove128K && pricing.InputCostPerVideoPerSecondAbove128kTokens != nil { - return *pricing.InputCostPerVideoPerSecondAbove128kTokens - } - if pricing.InputCostPerVideoPerSecond != nil { - return *pricing.InputCostPerVideoPerSecond - } - return 0 -} - -// tieredAudioTokenInputRate returns the effective per-token rate for audio input tokens. -// Falls back to the general tieredInputRate when no audio-specific rate is configured. -func tieredAudioTokenInputRate(pricing *configstoreTables.TableModelPricing, totalTokens int, tier serviceTier) float64 { - if pricing.InputCostPerAudioToken != nil { - return *pricing.InputCostPerAudioToken - } - return tieredInputRate(pricing, totalTokens, tier) -} - -// tieredAudioTokenOutputRate returns the effective per-token rate for audio output tokens. -// Falls back to the general tieredOutputRate when no audio-specific rate is configured. -func tieredAudioTokenOutputRate(pricing *configstoreTables.TableModelPricing, totalTokens int, tier serviceTier) float64 { - if pricing.OutputCostPerAudioToken != nil { - return *pricing.OutputCostPerAudioToken - } - return tieredOutputRate(pricing, totalTokens, tier) -} - -func tieredCacheReadInputTokenRate(pricing *configstoreTables.TableModelPricing, totalTokens int, tier serviceTier) float64 { - if tier.isFlex && pricing.CacheReadInputTokenCostFlex != nil { - return *pricing.CacheReadInputTokenCostFlex - } - if totalTokens > TokenTierAbove272K { - if tier.isPriority && pricing.CacheReadInputTokenCostAbove272kTokensPriority != nil { - return *pricing.CacheReadInputTokenCostAbove272kTokensPriority - } - if pricing.CacheReadInputTokenCostAbove272kTokens != nil { - return *pricing.CacheReadInputTokenCostAbove272kTokens - } - } - if totalTokens > TokenTierAbove200K { - if tier.isPriority && pricing.CacheReadInputTokenCostAbove200kTokensPriority != nil { - return *pricing.CacheReadInputTokenCostAbove200kTokensPriority - } - if pricing.CacheReadInputTokenCostAbove200kTokens != nil { - return *pricing.CacheReadInputTokenCostAbove200kTokens - } - } - if tier.isPriority && pricing.CacheReadInputTokenCostPriority != nil { - return *pricing.CacheReadInputTokenCostPriority - } - if pricing.CacheReadInputTokenCost != nil { - return *pricing.CacheReadInputTokenCost - } - return tieredInputRate(pricing, totalTokens, tier) -} - -// Note: flex tier is not checked here because cache creation is not a concept in -// OpenAI's pricing model (the only provider that uses flex tier). Only cache read -// has a flex-specific rate. -func tieredCacheCreationInputTokenRate(pricing *configstoreTables.TableModelPricing, totalTokens int, tier serviceTier) float64 { - if totalTokens > TokenTierAbove200K && pricing.CacheCreationInputTokenCostAbove200kTokens != nil { - return *pricing.CacheCreationInputTokenCostAbove200kTokens - } - if pricing.CacheCreationInputTokenCost != nil { - return *pricing.CacheCreationInputTokenCost - } - return tieredInputRate(pricing, totalTokens, tier) -} - -func tieredCacheCreationInputAbove1hrTokenRate(pricing *configstoreTables.TableModelPricing, totalTokens int, tier serviceTier) float64 { - if totalTokens > TokenTierAbove200K && pricing.CacheCreationInputTokenCostAbove1hrAbove200kTokens != nil { - return *pricing.CacheCreationInputTokenCostAbove1hrAbove200kTokens - } - if pricing.CacheCreationInputTokenCostAbove1hr != nil { - return *pricing.CacheCreationInputTokenCostAbove1hr - } - return tieredCacheCreationInputTokenRate(pricing, totalTokens, tier) -} - -func safeTotalTokens(usage *schemas.BifrostLLMUsage) int { - if usage == nil { - return 0 - } - return usage.TotalTokens -} - -// parseImagePixels parses a size string like "1024x1024" into total pixel count. -// Returns 0 if the size string is empty or malformed. -func parseImagePixels(size string) int { - if size == "" { - return 0 - } - parts := strings.SplitN(size, "x", 2) - if len(parts) != 2 { - return 0 - } - w, err := strconv.Atoi(parts[0]) - if err != nil || w <= 0 { - return 0 - } - h, err := strconv.Atoi(parts[1]) - if err != nil || h <= 0 { - return 0 - } - return w * h -} - -// populateOutputImageCount sets the output image count on ImageUsage from len(Data) -// when OutputTokensDetails.NImages is not already populated. -func populateOutputImageCount(imageUsage *schemas.ImageUsage, dataLen int) { - if imageUsage == nil || dataLen == 0 { - return - } - if imageUsage.OutputTokensDetails == nil { - imageUsage.OutputTokensDetails = &schemas.ImageTokenDetails{} - } - if imageUsage.OutputTokensDetails.NImages == 0 { - imageUsage.OutputTokensDetails.NImages = dataLen - } + return mc.datasheet.CalculateCost(result, (*datasheet.LookupScopes)(scopes)) } -// --------------------------------------------------------------------------- -// Pricing resolution -// --------------------------------------------------------------------------- - -// resolvePricing resolves the pricing entry for a model, trying deployment as fallback. -func (mc *ModelCatalog) resolvePricing(provider, originalModelRequested, resolvedModelUsed string, requestType schemas.RequestType, scopes PricingLookupScopes) *configstoreTables.TableModelPricing { - if resolvedModelUsed == "" { - resolvedModelUsed = originalModelRequested - } - mc.logger.Debug("looking up pricing for resolved model %s and provider %s of request type %s", resolvedModelUsed, provider, normalizeRequestType(requestType)) - - if scopes.Provider == "" { - scopes.Provider = provider - } - - base, exists := mc.getBasePricing(resolvedModelUsed, provider, requestType) - if exists && base != nil { - result, _ := mc.applyPricingOverrides(resolvedModelUsed, requestType, *base, scopes) - return &result - } - - mc.logger.Debug("pricing not found for resolved model %s, trying alias %s", resolvedModelUsed, originalModelRequested) - base, exists = mc.getBasePricing(originalModelRequested, provider, requestType) - if exists && base != nil { - // Apply overrides using the resolved model name, not the alias - result, _ := mc.applyPricingOverrides(resolvedModelUsed, requestType, *base, scopes) - return &result - } - - // No base catalog entry found; still try overrides in case the user defined - // override-only pricing for a model not in the built-in catalog. - mc.logger.Debug("pricing not found for resolved model %s and provider %s, trying override-only pricing", resolvedModelUsed, provider) - result, applied := mc.applyPricingOverrides(resolvedModelUsed, requestType, configstoreTables.TableModelPricing{}, scopes) - if applied { - return &result - } - mc.logger.Debug("no pricing found for resolved model %s and provider %s, skipping cost calculation", resolvedModelUsed, provider) - return nil -} - -// getBasePricing looks up catalog pricing for the given model, provider, and request type. -// It applies a provider-specific fallback chain when an exact match is not found: -// -// - Gemini: retries under the "vertex" provider, then falls back to chat mode for Responses requests. -// - Vertex: strips the "provider/model" prefix and retries, then falls back to chat mode for Responses requests. -// - Bedrock: prepends the "anthropic." namespace for Claude models, then falls back to chat mode for Responses requests. -// - All providers: for Responses/ResponsesStream requests, retries the lookup in chat mode. -// - All providers: for ImageEdit/ImageVariation requests, retries the lookup in image-generation mode. -// -// The method acquires a read lock for the duration of the lookup. -// -// Input: model — exact model name to look up. -// -// provider — provider identifier (e.g. "openai", "anthropic"). -// requestType — the request type used to derive the pricing mode. -// -// Output: TableModelPricing — the matched pricing row (zero value when not found). -// -// bool — true when a pricing entry was found, false otherwise. -func (mc *ModelCatalog) getBasePricing(model, provider string, requestType schemas.RequestType) (*configstoreTables.TableModelPricing, bool) { - mc.mu.RLock() - defer mc.mu.RUnlock() - - mode := normalizeRequestType(requestType) - - pricing, ok := mc.pricingData[makeKey(model, provider, mode)] - if ok { - return &pricing, true - } - - // Lookup in vertex if gemini not found - if provider == string(schemas.Gemini) { - mc.logger.Debug("primary lookup failed, trying vertex provider for the same model") - pricing, ok = mc.pricingData[makeKey(model, "vertex", mode)] - if ok { - return &pricing, true - } - - // Lookup in chat if responses not found - if requestType == schemas.ResponsesRequest || requestType == schemas.ResponsesStreamRequest || requestType == schemas.WebSocketResponsesRequest || requestType == schemas.RealtimeRequest || requestType == schemas.CompactionRequest { - mc.logger.Debug("secondary lookup failed, trying vertex provider for the same model in chat completion") - pricing, ok = mc.pricingData[makeKey(model, "vertex", normalizeRequestType(schemas.ChatCompletionRequest))] - if ok { - return &pricing, true - } - } - } - - if provider == string(schemas.Vertex) { - // Vertex models can be of the form "provider/model", so try to lookup the model without the provider prefix and keep the original provider - if strings.Contains(model, "/") { - modelWithoutProvider := strings.SplitN(model, "/", 2)[1] - mc.logger.Debug("primary lookup failed, trying vertex provider for the same model with provider/model format %s", modelWithoutProvider) - pricing, ok = mc.pricingData[makeKey(modelWithoutProvider, "vertex", mode)] - if ok { - return &pricing, true - } - - // Lookup in chat if responses not found - if requestType == schemas.ResponsesRequest || requestType == schemas.ResponsesStreamRequest || requestType == schemas.WebSocketResponsesRequest || requestType == schemas.RealtimeRequest || requestType == schemas.CompactionRequest { - mc.logger.Debug("secondary lookup failed, trying vertex provider for the same model in chat completion") - pricing, ok = mc.pricingData[makeKey(modelWithoutProvider, "vertex", normalizeRequestType(schemas.ChatCompletionRequest))] - if ok { - return &pricing, true - } - } - } - } - - if provider == string(schemas.Bedrock) { - // If model is claude without "anthropic." prefix, try with "anthropic." prefix - if !strings.Contains(model, "anthropic.") && schemas.IsAnthropicModel(model) { - mc.logger.Debug("primary lookup failed, trying with anthropic. prefix for the same model") - pricing, ok = mc.pricingData[makeKey("anthropic."+model, provider, mode)] - if ok { - return &pricing, true - } - - // Lookup in chat if responses not found - if requestType == schemas.ResponsesRequest || requestType == schemas.ResponsesStreamRequest || requestType == schemas.WebSocketResponsesRequest || requestType == schemas.RealtimeRequest || requestType == schemas.CompactionRequest { - mc.logger.Debug("secondary lookup failed, trying chat provider for the same model in chat completion") - pricing, ok = mc.pricingData[makeKey("anthropic."+model, provider, normalizeRequestType(schemas.ChatCompletionRequest))] - if ok { - return &pricing, true - } - } - } - } - - // Lookup in chat if responses/compaction not found - if requestType == schemas.ResponsesRequest || requestType == schemas.ResponsesStreamRequest || requestType == schemas.WebSocketResponsesRequest || requestType == schemas.RealtimeRequest || requestType == schemas.CompactionRequest { - mc.logger.Debug("primary lookup failed, trying chat provider for the same model in chat completion") - pricing, ok = mc.pricingData[makeKey(model, provider, normalizeRequestType(schemas.ChatCompletionRequest))] - if ok { - return &pricing, true - } - } - - // Lookup in image generation if image edit not found - if requestType == schemas.ImageEditRequest || - requestType == schemas.ImageEditStreamRequest || - requestType == schemas.ImageVariationRequest { - mc.logger.Debug("primary lookup failed, trying image generation provider for the same model") - pricing, ok = mc.pricingData[makeKey(model, provider, normalizeRequestType(schemas.ImageGenerationRequest))] - if ok { - return &pricing, true - } - } - - // Lookup fallback chain for container_create: - // 1. Try chat mode for the same model (e.g. "container-1g" in chat mode) - // 2. Try the base "container" model in chat mode (default rate when no memory-specific entry exists) - if requestType == schemas.ContainerCreateRequest { - mc.logger.Debug("primary lookup failed, trying chat mode for container create pricing") - pricing, ok = mc.pricingData[makeKey(model, provider, normalizeRequestType(schemas.ChatCompletionRequest))] - if ok { - return &pricing, true - } - if model != "container" { - mc.logger.Debug("memory-specific container pricing not found, falling back to base container entry") - pricing, ok = mc.pricingData[makeKey("container", provider, normalizeRequestType(schemas.ChatCompletionRequest))] - if ok { - return &pricing, true - } - } - } - - return nil, false -} - -// UpsertModelPricingAttributes writes the additional_attributes column for -// every pricing row that matches (model, provider), then reloads the pricing -// cache so the new values are immediately visible to list-models. Returns -// the number of rows updated (0 = no such pricing row, which callers must -// surface as a validation error). An empty/nil attrs map clears the column. +// UpsertModelPricingAttributes writes additional_attributes for every row +// matching (model, provider) and reloads the pricing cache. func (mc *ModelCatalog) UpsertModelPricingAttributes(ctx context.Context, model string, provider schemas.ModelProvider, attrs map[string]string) (int64, error) { - if mc.configStore == nil { - return 0, fmt.Errorf("model catalog requires a config store") - } - rows, err := mc.configStore.UpsertModelPricingAttributes(ctx, model, string(provider), attrs) - if err != nil { - return 0, err - } - if rows == 0 { - return 0, nil - } - if err := mc.loadPricingFromDatabase(ctx); err != nil { - return rows, fmt.Errorf("failed to reload pricing cache after attribute write: %w", err) - } - return rows, nil + return mc.datasheet.UpsertModelPricingAttributes(ctx, model, provider, attrs) } -// --------------------------------------------------------------------------- -// Passthrough pricing helpers -// --------------------------------------------------------------------------- - -// detectPassthroughRequestType maps a provider + stripped path to a RequestType. -func detectPassthroughRequestType(provider schemas.ModelProvider, path string) schemas.RequestType { - if idx := strings.IndexByte(path, '?'); idx >= 0 { - path = path[:idx] - } - path = strings.TrimRight(path, "/") - switch provider { - case schemas.OpenAI, schemas.Azure: - switch { - case strings.HasSuffix(path, "/chat/completions"): - return schemas.ChatCompletionRequest - case strings.HasSuffix(path, "/completions"): - return schemas.TextCompletionRequest - case strings.HasSuffix(path, "/embeddings"): - return schemas.EmbeddingRequest - case strings.HasSuffix(path, "/responses/compact"): - return schemas.CompactionRequest - case strings.HasSuffix(path, "/responses"): - return schemas.ResponsesRequest - case strings.HasSuffix(path, "/images/generations"): - return schemas.ImageGenerationRequest - case strings.HasSuffix(path, "/images/edits"): - return schemas.ImageEditRequest - case strings.HasSuffix(path, "/images/variations"): - return schemas.ImageVariationRequest - case strings.HasSuffix(path, "/audio/speech"): - return schemas.SpeechRequest - case strings.HasSuffix(path, "/audio/transcriptions"), - strings.HasSuffix(path, "/audio/translations"): - return schemas.TranscriptionRequest - case strings.HasSuffix(path, "/containers"): - return schemas.ContainerCreateRequest - case strings.Contains(path, "/video"): - return schemas.VideoGenerationRequest - default: - return schemas.ChatCompletionRequest - } - case schemas.Gemini, schemas.Vertex: - // Interactions API paths carry no colon action suffix. - if strings.Contains(path, "/interactions") { - return schemas.ResponsesRequest - } - colonIdx := strings.LastIndexByte(path, ':') - if colonIdx < 0 { - return schemas.ChatCompletionRequest - } - switch path[colonIdx+1:] { - case "generateContent", "streamGenerateContent": - return schemas.ResponsesRequest - case "embedContent", "batchEmbedContents": - return schemas.EmbeddingRequest - case "generateImages": - return schemas.ImageGenerationRequest - case "predict": - return schemas.EmbeddingRequest - case "predictLongRunning": - return schemas.VideoGenerationRequest - default: - return schemas.ChatCompletionRequest - } - case schemas.Anthropic: - switch { - case strings.HasSuffix(path, "/messages"): - return schemas.ResponsesRequest - case strings.HasSuffix(path, "/complete"): - return schemas.TextCompletionRequest - default: - return schemas.ResponsesRequest - } - default: - return schemas.ChatCompletionRequest - } +func (mc *ModelCatalog) SetPricingOverrides(rows []configstoreTables.TablePricingOverride) error { + return mc.datasheet.SetOverrides(rows) } -// inferPassthroughRequestType determines the request type from usage fields (primary) -// and falls back to path detection for text/embedding/responses where LLMUsage is ambiguous. -func inferPassthroughRequestType(provider schemas.ModelProvider, path string, su *schemas.BifrostPassthroughUsage) schemas.RequestType { - if su != nil { - if su.ContainerIdentifier != "" { - return schemas.ContainerCreateRequest - } - if su.ImageUsage != nil { - return schemas.ImageGenerationRequest - } - if su.AudioInputChars > 0 { - return schemas.SpeechRequest - } - if su.AudioTokenDetails != nil || su.AudioSeconds != nil { - return schemas.TranscriptionRequest - } - if su.VideoSeconds != nil { - return schemas.VideoGenerationRequest - } - } - return detectPassthroughRequestType(provider, path) +func (mc *ModelCatalog) UpsertPricingOverrides(rows ...*configstoreTables.TablePricingOverride) error { + return mc.datasheet.UpsertOverrides(rows...) } -// passthroughUsageToCostInput converts BifrostPassthroughUsage into costInput. -func passthroughUsageToCostInput(su *schemas.BifrostPassthroughUsage) costInput { - var input costInput - if su.LLMUsage != nil { - input.usage = su.LLMUsage - } - if su.ServiceTier != nil { - input.tier = tierFromString(su.ServiceTier) - } - if su.ImageUsage != nil { - input.imageUsage = su.ImageUsage - input.imageSize = su.ImageSize - input.imageQuality = su.ImageQuality - } - if su.AudioInputChars > 0 { - input.audioTextInputChars = su.AudioInputChars - } - if su.AudioSeconds != nil { - input.audioSeconds = su.AudioSeconds - } - if su.AudioTokenDetails != nil { - input.audioTokenDetails = su.AudioTokenDetails - } - if su.VideoSeconds != nil { - input.videoSeconds = su.VideoSeconds - } - if su.ContainerIdentifier != "" { - input.containerIdentifierString = su.ContainerIdentifier - } - return input +func (mc *ModelCatalog) DeletePricingOverride(id string) { + mc.datasheet.DeleteOverride(id) } diff --git a/framework/modelcatalog/refine_test.go b/framework/modelcatalog/refine_test.go deleted file mode 100644 index 297a055342..0000000000 --- a/framework/modelcatalog/refine_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package modelcatalog - -import ( - "testing" - - "github.com/maximhq/bifrost/core/schemas" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// TestRefineModelForProvider_ReplicateRefinesOpenAIModel verifies that -// Replicate can recover nested provider slugs for provider-pinned OpenAI-family models. -func TestRefineModelForProvider_ReplicateRefinesOpenAIModel(t *testing.T) { - mc := newTestCatalog(map[schemas.ModelProvider][]string{ - schemas.Replicate: {"openai/gpt-5-nano"}, - }, map[string]string{ - "openai/gpt-5-nano": "gpt-5-nano", - }) - - refined, err := mc.RefineModelForProvider(schemas.Replicate, "gpt-5-nano") - require.NoError(t, err) - assert.Equal(t, "openai/gpt-5-nano", refined) -} - -// TestRefineModelForProvider_ReplicatePreservesOwnerSlashModel verifies that -// standard Replicate owner/model slugs are not mistaken for nested provider slugs. -func TestRefineModelForProvider_ReplicatePreservesOwnerSlashModel(t *testing.T) { - mc := newTestCatalog(map[schemas.ModelProvider][]string{ - schemas.Replicate: {"meta/meta-llama-3-8b"}, - }, nil) - - refined, err := mc.RefineModelForProvider(schemas.Replicate, "meta/meta-llama-3-8b") - require.NoError(t, err) - assert.Equal(t, "meta/meta-llama-3-8b", refined) -} - -// TestRefineModelForProvider_ReplicateReturnsAmbiguousMatchError verifies that -// refinement fails fast when multiple nested provider slugs match the same base model. -func TestRefineModelForProvider_ReplicateReturnsAmbiguousMatchError(t *testing.T) { - mc := newTestCatalog(map[schemas.ModelProvider][]string{ - schemas.Replicate: { - "openai/gpt-5-nano", - "xai/gpt-5-nano", - }, - }, nil) - - refined, err := mc.RefineModelForProvider(schemas.Replicate, "gpt-5-nano") - require.Error(t, err) - assert.Empty(t, refined) - assert.Contains(t, err.Error(), "multiple compatible models found") -} diff --git a/framework/modelcatalog/sync.go b/framework/modelcatalog/sync.go deleted file mode 100644 index bb7c74e1e8..0000000000 --- a/framework/modelcatalog/sync.go +++ /dev/null @@ -1,545 +0,0 @@ -package modelcatalog - -import ( - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "os" - "slices" - "sync" - "time" - - bifrost "github.com/maximhq/bifrost/core" - providerUtils "github.com/maximhq/bifrost/core/providers/utils" - "github.com/maximhq/bifrost/core/schemas" - configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" - "github.com/tidwall/gjson" - "gorm.io/gorm" -) - -const ( - urlFetchMaxRetries = 3 // retries after the first attempt (4 attempts total) - urlFetchMaxBackoff = 10 * time.Second // cap for exponential backoff (steps start at 1s) -) - -// syncPricing syncs pricing data from URL to database and updates cache -func (mc *ModelCatalog) syncPricing(ctx context.Context) error { - if mc.shouldSyncGate != nil { - if !mc.shouldSyncGate(ctx) { - return nil - } - } - // Load pricing data from URL - pricingData, err := WithRetries(ctx, urlFetchMaxRetries, urlFetchMaxBackoff, func() (map[string]PricingEntry, error) { - return mc.loadPricingFromURL(ctx) - }) - if err != nil { - // Check if we have existing data in database - pricingRecords, pricingErr := mc.configStore.GetModelPrices(ctx) - if pricingErr != nil { - return fmt.Errorf("failed to get pricing records: %w", pricingErr) - } - if len(pricingRecords) > 0 { - mc.logger.Warn("failed to fetch pricing from URL, falling back to existing database records: %v", err) - return nil - } else { - return fmt.Errorf("failed to load pricing data from URL and no existing data in database: %w", err) - } - } - - // Update database in transaction - err = mc.configStore.ExecuteTransaction(ctx, func(tx *gorm.DB) error { - // Deduplicate and insert new pricing data - seen := make(map[string]bool) - for modelKey, entry := range pricingData { - pricing := convertPricingDataToTableModelPricing(modelKey, entry) - // Create composite key for deduplication - key := makeKey(pricing.Model, pricing.Provider, pricing.Mode) - // Skip if already seen - if exists, ok := seen[key]; ok && exists { - continue - } - // Mark as seen - seen[key] = true - if err := mc.configStore.UpsertModelPrices(ctx, &pricing, tx); err != nil { - return fmt.Errorf("failed to create pricing record for model %s: %w", pricing.Model, err) - } - } - - // Clear seen map - seen = nil - - return nil - }) - if err != nil { - return fmt.Errorf("failed to sync pricing data to database: %w", err) - } - - // Reload cache from database - if err := mc.loadPricingFromDatabase(ctx); err != nil { - return fmt.Errorf("failed to reload pricing cache: %w", err) - } - - // Populate model params cache from pricing datasheet max_output_tokens - mc.populateModelParamsFromPricing(pricingData) - - mc.logger.Debug("successfully synced %d pricing records", len(pricingData)) - return nil -} - -// populateModelParamsFromPricing extracts max_output_tokens from pricing entries -// and populates the model params cache so that providers can look up max output -// tokens without a separate model-parameters sync. -func (mc *ModelCatalog) populateModelParamsFromPricing(pricingData map[string]PricingEntry) { - modelParamsEntries := make(map[string]providerUtils.ModelParams) - for modelKey, entry := range pricingData { - if entry.MaxOutputTokens != nil { - modelName := extractModelName(modelKey) - params := providerUtils.ModelParams{ - MaxOutputTokens: entry.MaxOutputTokens, - } - modelParamsEntries[modelName] = params - } - } - if len(modelParamsEntries) > 0 { - providerUtils.BulkSetModelParams(modelParamsEntries) - mc.logger.Debug("populated %d model params entries from pricing datasheet", len(modelParamsEntries)) - } -} - -// loadPricingFromURL loads pricing data from the configured URL (supports file:// and http(s)://) -func (mc *ModelCatalog) loadPricingFromURL(ctx context.Context) (map[string]PricingEntry, error) { - rawURL := mc.getPricingURL() - - parsed, err := url.Parse(rawURL) - if err != nil { - return nil, fmt.Errorf("failed to parse pricing URL: %w", err) - } - - var data []byte - - if parsed.Scheme == "file" { - data, err = os.ReadFile(parsed.Path) - if err != nil { - return nil, fmt.Errorf("failed to read pricing file: %w", err) - } - } else { - if err := bifrost.ValidateExternalURL(rawURL, true); err != nil { - return nil, fmt.Errorf("pricing URL validation failed: %w", err) - } - client := &http.Client{Timeout: DefaultPricingTimeout} - req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) - if err != nil { - return nil, fmt.Errorf("failed to create HTTP request: %w", err) - } - resp, err := client.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to download pricing data: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("failed to download pricing data: HTTP %d", resp.StatusCode) - } - - data, err = io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read pricing data response: %w", err) - } - } - - var pricingData map[string]PricingEntry - if err := json.Unmarshal(data, &pricingData); err != nil { - return nil, fmt.Errorf("failed to unmarshal pricing data: %w", err) - } - - mc.logger.Debug("successfully loaded and parsed %d pricing records", len(pricingData)) - return pricingData, nil -} - -// loadPricingIntoMemoryFromURL loads pricing data from URL into memory cache (when config store is not available) -func (mc *ModelCatalog) loadPricingIntoMemoryFromURL(ctx context.Context) error { - pricingData, err := WithRetries(ctx, urlFetchMaxRetries, urlFetchMaxBackoff, func() (map[string]PricingEntry, error) { - return mc.loadPricingFromURL(ctx) - }) - if err != nil { - return fmt.Errorf("failed to load pricing data from URL: %w", err) - } - - mc.mu.Lock() - defer mc.mu.Unlock() - - // Clear and rebuild the pricing map - mc.pricingData = make(map[string]configstoreTables.TableModelPricing, len(pricingData)) - for modelKey, entry := range pricingData { - pricing := convertPricingDataToTableModelPricing(modelKey, entry) - key := makeKey(pricing.Model, pricing.Provider, pricing.Mode) - mc.pricingData[key] = pricing - } - - // Populate model params cache from pricing datasheet max_output_tokens - mc.populateModelParamsFromPricing(pricingData) - - return nil -} - -// ReloadPricing re-reads the pricing table into the in-memory cache. The -// management API uses this after a batched write so the new attributes are -// observable immediately. The existing 24-hour sync owns refreshing pricing -// fields from the upstream datasheet; this method just refreshes the cache. -func (mc *ModelCatalog) ReloadPricing(ctx context.Context) error { - return mc.loadPricingFromDatabase(ctx) -} - -// loadPricingFromDatabase loads pricing data from database into memory cache -func (mc *ModelCatalog) loadPricingFromDatabase(ctx context.Context) error { - if mc.configStore == nil { - return nil - } - - pricingRecords, err := mc.configStore.GetModelPrices(ctx) - if err != nil { - return fmt.Errorf("failed to load pricing from database: %w", err) - } - - mc.mu.Lock() - defer mc.mu.Unlock() - - // Clear and rebuild the pricing map - mc.pricingData = make(map[string]configstoreTables.TableModelPricing, len(pricingRecords)) - for _, pricing := range pricingRecords { - key := makeKey(pricing.Model, pricing.Provider, pricing.Mode) - mc.pricingData[key] = pricing - } - - mc.logger.Debug("loaded %d pricing records from database into memory", len(mc.pricingData)) - return nil -} - -// loadModelParametersFromDatabase bulk-loads model parameters from the DB into the provider -// utils cache (startup / ReloadFromDB). The SetCacheMissHandler path still loads one row at -// a time on cache miss; both use the same table JSON shape. -// Returns the number of rows loaded so callers can decide whether to background-sync from URL. -func (mc *ModelCatalog) loadModelParametersFromDatabase(ctx context.Context) (int, error) { - if mc.configStore == nil { - return 0, nil - } - - rows, err := mc.configStore.GetModelParameters(ctx) - if err != nil { - return 0, fmt.Errorf("failed to load model parameters from database: %w", err) - } - if len(rows) == 0 { - mc.logger.Debug("no model parameters rows in database") - return 0, nil - } - - paramsData := make(map[string]json.RawMessage, len(rows)) - for _, row := range rows { - paramsData[row.Model] = json.RawMessage(row.Data) - } - mc.applyModelParameters(paramsData) - mc.logger.Debug("loaded %d model parameters records from database into cache", len(rows)) - return len(rows), nil -} - -// startSyncWorker starts the background sync worker -func (mc *ModelCatalog) startSyncWorker(ctx context.Context) { - // IMPORTANT: scheduling model - // - // The sync worker wakes on a fixed ticker (syncWorkerTickerPeriod). On each - // wake it checks: - // - // time.Since(lastSyncTimestamp) >= pricingSyncInterval - // - // pricingSyncInterval defines the minimum elapsed time between syncs. The - // ticker period is the check granularity and must stay well below the - // minimum supported pricingSyncInterval, otherwise ticker drift (the few - // seconds a sync takes to complete) pushes the next check just under the - // threshold and the effective cadence doubles. - mc.syncTicker = time.NewTicker(syncWorkerTickerPeriod) - mc.wg.Add(1) - go mc.syncWorker(ctx) -} - -// withDistributedLock acquires a named distributed lock and executes fn under it. -// Pass retries=0 to block until acquired (Lock); pass retries>0 to use LockWithRetry. -func (mc *ModelCatalog) withDistributedLock(ctx context.Context, key string, retries int, fn func() error) error { - lock, err := mc.distributedLockManager.NewLock(key) - if err != nil { - return fmt.Errorf("failed to create lock %q: %w", key, err) - } - if retries > 0 { - if err := lock.LockWithRetry(ctx, retries); err != nil { - return fmt.Errorf("failed to acquire lock %q: %w", key, err) - } - } else { - if err := lock.Lock(ctx); err != nil { - return fmt.Errorf("failed to acquire lock %q: %w", key, err) - } - } - // Use a fresh context for unlock so that a cancelled or timed-out work context - // does not prevent the lock row from being deleted. If we reused ctx and it was - // already cancelled when the defer fires, ReleaseLock's DB call would fail - // silently and the lock would stay in the database until TTL expiry (30s), - // blocking every other node from acquiring it during that window. - defer func() { - if err := lock.Unlock(context.Background()); err != nil { - mc.logger.Warn("failed to release distributed lock %q: %v", key, err) - } - }() - return fn() -} - -// syncTick performs a single sync tick with proper lock management -// if the last sync was more than the sync interval ago, sync pricing and model parameters in parallel -func (mc *ModelCatalog) syncTick(ctx context.Context) { - mc.syncMu.RLock() - lastSync := mc.lastSyncedAt - interval := mc.syncInterval - mc.syncMu.RUnlock() - - if time.Since(lastSync) >= interval { - mc.logger.Debug("starting model catalog background sync") - if err := mc.withDistributedLock(ctx, "model_catalog_pricing_sync", 10, func() error { - // Sync pricing and model parameters in parallel - var wg sync.WaitGroup - var pricingErr, paramsErr error - wg.Add(2) - go func() { - defer wg.Done() - if err := mc.syncPricing(ctx); err != nil { - mc.logger.Error("background pricing sync failed: %v", err) - pricingErr = err - } - }() - go func() { - defer wg.Done() - if err := mc.syncModelParameters(ctx); err != nil { - mc.logger.Error("background model parameters sync failed: %v", err) - paramsErr = err - } - }() - wg.Wait() - - if pricingErr == nil && paramsErr == nil { - if mc.afterSyncHook != nil { - mc.afterSyncHook(ctx) - } - mc.syncMu.Lock() - mc.lastSyncedAt = time.Now() - mc.syncMu.Unlock() - } - if pricingErr != nil { - return pricingErr - } - return paramsErr - }); err != nil { - mc.logger.Error("failed to run model catalog sync: %v", err) - } - mc.logger.Debug("model catalog background sync completed") - } -} - -// syncWorker runs the background sync check -func (mc *ModelCatalog) syncWorker(ctx context.Context) { - defer mc.wg.Done() - defer mc.syncTicker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-mc.syncTicker.C: - mc.syncTick(ctx) - case <-mc.done: - return - } - } -} - -// --- Model Parameters sync --- - -func (mc *ModelCatalog) applyModelParameters(paramsData map[string]json.RawMessage) { - modelParamsEntries := make(map[string]providerUtils.ModelParams, len(paramsData)) - newResponseTypes := make(map[string][]string, len(paramsData)) - newParamsIndex := make(map[string][]string, len(paramsData)) - - for model, rawData := range paramsData { - var parsed modelParametersParseResult - if err := json.Unmarshal(rawData, &parsed); err != nil { - mc.logger.Warn("model-parameters-sync: skipping malformed parameters for model %s: %v", model, err) - continue - } - - outputs := make([]string, 0, len(parsed.SupportedEndpoints)) - for _, endpoint := range parsed.SupportedEndpoints { - if normalized := normalizeEndpointToOutputType(endpoint); normalized != "" && !slices.Contains(outputs, normalized) { - outputs = append(outputs, normalized) - } - } - - if parsed.Mode != nil { - if normalized := normalizeModeToOutputType(*parsed.Mode); normalized != "" && !slices.Contains(outputs, normalized) { - outputs = append(outputs, normalized) - } - } - - if !slices.Contains(outputs, "text_completion") { - provider := gjson.GetBytes(rawData, "provider") - if provider.Exists() { - key := makeKey(model, normalizeProvider(provider.String()), normalizeRequestType(schemas.TextCompletionRequest)) - - mc.mu.RLock() - _, ok := mc.pricingData[key] - mc.mu.RUnlock() - if ok { - outputs = append(outputs, "text_completion") - } - } - } - - if len(outputs) > 0 { - newResponseTypes[model] = outputs - } - - supported := extractSupportedParams(&parsed) - if len(supported) > 0 { - newParamsIndex[model] = supported - } - - var p struct { - MaxOutputTokens *int `json:"max_output_tokens"` - } - if err := json.Unmarshal(rawData, &p); err == nil && (p.MaxOutputTokens != nil || parsed.VertexMultiRegionOnly != nil) { - modelParamsEntries[model] = providerUtils.ModelParams{ - MaxOutputTokens: p.MaxOutputTokens, - IsVertexMultiRegionOnly: parsed.VertexMultiRegionOnly, - } - } - } - - mc.mu.Lock() - mc.supportedResponseTypes = newResponseTypes - mc.supportedParams = newParamsIndex - mc.mu.Unlock() - - if len(modelParamsEntries) > 0 { - providerUtils.BulkSetModelParams(modelParamsEntries) - } -} - -// loadModelParametersIntoMemoryFromURL loads model parameters from the remote URL into the -// provider utils cache (when config store is not available). -func (mc *ModelCatalog) loadModelParametersIntoMemoryFromURL(ctx context.Context) error { - paramsData, err := WithRetries(ctx, urlFetchMaxRetries, urlFetchMaxBackoff, func() (map[string]json.RawMessage, error) { - return mc.loadModelParametersFromURL(ctx) - }) - if err != nil { - return fmt.Errorf("failed to load model parameters from URL: %w", err) - } - mc.applyModelParameters(paramsData) - return nil -} - -// syncModelParameters syncs model parameters data from URL into memory cache -func (mc *ModelCatalog) syncModelParameters(ctx context.Context) error { - if mc.shouldSyncGate != nil { - if !mc.shouldSyncGate(ctx) { - mc.logger.Debug("model parameters sync cancelled by custom gate") - return nil - } - } - mc.logger.Debug("starting model parameters synchronization") - - paramsData, err := WithRetries(ctx, urlFetchMaxRetries, urlFetchMaxBackoff, func() (map[string]json.RawMessage, error) { - return mc.loadModelParametersFromURL(ctx) - }) - if err != nil { - if mc.configStore != nil { - rows, dbErr := mc.configStore.GetModelParameters(ctx) - if dbErr == nil && len(rows) > 0 { - mc.logger.Error("failed to load model parameters from URL, falling back to existing database records: %v", err) - return nil - } - } - return fmt.Errorf("failed to load model parameters from URL and no existing data in database: %w", err) - } - - // Persist to database if config store is available - if mc.configStore != nil { - err = mc.configStore.ExecuteTransaction(ctx, func(tx *gorm.DB) error { - for model, data := range paramsData { - params := &configstoreTables.TableModelParameters{ - Model: model, - Data: string(data), - } - if err := mc.configStore.UpsertModelParameters(ctx, params, tx); err != nil { - return fmt.Errorf("failed to upsert model parameters for model %s: %w", model, err) - } - } - return nil - }) - if err != nil { - return fmt.Errorf("failed to sync model parameters to database: %w", err) - } - } - - mc.applyModelParameters(paramsData) - - mc.logger.Info("successfully synced %d model parameters records", len(paramsData)) - return nil -} - -// loadModelParametersFromURL loads model parameters data from the configured URL (supports file:// and http(s)://) -func (mc *ModelCatalog) loadModelParametersFromURL(ctx context.Context) (map[string]json.RawMessage, error) { - rawURL := mc.getModelParametersURL() - - parsed, err := url.Parse(rawURL) - if err != nil { - return nil, fmt.Errorf("failed to parse model parameters URL: %w", err) - } - - var data []byte - - if parsed.Scheme == "file" { - data, err = os.ReadFile(parsed.Path) - if err != nil { - return nil, fmt.Errorf("failed to read model parameters file: %w", err) - } - } else { - if err := bifrost.ValidateExternalURL(rawURL, true); err != nil { - return nil, fmt.Errorf("model parameters URL validation failed: %w", err) - } - client := &http.Client{Timeout: DefaultModelParametersTimeout} - req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) - if err != nil { - return nil, fmt.Errorf("failed to create HTTP request: %w", err) - } - resp, err := client.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to download model parameters data: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("failed to download model parameters data: HTTP %d", resp.StatusCode) - } - - data, err = io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read model parameters response: %w", err) - } - } - - var paramsData map[string]json.RawMessage - if err := json.Unmarshal(data, ¶msData); err != nil { - return nil, fmt.Errorf("failed to unmarshal model parameters data: %w", err) - } - - mc.logger.Debug("successfully loaded and parsed %d model parameters records", len(paramsData)) - return paramsData, nil -} diff --git a/framework/modelcatalog/utils.go b/framework/modelcatalog/utils.go deleted file mode 100644 index b26cfa15cd..0000000000 --- a/framework/modelcatalog/utils.go +++ /dev/null @@ -1,458 +0,0 @@ -package modelcatalog - -import ( - "context" - "slices" - "strings" - "time" - - "github.com/bytedance/sonic" - "github.com/maximhq/bifrost/core/schemas" - configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" -) - -const retryBackoffMin = time.Second - -// WithRetries runs op until it succeeds or maxRetries retries are exhausted -// (1 initial attempt + maxRetries retries). After each failure it waits with -// exponential backoff starting at 1 second (retryBackoffMin), capped at maxBackoff -// when maxBackoff > 0. If maxBackoff is zero, there is no upper cap on the delay. -func WithRetries[T any](ctx context.Context, maxRetries int, maxBackoff time.Duration, op func() (T, error)) (T, error) { - var zero T - if maxRetries < 0 { - maxRetries = 0 - } - var lastErr error - for attempt := 0; attempt <= maxRetries; attempt++ { - select { - case <-ctx.Done(): - return zero, ctx.Err() - default: - } - - if attempt > 0 { - backoff := retryBackoffMin * time.Duration(1< 0 && backoff > maxBackoff { - backoff = maxBackoff - } - select { - case <-ctx.Done(): - return zero, ctx.Err() - case <-time.After(backoff): - } - } - v, err := op() - if err == nil { - return v, nil - } - lastErr = err - } - return zero, lastErr -} - -// makeKey creates a unique key for a model, provider, and mode for pricingData map -func makeKey(model, provider, mode string) string { return model + "|" + provider + "|" + mode } - -// normalizeProvider normalizes the provider name to a consistent format -func normalizeProvider(p string) string { - if strings.Contains(p, "vertex_ai") || p == "google-vertex" { - return string(schemas.Vertex) - } else if strings.Contains(p, "bedrock") { - return string(schemas.Bedrock) - } else if strings.Contains(p, "cohere") { - return string(schemas.Cohere) - } else if strings.Contains(p, "runwayml") { - return string(schemas.Runway) - } else if strings.Contains(p, "fireworks_ai") { - return string(schemas.Fireworks) - } else { - return p - } -} - -// normalizeRequestType normalizes the request type to a consistent format -func normalizeRequestType(reqType schemas.RequestType) string { - baseType := "unknown" - - switch reqType { - case schemas.TextCompletionRequest, schemas.TextCompletionStreamRequest: - baseType = "completion" - case schemas.ChatCompletionRequest, schemas.ChatCompletionStreamRequest: - baseType = "chat" - case schemas.ResponsesRequest, schemas.ResponsesStreamRequest, schemas.WebSocketResponsesRequest, schemas.RealtimeRequest, schemas.CompactionRequest: - baseType = "responses" - case schemas.EmbeddingRequest: - baseType = "embedding" - case schemas.RerankRequest: - baseType = "rerank" - case schemas.SpeechRequest, schemas.SpeechStreamRequest: - baseType = "audio_speech" - case schemas.TranscriptionRequest, schemas.TranscriptionStreamRequest: - baseType = "audio_transcription" - case schemas.ImageGenerationRequest, schemas.ImageGenerationStreamRequest, schemas.ImageVariationRequest: - baseType = "image_generation" - case schemas.ImageEditRequest, schemas.ImageEditStreamRequest: - baseType = "image_edit" - case schemas.VideoGenerationRequest, schemas.VideoRemixRequest: - baseType = "video_generation" - case schemas.OCRRequest: - baseType = "ocr" - case schemas.ContainerCreateRequest: - baseType = "container_create" - } - - return baseType -} - -// normalizeStreamRequestType normalizes the stream request type to a consistent format -// It returns the base request type for the stream request type. -func normalizeStreamRequestType(rt schemas.RequestType) schemas.RequestType { - switch rt { - case schemas.TextCompletionStreamRequest: - return schemas.TextCompletionRequest - case schemas.ChatCompletionStreamRequest: - return schemas.ChatCompletionRequest - case schemas.ResponsesStreamRequest, schemas.WebSocketResponsesRequest: - return schemas.ResponsesRequest - case schemas.RealtimeRequest: - return schemas.RealtimeRequest - case schemas.SpeechStreamRequest: - return schemas.SpeechRequest - case schemas.TranscriptionStreamRequest: - return schemas.TranscriptionRequest - case schemas.ImageGenerationStreamRequest: - return schemas.ImageGenerationRequest - case schemas.ImageEditStreamRequest: - return schemas.ImageEditRequest - default: - return rt - } -} - -// extractModelName extracts the model name from a model key that may be in provider/model format -func extractModelName(modelKey string) string { - if strings.Contains(modelKey, "/") { - parts := strings.Split(modelKey, "/") - if len(parts) > 1 { - return strings.Join(parts[1:], "/") - } - } - return modelKey -} - -// convertPricingDataToTableModelPricing converts the pricing data to a TableModelPricing struct -func convertPricingDataToTableModelPricing(modelKey string, entry PricingEntry) configstoreTables.TableModelPricing { - provider := normalizeProvider(entry.Provider) - modelName := extractModelName(modelKey) - - return configstoreTables.TableModelPricing{ - Model: modelName, - BaseModel: entry.BaseModel, - Provider: provider, - Mode: entry.Mode, - ContextLength: entry.ContextLength, - MaxInputTokens: entry.MaxInputTokens, - MaxOutputTokens: entry.MaxOutputTokens, - Architecture: entry.Architecture, - - // Costs - Text - InputCostPerToken: entry.InputCostPerToken, - OutputCostPerToken: entry.OutputCostPerToken, - InputCostPerTokenBatches: entry.InputCostPerTokenBatches, - OutputCostPerTokenBatches: entry.OutputCostPerTokenBatches, - InputCostPerTokenPriority: entry.InputCostPerTokenPriority, - OutputCostPerTokenPriority: entry.OutputCostPerTokenPriority, - InputCostPerTokenFlex: entry.InputCostPerTokenFlex, - OutputCostPerTokenFlex: entry.OutputCostPerTokenFlex, - InputCostPerTokenAbove200kTokens: entry.InputCostPerTokenAbove200kTokens, - InputCostPerTokenAbove200kTokensPriority: entry.InputCostPerTokenAbove200kTokensPriority, - OutputCostPerTokenAbove200kTokens: entry.OutputCostPerTokenAbove200kTokens, - OutputCostPerTokenAbove200kTokensPriority: entry.OutputCostPerTokenAbove200kTokensPriority, - // Costs - 272k Tier - InputCostPerTokenAbove272kTokens: entry.InputCostPerTokenAbove272kTokens, - InputCostPerTokenAbove272kTokensPriority: entry.InputCostPerTokenAbove272kTokensPriority, - OutputCostPerTokenAbove272kTokens: entry.OutputCostPerTokenAbove272kTokens, - OutputCostPerTokenAbove272kTokensPriority: entry.OutputCostPerTokenAbove272kTokensPriority, - // Costs - Character - InputCostPerCharacter: entry.InputCostPerCharacter, - // Costs - 128k Tier - InputCostPerTokenAbove128kTokens: entry.InputCostPerTokenAbove128kTokens, - InputCostPerImageAbove128kTokens: entry.InputCostPerImageAbove128kTokens, - InputCostPerVideoPerSecondAbove128kTokens: entry.InputCostPerVideoPerSecondAbove128kTokens, - InputCostPerAudioPerSecondAbove128kTokens: entry.InputCostPerAudioPerSecondAbove128kTokens, - OutputCostPerTokenAbove128kTokens: entry.OutputCostPerTokenAbove128kTokens, - - // Costs - Cache - CacheCreationInputTokenCost: entry.CacheCreationInputTokenCost, - CacheReadInputTokenCost: entry.CacheReadInputTokenCost, - CacheCreationInputTokenCostAbove200kTokens: entry.CacheCreationInputTokenCostAbove200kTokens, - CacheReadInputTokenCostAbove200kTokens: entry.CacheReadInputTokenCostAbove200kTokens, - CacheReadInputTokenCostAbove200kTokensPriority: entry.CacheReadInputTokenCostAbove200kTokensPriority, - CacheCreationInputTokenCostAbove1hr: entry.CacheCreationInputTokenCostAbove1hr, - CacheCreationInputTokenCostAbove1hrAbove200kTokens: entry.CacheCreationInputTokenCostAbove1hrAbove200kTokens, - CacheCreationInputAudioTokenCost: entry.CacheCreationInputAudioTokenCost, - CacheReadInputTokenCostPriority: entry.CacheReadInputTokenCostPriority, - CacheReadInputTokenCostFlex: entry.CacheReadInputTokenCostFlex, - CacheReadInputImageTokenCost: entry.CacheReadInputImageTokenCost, - CacheReadInputTokenCostAbove272kTokens: entry.CacheReadInputTokenCostAbove272kTokens, - CacheReadInputTokenCostAbove272kTokensPriority: entry.CacheReadInputTokenCostAbove272kTokensPriority, - - // Costs - Image - InputCostPerImage: entry.InputCostPerImage, - InputCostPerPixel: entry.InputCostPerPixel, - OutputCostPerImage: entry.OutputCostPerImage, - OutputCostPerPixel: entry.OutputCostPerPixel, - OutputCostPerImagePremiumImage: entry.OutputCostPerImagePremiumImage, - OutputCostPerImageAbove512x512Pixels: entry.OutputCostPerImageAbove512x512Pixels, - OutputCostPerImageAbove512x512PixelsPremium: entry.OutputCostPerImageAbove512x512PixelsPremium, - OutputCostPerImageAbove1024x1024Pixels: entry.OutputCostPerImageAbove1024x1024Pixels, - OutputCostPerImageAbove1024x1024PixelsPremium: entry.OutputCostPerImageAbove1024x1024PixelsPremium, - OutputCostPerImageAbove2048x2048Pixels: entry.OutputCostPerImageAbove2048x2048Pixels, - OutputCostPerImageAbove4096x4096Pixels: entry.OutputCostPerImageAbove4096x4096Pixels, - OutputCostPerImageLowQuality: entry.OutputCostPerImageLowQuality, - OutputCostPerImageMediumQuality: entry.OutputCostPerImageMediumQuality, - OutputCostPerImageHighQuality: entry.OutputCostPerImageHighQuality, - OutputCostPerImageAutoQuality: entry.OutputCostPerImageAutoQuality, - // Costs - Image Token - InputCostPerImageToken: entry.InputCostPerImageToken, - OutputCostPerImageToken: entry.OutputCostPerImageToken, - - // Costs - Audio/Video - InputCostPerAudioToken: entry.InputCostPerAudioToken, - InputCostPerAudioPerSecond: entry.InputCostPerAudioPerSecond, - InputCostPerSecond: entry.InputCostPerSecond, - InputCostPerVideoPerSecond: entry.InputCostPerVideoPerSecond, - OutputCostPerAudioToken: entry.OutputCostPerAudioToken, - OutputCostPerVideoPerSecond: entry.OutputCostPerVideoPerSecond, - OutputCostPerSecond: entry.OutputCostPerSecond, - - // Costs - Other - SearchContextCostPerQuery: entry.SearchContextCostPerQuery, - CodeInterpreterCostPerSession: entry.CodeInterpreterCostPerSession, - - // Costs - OCR - OCRCostPerPage: entry.OCRCostPerPage, - AnnotationCostPerPage: entry.AnnotationCostPerPage, - } -} - -// convertTableModelPricingToPricingData converts the TableModelPricing struct to a PricingEntry struct -func convertTableModelPricingToPricingData(pricing *configstoreTables.TableModelPricing) *PricingEntry { - options := PricingOptions{ - // Costs - Text - InputCostPerToken: pricing.InputCostPerToken, - OutputCostPerToken: pricing.OutputCostPerToken, - InputCostPerTokenBatches: pricing.InputCostPerTokenBatches, - OutputCostPerTokenBatches: pricing.OutputCostPerTokenBatches, - InputCostPerTokenPriority: pricing.InputCostPerTokenPriority, - OutputCostPerTokenPriority: pricing.OutputCostPerTokenPriority, - InputCostPerTokenFlex: pricing.InputCostPerTokenFlex, - OutputCostPerTokenFlex: pricing.OutputCostPerTokenFlex, - InputCostPerTokenAbove200kTokens: pricing.InputCostPerTokenAbove200kTokens, - InputCostPerTokenAbove200kTokensPriority: pricing.InputCostPerTokenAbove200kTokensPriority, - OutputCostPerTokenAbove200kTokens: pricing.OutputCostPerTokenAbove200kTokens, - OutputCostPerTokenAbove200kTokensPriority: pricing.OutputCostPerTokenAbove200kTokensPriority, - // Costs - 272k Tier - InputCostPerTokenAbove272kTokens: pricing.InputCostPerTokenAbove272kTokens, - InputCostPerTokenAbove272kTokensPriority: pricing.InputCostPerTokenAbove272kTokensPriority, - OutputCostPerTokenAbove272kTokens: pricing.OutputCostPerTokenAbove272kTokens, - OutputCostPerTokenAbove272kTokensPriority: pricing.OutputCostPerTokenAbove272kTokensPriority, - // Costs - Character - InputCostPerCharacter: pricing.InputCostPerCharacter, - // Costs - 128k Tier - InputCostPerTokenAbove128kTokens: pricing.InputCostPerTokenAbove128kTokens, - InputCostPerImageAbove128kTokens: pricing.InputCostPerImageAbove128kTokens, - InputCostPerVideoPerSecondAbove128kTokens: pricing.InputCostPerVideoPerSecondAbove128kTokens, - InputCostPerAudioPerSecondAbove128kTokens: pricing.InputCostPerAudioPerSecondAbove128kTokens, - OutputCostPerTokenAbove128kTokens: pricing.OutputCostPerTokenAbove128kTokens, - - // Costs - Cache - CacheCreationInputTokenCost: pricing.CacheCreationInputTokenCost, - CacheReadInputTokenCost: pricing.CacheReadInputTokenCost, - CacheCreationInputTokenCostAbove200kTokens: pricing.CacheCreationInputTokenCostAbove200kTokens, - CacheReadInputTokenCostAbove200kTokens: pricing.CacheReadInputTokenCostAbove200kTokens, - CacheReadInputTokenCostAbove200kTokensPriority: pricing.CacheReadInputTokenCostAbove200kTokensPriority, - CacheCreationInputTokenCostAbove1hr: pricing.CacheCreationInputTokenCostAbove1hr, - CacheCreationInputTokenCostAbove1hrAbove200kTokens: pricing.CacheCreationInputTokenCostAbove1hrAbove200kTokens, - CacheCreationInputAudioTokenCost: pricing.CacheCreationInputAudioTokenCost, - CacheReadInputTokenCostPriority: pricing.CacheReadInputTokenCostPriority, - CacheReadInputTokenCostFlex: pricing.CacheReadInputTokenCostFlex, - CacheReadInputImageTokenCost: pricing.CacheReadInputImageTokenCost, - CacheReadInputTokenCostAbove272kTokens: pricing.CacheReadInputTokenCostAbove272kTokens, - CacheReadInputTokenCostAbove272kTokensPriority: pricing.CacheReadInputTokenCostAbove272kTokensPriority, - - // Costs - Image - InputCostPerImage: pricing.InputCostPerImage, - InputCostPerPixel: pricing.InputCostPerPixel, - OutputCostPerImage: pricing.OutputCostPerImage, - OutputCostPerPixel: pricing.OutputCostPerPixel, - OutputCostPerImagePremiumImage: pricing.OutputCostPerImagePremiumImage, - OutputCostPerImageAbove512x512Pixels: pricing.OutputCostPerImageAbove512x512Pixels, - OutputCostPerImageAbove512x512PixelsPremium: pricing.OutputCostPerImageAbove512x512PixelsPremium, - OutputCostPerImageAbove1024x1024Pixels: pricing.OutputCostPerImageAbove1024x1024Pixels, - OutputCostPerImageAbove1024x1024PixelsPremium: pricing.OutputCostPerImageAbove1024x1024PixelsPremium, - OutputCostPerImageAbove2048x2048Pixels: pricing.OutputCostPerImageAbove2048x2048Pixels, - OutputCostPerImageAbove4096x4096Pixels: pricing.OutputCostPerImageAbove4096x4096Pixels, - OutputCostPerImageLowQuality: pricing.OutputCostPerImageLowQuality, - OutputCostPerImageMediumQuality: pricing.OutputCostPerImageMediumQuality, - OutputCostPerImageHighQuality: pricing.OutputCostPerImageHighQuality, - OutputCostPerImageAutoQuality: pricing.OutputCostPerImageAutoQuality, - // Costs - Image Token - InputCostPerImageToken: pricing.InputCostPerImageToken, - OutputCostPerImageToken: pricing.OutputCostPerImageToken, - - // Costs - Audio/Video - InputCostPerAudioToken: pricing.InputCostPerAudioToken, - InputCostPerAudioPerSecond: pricing.InputCostPerAudioPerSecond, - InputCostPerSecond: pricing.InputCostPerSecond, - InputCostPerVideoPerSecond: pricing.InputCostPerVideoPerSecond, - OutputCostPerAudioToken: pricing.OutputCostPerAudioToken, - OutputCostPerVideoPerSecond: pricing.OutputCostPerVideoPerSecond, - OutputCostPerSecond: pricing.OutputCostPerSecond, - - // Costs - Other - SearchContextCostPerQuery: pricing.SearchContextCostPerQuery, - CodeInterpreterCostPerSession: pricing.CodeInterpreterCostPerSession, - - // Costs - OCR - OCRCostPerPage: pricing.OCRCostPerPage, - AnnotationCostPerPage: pricing.AnnotationCostPerPage, - } - return &PricingEntry{ - BaseModel: pricing.BaseModel, - Provider: pricing.Provider, - Mode: pricing.Mode, - ContextLength: pricing.ContextLength, - MaxInputTokens: pricing.MaxInputTokens, - MaxOutputTokens: pricing.MaxOutputTokens, - Architecture: pricing.Architecture, - AdditionalAttributes: pricing.AdditionalAttributes, - PricingOptions: options, - } -} - -// convertTablePricingOverrideToPricingOverride converts a TablePricingOverride to a PricingOverride. -func convertTablePricingOverrideToPricingOverride(override *configstoreTables.TablePricingOverride) (PricingOverride, error) { - var options PricingOptions - if err := sonic.Unmarshal([]byte(override.PricingPatchJSON), &options); err != nil { - return PricingOverride{}, err - } - return PricingOverride{ - ID: override.ID, - Name: override.Name, - ScopeKind: ScopeKind(override.ScopeKind), - VirtualKeyID: override.VirtualKeyID, - ProviderID: override.ProviderID, - ProviderKeyID: override.ProviderKeyID, - MatchType: MatchType(override.MatchType), - Pattern: override.Pattern, - RequestTypes: override.RequestTypes, - Options: options, - }, nil -} - -// normalizeEndpointToOutputType converts a supported_endpoints URL path to a normalized output type. -// Returns empty string for unrecognized endpoints. -func normalizeEndpointToOutputType(endpoint string) string { - switch { - case strings.Contains(endpoint, "/chat/completions"): - return "chat_completion" - case strings.Contains(endpoint, "/responses"): - return "responses" - case strings.Contains(endpoint, "/completions"): - return "text_completion" - default: - return "" - } -} - -// normalizeModeToOutputType converts mode to a normalized output type. -func normalizeModeToOutputType(mode string) string { - switch mode { - case "chat": - return "chat_completion" - case "completion": - return "text_completion" - case "responses": - return "responses" - default: - return "" - } -} - -// modelParametersParseResult is the parsed result type used by buildSupportedOutputsIndex. -type modelParametersParseResult struct { - Mode *string `json:"mode,omitempty"` - SupportedEndpoints []string `json:"supported_endpoints,omitempty"` - ModelParameters []struct { - ID string `json:"id"` - } `json:"model_parameters,omitempty"` - SupportsAssistantPrefill *bool `json:"supports_assistant_prefill,omitempty"` - SupportsFunctionCalling *bool `json:"supports_function_calling,omitempty"` - SupportsParallelFunctionCalling *bool `json:"supports_parallel_function_calling,omitempty"` - SupportsToolChoice *bool `json:"supports_tool_choice,omitempty"` - SupportsReasoning *bool `json:"supports_reasoning,omitempty"` - SupportsResponseSchema *bool `json:"supports_response_schema,omitempty"` - SupportsServiceTier *bool `json:"supports_service_tier,omitempty"` - SupportsPromptCaching *bool `json:"supports_prompt_caching,omitempty"` - VertexMultiRegionOnly *bool `json:"vertex_multi_region_only,omitempty"` -} - -// extractSupportedParams builds a list of supported OpenAI-compatible parameter -// names from model_parameters[].id values and supports_* boolean flags. -func extractSupportedParams(parsed *modelParametersParseResult) []string { - var supported []string - addParam := func(name string) { - if !slices.Contains(supported, name) { - supported = append(supported, name) - } - } - - // From model_parameters[].id — map IDs to request param names - for _, mp := range parsed.ModelParameters { - switch mp.ID { - case "reasoning_effort", "reasoning_summary": - addParam("reasoning") - case "web_search": - addParam("web_search_options") - case "promptTools", "image_detail", "stream": - // skip — not top-level request parameters - default: - addParam(mp.ID) - } - } - - // From supports_* boolean flags - if parsed.SupportsAssistantPrefill != nil && *parsed.SupportsAssistantPrefill { - // not an actual model parameter; if present, trailing assistant messages - // for anthropic and bedrock's anthropic models will not be trimmed - addParam("assistant_prefill") - } - if parsed.SupportsFunctionCalling != nil && *parsed.SupportsFunctionCalling { - addParam("tools") - } - if parsed.SupportsParallelFunctionCalling != nil && *parsed.SupportsParallelFunctionCalling { - addParam("parallel_tool_calls") - } - if parsed.SupportsToolChoice != nil && *parsed.SupportsToolChoice { - addParam("tool_choice") - } - if parsed.SupportsReasoning != nil && *parsed.SupportsReasoning { - addParam("reasoning") - } - if parsed.SupportsResponseSchema != nil && *parsed.SupportsResponseSchema { - addParam("response_format") - addParam("text") - } - if parsed.SupportsServiceTier != nil && *parsed.SupportsServiceTier { - addParam("service_tier") - } - if parsed.SupportsPromptCaching != nil && *parsed.SupportsPromptCaching { - addParam("cachePoint") - addParam("cache_control") - addParam("prompt_cache_key") - addParam("prompt_cache_retention") - } - - return supported -} diff --git a/framework/plugins/main.go b/framework/plugins/main.go index b7bc20bb06..5ce955aad2 100644 --- a/framework/plugins/main.go +++ b/framework/plugins/main.go @@ -27,7 +27,7 @@ func AsLLMPlugin(plugin schemas.BasePlugin) schemas.LLMPlugin { // Check if it's a DynamicPlugin first if dp, ok := plugin.(*DynamicPlugin); ok { // Only return as LLMPlugin if it actually has LLM hooks - if dp.preLLMHook != nil || dp.postLLMHook != nil { + if dp.preRequestHook != nil || dp.preLLMHook != nil || dp.postLLMHook != nil { return dp } return nil diff --git a/framework/plugins/soloader.go b/framework/plugins/soloader.go index face5772a7..afbec9d6f2 100644 --- a/framework/plugins/soloader.go +++ b/framework/plugins/soloader.go @@ -94,6 +94,15 @@ func (l *SharedObjectPluginLoader) LoadPlugin(path string, config any) (schemas. } } + // Optional: PreRequestHook — new .so plugins built against LLMPlugin can export this + // to participate in routing. Legacy plugins predating PreRequestHook keep working; + // DynamicPlugin's default PreRequestHook is a no-op passthrough. + if sym, err := pluginObj.Lookup("PreRequestHook"); err == nil { + if dp.preRequestHook, ok = sym.(func(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) error); !ok { + return nil, fmt.Errorf("failed to cast PreRequestHook to expected signature") + } + } + // Optional: PreLLMHook (with backward compatibility for legacy PreHook) if sym, err := pluginObj.Lookup("PreLLMHook"); err == nil { if dp.preLLMHook, ok = sym.(func(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) (*schemas.BifrostRequest, *schemas.LLMPluginShortCircuit, error)); !ok { diff --git a/framework/plugins/soplugin.go b/framework/plugins/soplugin.go index 196a1ceed2..8b73e521ae 100644 --- a/framework/plugins/soplugin.go +++ b/framework/plugins/soplugin.go @@ -27,8 +27,12 @@ type DynamicPlugin struct { httpTransportStreamChunkHook func(ctx *schemas.BifrostContext, req *schemas.HTTPRequest, stream *schemas.BifrostStreamChunk) (*schemas.BifrostStreamChunk, error) // LLMPlugin (optional) - preLLMHook func(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) (*schemas.BifrostRequest, *schemas.LLMPluginShortCircuit, error) - postLLMHook func(ctx *schemas.BifrostContext, resp *schemas.BifrostResponse, bifrostErr *schemas.BifrostError) (*schemas.BifrostResponse, *schemas.BifrostError, error) + // preRequestHook is forward-compat: new .so plugins built against LLMPlugin can export + // PreRequestHook to participate in the per-request routing phase. Legacy plugins predating + // PreRequestHook leave it nil and silently no-op for routing. + preRequestHook func(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) error + preLLMHook func(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) (*schemas.BifrostRequest, *schemas.LLMPluginShortCircuit, error) + postLLMHook func(ctx *schemas.BifrostContext, resp *schemas.BifrostResponse, bifrostErr *schemas.BifrostError) (*schemas.BifrostResponse, *schemas.BifrostError, error) // MCPPlugin (optional) preMCPHook func(ctx *schemas.BifrostContext, req *schemas.BifrostMCPRequest) (*schemas.BifrostMCPRequest, *schemas.MCPPluginShortCircuit, error) @@ -79,6 +83,16 @@ func (dp *DynamicPlugin) HTTPTransportStreamChunkHook(ctx *schemas.BifrostContex return dp.httpTransportStreamChunkHook(ctx, req, stream) } +// PreRequestHook is invoked once per top-level request to decide provider/model/fallbacks +// (LLMPlugin interface). Defaults to a no-op passthrough for legacy plugins that don't +// export PreRequestHook. +func (dp *DynamicPlugin) PreRequestHook(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) error { + if dp.preRequestHook == nil { + return nil + } + return dp.preRequestHook(ctx, req) +} + // PreLLMHook is invoked before LLM provider calls (LLMPlugin interface) func (dp *DynamicPlugin) PreLLMHook(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) (*schemas.BifrostRequest, *schemas.LLMPluginShortCircuit, error) { if dp.preLLMHook == nil { diff --git a/framework/streaming/audio.go b/framework/streaming/audio.go index 0390ea5aaf..07d7636db1 100644 --- a/framework/streaming/audio.go +++ b/framework/streaming/audio.go @@ -181,6 +181,7 @@ func (a *Accumulator) processAudioStreamingResponse(ctx *schemas.BifrostContext, StreamType: StreamTypeAudio, RequestedModel: requestedModel, ResolvedModel: resolvedModel, + RoutingInfo: bifrost.GetResponseRoutingInfo(result, bifrostErr), Provider: provider, Data: data, RawRequest: &rawRequest, @@ -193,6 +194,7 @@ func (a *Accumulator) processAudioStreamingResponse(ctx *schemas.BifrostContext, StreamType: StreamTypeAudio, RequestedModel: requestedModel, ResolvedModel: resolvedModel, + RoutingInfo: bifrost.GetResponseRoutingInfo(result, bifrostErr), Provider: provider, Data: nil, }, nil diff --git a/framework/streaming/chat.go b/framework/streaming/chat.go index 0c03ab3d40..5e2d0a66c9 100644 --- a/framework/streaming/chat.go +++ b/framework/streaming/chat.go @@ -568,6 +568,7 @@ func (a *Accumulator) processChatStreamingResponse(ctx *schemas.BifrostContext, Provider: provider, RequestedModel: model, ResolvedModel: resolvedModel, + RoutingInfo: bifrost.GetResponseRoutingInfo(result, bifrostErr), Data: data, RawRequest: &rawRequest, }, nil @@ -580,6 +581,7 @@ func (a *Accumulator) processChatStreamingResponse(ctx *schemas.BifrostContext, Provider: provider, RequestedModel: model, ResolvedModel: resolvedModel, + RoutingInfo: bifrost.GetResponseRoutingInfo(result, bifrostErr), Data: nil, }, nil } diff --git a/framework/streaming/images.go b/framework/streaming/images.go index 367b52c037..781813e8ba 100644 --- a/framework/streaming/images.go +++ b/framework/streaming/images.go @@ -315,6 +315,7 @@ func (a *Accumulator) processImageStreamingResponse(ctx *schemas.BifrostContext, Provider: provider, RequestedModel: requestedModel, ResolvedModel: resolvedModel, + RoutingInfo: bifrost.GetResponseRoutingInfo(result, bifrostErr), Data: data, RawRequest: &rawRequest, }, nil @@ -331,6 +332,7 @@ func (a *Accumulator) processImageStreamingResponse(ctx *schemas.BifrostContext, Provider: provider, RequestedModel: requestedModel, ResolvedModel: resolvedModel, + RoutingInfo: bifrost.GetResponseRoutingInfo(result, bifrostErr), Data: nil, }, nil } diff --git a/framework/streaming/passthrough.go b/framework/streaming/passthrough.go index 1cdad9870d..59dd1232f7 100644 --- a/framework/streaming/passthrough.go +++ b/framework/streaming/passthrough.go @@ -61,6 +61,7 @@ func (a *Accumulator) processPassthroughStreamingResponse(ctx *schemas.BifrostCo StreamType: StreamTypePassthrough, RequestedModel: requestedModel, ResolvedModel: resolvedModel, + RoutingInfo: bifrost.GetResponseRoutingInfo(result, bifrostErr), Provider: provider, Data: nil, }, nil @@ -134,6 +135,7 @@ func (a *Accumulator) processPassthroughStreamingResponse(ctx *schemas.BifrostCo StreamType: StreamTypePassthrough, RequestedModel: requestedModel, ResolvedModel: resolvedModel, + RoutingInfo: bifrost.GetResponseRoutingInfo(result, bifrostErr), Provider: provider, Data: data, RawRequest: &rawRequest, diff --git a/framework/streaming/responses.go b/framework/streaming/responses.go index 91b0326eb7..63c05c10dd 100644 --- a/framework/streaming/responses.go +++ b/framework/streaming/responses.go @@ -996,6 +996,7 @@ func (a *Accumulator) processResponsesStreamingResponse(ctx *schemas.BifrostCont Provider: provider, RequestedModel: requestedModel, ResolvedModel: resolvedModel, + RoutingInfo: bifrost.GetResponseRoutingInfo(result, bifrostErr), Data: data, RawRequest: &rawRequest, }, nil @@ -1007,6 +1008,7 @@ func (a *Accumulator) processResponsesStreamingResponse(ctx *schemas.BifrostCont Provider: provider, RequestedModel: requestedModel, ResolvedModel: resolvedModel, + RoutingInfo: bifrost.GetResponseRoutingInfo(result, bifrostErr), Data: nil, }, nil } diff --git a/framework/streaming/transcription.go b/framework/streaming/transcription.go index 3367e25ad6..40debfd1d5 100644 --- a/framework/streaming/transcription.go +++ b/framework/streaming/transcription.go @@ -199,6 +199,7 @@ func (a *Accumulator) processTranscriptionStreamingResponse(ctx *schemas.Bifrost Provider: provider, RequestedModel: requestedModel, ResolvedModel: resolvedModel, + RoutingInfo: bifrost.GetResponseRoutingInfo(result, bifrostErr), Data: data, RawRequest: &rawRequest, }, nil @@ -211,6 +212,7 @@ func (a *Accumulator) processTranscriptionStreamingResponse(ctx *schemas.Bifrost Provider: provider, RequestedModel: requestedModel, ResolvedModel: resolvedModel, + RoutingInfo: bifrost.GetResponseRoutingInfo(result, bifrostErr), Data: nil, }, nil } diff --git a/framework/streaming/types.go b/framework/streaming/types.go index c673963331..9dcb3b1708 100644 --- a/framework/streaming/types.go +++ b/framework/streaming/types.go @@ -244,6 +244,7 @@ type ProcessedStreamResponse struct { Provider schemas.ModelProvider RequestedModel string // original model requested by the caller ResolvedModel string // actual model used by the provider (equals RequestedModel when no alias mapping exists) + RoutingInfo schemas.RoutingInfo Data *AccumulatedData RawRequest *interface{} } diff --git a/framework/tracing/tracer_test.go b/framework/tracing/tracer_test.go index 2b467509af..d8e9e6b8cc 100644 --- a/framework/tracing/tracer_test.go +++ b/framework/tracing/tracer_test.go @@ -14,6 +14,9 @@ type testRealtimeObservabilityPlugin struct { func (p *testRealtimeObservabilityPlugin) GetName() string { return "test-observability" } func (p *testRealtimeObservabilityPlugin) Cleanup() error { return nil } +func (p *testRealtimeObservabilityPlugin) PreRequestHook(_ *schemas.BifrostContext, _ *schemas.BifrostRequest) error { + return nil +} func (p *testRealtimeObservabilityPlugin) PreLLMHook(_ *schemas.BifrostContext, req *schemas.BifrostRequest) (*schemas.BifrostRequest, *schemas.LLMPluginShortCircuit, error) { return req, nil, nil } diff --git a/helm-charts/bifrost/README.md b/helm-charts/bifrost/README.md index e4a4a703c6..6d8a9909de 100644 --- a/helm-charts/bifrost/README.md +++ b/helm-charts/bifrost/README.md @@ -8,19 +8,25 @@ Official Helm charts for deploying [Bifrost](https://github.com/maximhq/bifrost) ## Changelog +### Upcoming [2.1.23] + +- Added `bifrost.governance.complexityAnalyzerConfig` to configure complexity router analyzer boundaries and keyword lists from Helm. The value renders into `governance.complexity_analyzer_config` in the generated `config.json` and remains opt-in, so existing installs are unchanged unless the value is set. + ### 2.1.22 - Added `bifrost.governance.roles` array to `values.yaml`, `values.schema.json`, and `_helpers.tpl`. Each role requires a `name` and accepts optional `description`, `dac` (`own-data` | `team-data` | `all-data`, default `all-data`), `access_profile`, and `permissions[]` (`resource` + `operation`). - `bifrost.plugins.otel.config` now accepts either the existing single-profile shape or a new `profiles` wrapper (`otelProfilesConfig`) with an array of profiles. Each profile is independently enabled/disabled. A shared `plugin_span_filter` can be set at the top level in either shape. - Added `disable_content_logging` to OTEL config (both single-profile and per-profile). When `true`, message content (input/output messages, embeddings, tool definitions, tool call arguments/results) is dropped from exported spans — only metadata (model, tokens, latency) is sent to the collector. -- Added `otelPluginSpanFilter` (`mode`: `include`/`exclude`, `plugins` array) to the OTEL config schema, available in both single-profile and multi-profile shapes. +- Added `pluginSpanFilter` (`mode`: `include`/`exclude`, `plugins` array) to the OTEL config schema, available in both single-profile and multi-profile shapes. - Added `calendar_aligned` to `bifrost.governance.modelConfigs[]`. - Added `model_config_id` and `customer_id` as budget owner fields in `governance.budgets[]`, alongside the existing `virtual_key_id`, `provider_config_id`, and `team_id`. - Extended `attributeTeamMappings` and `attributeBusinessUnitMappings` in SCIM auth config with optional `attributeType` (`user` | `group`) and `attributeValue` fields to enable SCIM-driven team/business-unit provisioning. - Added OAuth MCP client config example to `values.yaml` showing `authType: oauth` with `oauthConfigId`. - Added `bifrost.sourceOfTruth` (`split` | `config.json`, optional). When set to `"config.json"`, sections explicitly present in the file become authoritative on startup — database-only rows for those sections are pruned. Omitting the field preserves the default `"split"` merge behavior. - Added `allow_private_network` to `networkConfig` in `values.schema.json`. When `true`, allows connections to RFC 1918 private IPs (10.x, 172.16.x, 192.168.x) — useful for providers on a k8s pod network, LAN, or private VPC. - +- Added `plugin_span_filter` (`mode`: `include`/`exclude`, `plugins` array) to the Datadog plugin config in `values.yaml`, `values.schema.json`, and `_helpers.tpl`. Selects which plugin hook spans are exported to Datadog; omit to export all. Each observability connector keeps its own independent filter. +- Added the `bigquery` plugin (BigQuery traces) to the chart — `values.yaml`, `values.schema.json`, and `_helpers.tpl`. Supports `project_id`, `dataset_id`, `table_id`, `location`, `service_account_key` (literal or `env.VAR`; omit for Application Default Credentials), `create_table_if_not_exists`, `flush_interval_seconds`, `buffer_size`, `custom_labels`, `disable_content_logging`, `request_headers`, and `plugin_span_filter`. Includes the same `version` validation guard as the other built-in plugins. +- The `pluginSpanFilter` schema definition is shared across the OTEL, Datadog, and BigQuery plugin configs (one reusable `$defs` shape rather than per-connector copies). This is a schema-definition naming detail only — the user-facing `plugin_span_filter` config key is unchanged. ### 2.1.21 diff --git a/helm-charts/bifrost/templates/_helpers.tpl b/helm-charts/bifrost/templates/_helpers.tpl index 0f54e259f0..65056996d7 100644 --- a/helm-charts/bifrost/templates/_helpers.tpl +++ b/helm-charts/bifrost/templates/_helpers.tpl @@ -487,6 +487,9 @@ false {{- if .Values.bifrost.governance.pricingOverrides }} {{- $_ := set $governance "pricing_overrides" .Values.bifrost.governance.pricingOverrides }} {{- end }} +{{- if .Values.bifrost.governance.complexityAnalyzerConfig }} +{{- $_ := set $governance "complexity_analyzer_config" .Values.bifrost.governance.complexityAnalyzerConfig }} +{{- end }} {{- if .Values.bifrost.governance.authConfig }} {{- $authConfig := dict }} {{- if and .Values.bifrost.governance.authConfig.existingSecret .Values.bifrost.governance.authConfig.usernameKey }} @@ -509,7 +512,7 @@ false {{- $_ := set $governance "auth_config" $authConfig }} {{- end }} {{- end }} -{{- if or $governance.budgets $governance.rate_limits $governance.customers $governance.teams $governance.business_units $governance.roles $governance.virtual_keys $governance.routing_rules $governance.model_configs $governance.providers $governance.pricing_overrides $governance.auth_config }} +{{- if or $governance.budgets $governance.rate_limits $governance.customers $governance.teams $governance.business_units $governance.roles $governance.virtual_keys $governance.routing_rules $governance.model_configs $governance.providers $governance.pricing_overrides $governance.complexity_analyzer_config $governance.auth_config }} {{- $_ := set $config "governance" $governance }} {{- end }} {{- end }} @@ -1182,9 +1185,15 @@ false {{- if $inputConfig.service_name }} {{- $_ := set $datadogConfig "service_name" $inputConfig.service_name }} {{- end }} +{{- if $inputConfig.ml_app }} +{{- $_ := set $datadogConfig "ml_app" $inputConfig.ml_app }} +{{- end }} {{- if $inputConfig.agent_addr }} {{- $_ := set $datadogConfig "agent_addr" $inputConfig.agent_addr }} {{- end }} +{{- if $inputConfig.dogstatsd_addr }} +{{- $_ := set $datadogConfig "dogstatsd_addr" $inputConfig.dogstatsd_addr }} +{{- end }} {{- if $inputConfig.env }} {{- $_ := set $datadogConfig "env" $inputConfig.env }} {{- end }} @@ -1194,13 +1203,80 @@ false {{- if $inputConfig.custom_tags }} {{- $_ := set $datadogConfig "custom_tags" $inputConfig.custom_tags }} {{- end }} +{{- if hasKey $inputConfig "enable_metrics" }} +{{- $_ := set $datadogConfig "enable_metrics" $inputConfig.enable_metrics }} +{{- end }} {{- if hasKey $inputConfig "enable_traces" }} {{- $_ := set $datadogConfig "enable_traces" $inputConfig.enable_traces }} {{- end }} +{{- if hasKey $inputConfig "enable_llm_obs" }} +{{- $_ := set $datadogConfig "enable_llm_obs" $inputConfig.enable_llm_obs }} +{{- end }} +{{- if hasKey $inputConfig "disable_content_logging" }} +{{- $_ := set $datadogConfig "disable_content_logging" $inputConfig.disable_content_logging }} +{{- end }} +{{- if hasKey $inputConfig "agentless" }} +{{- $_ := set $datadogConfig "agentless" $inputConfig.agentless }} +{{- end }} +{{- if $inputConfig.api_key }} +{{- $_ := set $datadogConfig "api_key" $inputConfig.api_key }} +{{- end }} +{{- if $inputConfig.site }} +{{- $_ := set $datadogConfig "site" $inputConfig.site }} +{{- end }} +{{- if $inputConfig.request_headers }} +{{- $_ := set $datadogConfig "request_headers" $inputConfig.request_headers }} +{{- end }} +{{- if $inputConfig.plugin_span_filter }} +{{- $_ := set $datadogConfig "plugin_span_filter" $inputConfig.plugin_span_filter }} +{{- end }} {{- $plugin := dict "enabled" true "name" "datadog" "config" $datadogConfig }} {{- if hasKey .Values.bifrost.plugins.datadog "version" }}{{- $_ := set $plugin "version" (.Values.bifrost.plugins.datadog.version | int) }}{{- end }} {{- $plugins = append $plugins $plugin }} {{- end }} +{{- if .Values.bifrost.plugins.bigquery.enabled }} +{{- $bigqueryConfig := dict }} +{{- $inputConfig := .Values.bifrost.plugins.bigquery.config | default dict }} +{{- if $inputConfig.project_id }} +{{- $_ := set $bigqueryConfig "project_id" $inputConfig.project_id }} +{{- end }} +{{- if $inputConfig.dataset_id }} +{{- $_ := set $bigqueryConfig "dataset_id" $inputConfig.dataset_id }} +{{- end }} +{{- if $inputConfig.table_id }} +{{- $_ := set $bigqueryConfig "table_id" $inputConfig.table_id }} +{{- end }} +{{- if $inputConfig.location }} +{{- $_ := set $bigqueryConfig "location" $inputConfig.location }} +{{- end }} +{{- if $inputConfig.service_account_key }} +{{- $_ := set $bigqueryConfig "service_account_key" $inputConfig.service_account_key }} +{{- end }} +{{- if hasKey $inputConfig "create_table_if_not_exists" }} +{{- $_ := set $bigqueryConfig "create_table_if_not_exists" $inputConfig.create_table_if_not_exists }} +{{- end }} +{{- if hasKey $inputConfig "flush_interval_seconds" }} +{{- $_ := set $bigqueryConfig "flush_interval_seconds" $inputConfig.flush_interval_seconds }} +{{- end }} +{{- if hasKey $inputConfig "buffer_size" }} +{{- $_ := set $bigqueryConfig "buffer_size" $inputConfig.buffer_size }} +{{- end }} +{{- if $inputConfig.custom_labels }} +{{- $_ := set $bigqueryConfig "custom_labels" $inputConfig.custom_labels }} +{{- end }} +{{- if hasKey $inputConfig "disable_content_logging" }} +{{- $_ := set $bigqueryConfig "disable_content_logging" $inputConfig.disable_content_logging }} +{{- end }} +{{- if $inputConfig.request_headers }} +{{- $_ := set $bigqueryConfig "request_headers" $inputConfig.request_headers }} +{{- end }} +{{- if $inputConfig.plugin_span_filter }} +{{- $_ := set $bigqueryConfig "plugin_span_filter" $inputConfig.plugin_span_filter }} +{{- end }} +{{- $plugin := dict "enabled" true "name" "bigquery" "config" $bigqueryConfig }} +{{- if hasKey .Values.bifrost.plugins.bigquery "version" }}{{- $_ := set $plugin "version" (.Values.bifrost.plugins.bigquery.version | int) }}{{- end }} +{{- $plugins = append $plugins $plugin }} +{{- end }} {{- /* Custom plugins */ -}} {{- if .Values.bifrost.plugins.custom }} {{- range .Values.bifrost.plugins.custom }} @@ -1359,6 +1435,19 @@ Call this template at the beginning of deployment/stateful templates {{- if and .Values.bifrost.plugins.datadog.enabled (hasKey .Values.bifrost.plugins.datadog "version") (gt (int .Values.bifrost.plugins.datadog.version) 32767) }} {{- fail "ERROR: bifrost.plugins.datadog.version must be <= 32767." }} {{- end }} +{{- $ddCfg := (.Values.bifrost.plugins.datadog.config | default dict) }} +{{- if and .Values.bifrost.plugins.datadog.enabled $ddCfg.agentless (not $ddCfg.api_key) }} +{{- fail "ERROR: bifrost.plugins.datadog.config.api_key is required when bifrost.plugins.datadog.config.agentless is true." }} +{{- end }} +{{- if and .Values.bifrost.plugins.bigquery.enabled (hasKey .Values.bifrost.plugins.bigquery "version") (lt (int .Values.bifrost.plugins.bigquery.version) 1) }} +{{- fail "ERROR: bifrost.plugins.bigquery.version must be >= 1. Bump to >1 to force DB-backed plugin config updates." }} +{{- end }} +{{- if and .Values.bifrost.plugins.bigquery.enabled (hasKey .Values.bifrost.plugins.bigquery "version") (gt (int .Values.bifrost.plugins.bigquery.version) 32767) }} +{{- fail "ERROR: bifrost.plugins.bigquery.version must be <= 32767." }} +{{- end }} +{{- if and .Values.bifrost.plugins.bigquery.enabled (not (.Values.bifrost.plugins.bigquery.config | default dict).project_id) }} +{{- fail "ERROR: bifrost.plugins.bigquery.config.project_id is required when the BigQuery plugin is enabled." }} +{{- end }} {{/* Validate semantic cache plugin when enabled */}} {{- if .Values.bifrost.plugins.semanticCache.enabled }} diff --git a/helm-charts/bifrost/values.schema.json b/helm-charts/bifrost/values.schema.json index 2874dc1110..99d6545a4f 100644 --- a/helm-charts/bifrost/values.schema.json +++ b/helm-charts/bifrost/values.schema.json @@ -872,14 +872,27 @@ "enabled": { "type": "boolean" }, + "version": { + "type": "integer", + "minimum": 1 + }, "config": { "type": "object", "properties": { "service_name": { "type": "string" }, + "ml_app": { + "type": "string", + "description": "ML application name for Datadog LLM Observability grouping (defaults to service_name)" + }, "agent_addr": { - "type": "string" + "type": "string", + "description": "Datadog Agent address for APM traces (agent mode only). Supports env.VAR_NAME prefix for environment variable substitution (e.g. env.DD_AGENT_ADDR)" + }, + "dogstatsd_addr": { + "type": "string", + "description": "DogStatsD server address for metrics (agent mode only). Supports env.VAR_NAME prefix for environment variable substitution (e.g. env.DD_DOGSTATSD_ADDR)" }, "env": { "type": "string" @@ -888,15 +901,120 @@ "type": "string" }, "custom_tags": { - "type": "object" + "type": "object", + "description": "Custom tags for Datadog metrics and traces. Values support env.VAR_NAME prefix for environment variable substitution", + "additionalProperties": { + "type": "string" + } + }, + "enable_metrics": { + "type": "boolean", + "description": "Enable Datadog metrics emission (default: true)" }, "enable_traces": { - "type": "boolean" + "type": "boolean", + "description": "Enable Datadog APM traces (default: true)" + }, + "enable_llm_obs": { + "type": "boolean", + "description": "Enable Datadog LLM Observability (default: true)" + }, + "disable_content_logging": { + "type": "boolean", + "description": "When true, sensitive content (prompts, completions, embeddings) is excluded from traces (default: false)" + }, + "agentless": { + "type": "boolean", + "description": "Use agentless mode to send data directly to Datadog APIs instead of a local agent. Requires api_key (default: false)" + }, + "api_key": { + "type": "string", + "description": "Datadog API key, required for agentless mode. Supports env.VAR_NAME prefix for environment variable substitution (e.g. env.DD_API_KEY)" + }, + "site": { + "type": "string", + "description": "Datadog site/region for agentless mode (e.g. datadoghq.com, datadoghq.eu)" + }, + "request_headers": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Request header names to capture as Datadog span tags" + }, + "plugin_span_filter": { + "$ref": "#/$defs/pluginSpanFilter" } } } } }, + "bigquery": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "version": { + "type": "integer", + "minimum": 1 + }, + "config": { + "type": "object", + "properties": { + "project_id": { + "type": "string" + }, + "dataset_id": { + "type": "string" + }, + "table_id": { + "type": "string" + }, + "location": { + "type": "string" + }, + "service_account_key": { + "anyOf": [ + { "type": "string" }, + { + "type": "object", + "properties": { + "value": { "type": "string" }, + "env_var": { "type": "string" }, + "from_env": { "type": "boolean" } + }, + "additionalProperties": false + } + ] + }, + "create_table_if_not_exists": { + "type": "boolean" + }, + "flush_interval_seconds": { + "type": "integer" + }, + "buffer_size": { + "type": "integer" + }, + "custom_labels": { + "type": "object" + }, + "disable_content_logging": { + "type": "boolean" + }, + "request_headers": { + "type": "array", + "items": { "type": "string" } + }, + "plugin_span_filter": { + "$ref": "#/$defs/pluginSpanFilter" + } + }, + "required": ["project_id"] + } + } + }, "custom": { "type": "array", "items": { @@ -1487,6 +1605,75 @@ }, "required": ["id", "name", "scope_kind", "match_type", "pattern", "request_types"] } + }, + "complexityAnalyzerConfig": { + "type": ["object", "null"], + "description": "Runtime configuration for complexity_tier CEL routing. Renders into governance.complexity_analyzer_config in config.json.", + "properties": { + "tier_boundaries": { + "type": "object", + "properties": { + "simple_medium": { + "type": "number", + "exclusiveMinimum": 0, + "exclusiveMaximum": 1 + }, + "medium_complex": { + "type": "number", + "exclusiveMinimum": 0, + "exclusiveMaximum": 1 + }, + "complex_reasoning": { + "type": "number", + "exclusiveMinimum": 0, + "exclusiveMaximum": 1 + } + }, + "required": ["simple_medium", "medium_complex", "complex_reasoning"], + "additionalProperties": false + }, + "keywords": { + "type": "object", + "properties": { + "code_keywords": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + }, + "reasoning_keywords": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + }, + "technical_keywords": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + }, + "simple_keywords": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + } + }, + "required": ["code_keywords", "reasoning_keywords", "technical_keywords", "simple_keywords"], + "additionalProperties": false + } + }, + "required": ["tier_boundaries", "keywords"], + "additionalProperties": false } }, "additionalProperties": false @@ -2593,6 +2780,13 @@ } }, "additionalProperties": false + }, + "key_ids": { + "type": "array", + "description": "Key IDs allowed for this provider config. Use [\"*\"] to allow all keys; empty array or omitted denies all keys. Specific IDs restrict access to those keys only.", + "items": { + "type": "string" + } } }, "required": ["provider_name"], @@ -3391,9 +3585,9 @@ } ] }, - "otelPluginSpanFilter": { + "pluginSpanFilter": { "type": "object", - "description": "Controls which plugin hook spans are exported to the OTEL collector. Omit to export all plugin spans.", + "description": "Controls which plugin hook spans this observability connector exports. Omit to export all plugin spans. Mode \"include\" exports only the listed plugins; mode \"exclude\" exports everything except them.", "properties": { "mode": { "type": "string", @@ -3475,7 +3669,7 @@ "default": false }, "plugin_span_filter": { - "$ref": "#/$defs/otelPluginSpanFilter" + "$ref": "#/$defs/pluginSpanFilter" } }, "allOf": [ @@ -3523,7 +3717,7 @@ "minItems": 1 }, "plugin_span_filter": { - "$ref": "#/$defs/otelPluginSpanFilter" + "$ref": "#/$defs/pluginSpanFilter" }, "enabled": { "type": "boolean", @@ -3602,7 +3796,8 @@ }, "disableAuthOnInference": { "type": "boolean", - "description": "Whether authentication is disabled on inference" + "deprecated": true, + "description": "Deprecated and ignored. Use client.enforceAuthOnInference instead." }, "existingSecret": { "type": "string" @@ -4442,7 +4637,7 @@ }, "key_ids": { "type": "array", - "description": "Key identifiers allowed for this provider config. Use [\"*\"] to allow all keys; empty array denies all (deny-by-default). In Helm values, use provider key names.", + "description": "Key identifiers allowed for this provider config. Use [\"*\"] to allow all keys; empty array denies all (deny-by-default).", "items": { "type": "string" } diff --git a/helm-charts/bifrost/values.yaml b/helm-charts/bifrost/values.yaml index 06d52c9e5f..a437b82944 100644 --- a/helm-charts/bifrost/values.yaml +++ b/helm-charts/bifrost/values.yaml @@ -192,7 +192,6 @@ bifrost: adminUsername: "" adminPassword: "" isEnabled: false - disableAuthOnInference: false # Use existing Kubernetes secret for admin credentials existingSecret: "" usernameKey: "username" @@ -230,7 +229,7 @@ bifrost: # Deprecated: use enforceAuthOnInference instead. enforceGovernanceHeader: false # Require auth (VK, API key, or user token) on inference endpoints. - # When unset, inference endpoints follow authConfig.disableAuthOnInference behavior. + # Open by default in the raw binary; this chart enables enforcement for production. enforceAuthOnInference: true maxRequestBodySizeMb: 100 compat: @@ -515,11 +514,46 @@ bifrost: version: 1 config: service_name: "bifrost" + # Datadog Agent address. Supports env.VAR_NAME references — e.g. set + # agent_addr: "env.DD_AGENT_ADDR" and inject DD_AGENT_ADDR via the + # top-level `env:` (e.g. from status.hostIP for a node-local agent DaemonSet). agent_addr: "localhost:8126" + # dogstatsd_addr: "localhost:8125" # DogStatsD address (supports env.VAR_NAME) env: "" version: "" custom_tags: {} enable_traces: true + # ml_app: "" # ML app name for LLM Observability (defaults to service_name) + # enable_metrics: true + # enable_llm_obs: true + # disable_content_logging: false + # request_headers: [] # Header name patterns (exact or wildcard like "x-custom-*") + # Agentless mode (direct to Datadog API, no local agent): + # agentless: true + # api_key: "env.DD_API_KEY" # Required for agentless mode (supports env.VAR_NAME) + # site: "datadoghq.com" # Datadog site/region (e.g. datadoghq.eu) + # plugin_span_filter: # Optional: filter which plugin hook spans are exported + # mode: "exclude" # "include" or "exclude" + # plugins: ["logging"] + + bigquery: + enabled: false + version: 1 + config: + project_id: "" # GCP project ID (required when enabled) + dataset_id: "bifrost_traces" + table_id: "traces" + location: "US" + # service_account_key: "" # Service account key JSON, or "env.VAR". Omit to use ADC. + create_table_if_not_exists: true + flush_interval_seconds: 5 + buffer_size: 500 + custom_labels: {} + disable_content_logging: false + # request_headers: [] # Header name patterns (exact or wildcard like "x-custom-*") + # plugin_span_filter: # Optional: filter which plugin hook spans are exported + # mode: "exclude" # "include" or "exclude" + # plugins: ["logging"] # Custom/dynamic plugins custom: [] @@ -638,11 +672,20 @@ bifrost: # pattern: "gpt-4o-mini" # request_types: ["chat_completion"] # pricing_patch: "{\"input_cost_per_token\":0.000001,\"output_cost_per_token\":0.000002}" + complexityAnalyzerConfig: null + # tier_boundaries: + # simple_medium: 0.15 + # medium_complex: 0.35 + # complex_reasoning: 0.60 + # keywords: + # code_keywords: ["function", "class", "api", "debug", "deploy"] + # reasoning_keywords: ["step by step", "explain why", "tradeoffs", "root cause analysis"] + # technical_keywords: ["architecture", "kubernetes", "latency", "authentication"] + # simple_keywords: ["hello", "hi", "thanks", "what is", "define"] authConfig: adminUsername: "" adminPassword: "" isEnabled: false - disableAuthOnInference: false # Use existing Kubernetes secret for admin credentials existingSecret: "" usernameKey: "username" diff --git a/plugins/compat/go.mod b/plugins/compat/go.mod index 21702d6933..e394ba4015 100644 --- a/plugins/compat/go.mod +++ b/plugins/compat/go.mod @@ -14,7 +14,7 @@ require ( cloud.google.com/go/auth v0.20.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect - cloud.google.com/go/iam v1.5.3 // indirect + cloud.google.com/go/iam v1.7.0 // indirect cloud.google.com/go/monitoring v1.24.3 // indirect cloud.google.com/go/storage v1.61.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect @@ -26,13 +26,13 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect github.com/andybalholm/brotli v1.2.1 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect - github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect + github.com/aws/aws-sdk-go-v2 v1.41.12 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect github.com/aws/aws-sdk-go-v2/config v1.32.11 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect @@ -44,7 +44,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect - github.com/aws/smithy-go v1.25.1 // indirect + github.com/aws/smithy-go v1.27.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.2 // indirect github.com/bytedance/gopkg v0.1.3 // indirect diff --git a/plugins/compat/go.sum b/plugins/compat/go.sum index 0e3b3e6274..2aa5cb03f9 100644 --- a/plugins/compat/go.sum +++ b/plugins/compat/go.sum @@ -8,8 +8,8 @@ cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIi cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= -cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= +cloud.google.com/go/iam v1.7.0 h1:JD3zh0C6LHl16aCn5Akff0+GELdp1+4hmh6ndoFLl8U= +cloud.google.com/go/iam v1.7.0/go.mod h1:tetWZW1PD/m6vcuY2Zj/aU0eCHNPuxedbnbRTyKXvdY= cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHAjA= cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZCn6hSBtKak= cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8= @@ -45,8 +45,8 @@ github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eT github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= -github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= -github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2 v1.41.12 h1:DIKX2c31ekm9RA2D9FBj1EWXx++9AdAqRw+e78Tq2Ck= +github.com/aws/aws-sdk-go-v2 v1.41.12/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= github.com/aws/aws-sdk-go-v2/config v1.32.11 h1:ftxI5sgz8jZkckuUHXfC/wMUc8u3fG1vQS0plr2F2Zs= @@ -55,10 +55,10 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8TH github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 h1:Xf2j7NdVcUKomlZ4iihOP4AZ3Fzlr8h4yKpXeP+OFPg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28/go.mod h1:O8cDo1dW63jU7ki//kRe1z+tLGcpnD1jrouitsQddDw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 h1:KqIfN9kpkKkcBqBbNpNGTIrXO6ExTUvFKvXkC+YAzVo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28/go.mod h1:uxtQiKvLtNS4iXVsH2McVD/ls8FKN/uUhe1hGxPjrw0= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 h1:clHU5fm//kWS1C2HgtgWxfQbFbx4b6rx+5jzhgX9HrI= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= @@ -81,8 +81,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6f github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= -github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= -github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8= +github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= diff --git a/plugins/compat/main.go b/plugins/compat/main.go index 71e83bb95b..64b7f2303f 100644 --- a/plugins/compat/main.go +++ b/plugins/compat/main.go @@ -89,6 +89,11 @@ func (p *CompatPlugin) HTTPTransportStreamChunkHook(ctx *schemas.BifrostContext, return chunk, nil } +// PreRequestHook implements schemas.LLMPlugin (no-op — required for plugin indexing). +func (p *CompatPlugin) PreRequestHook(_ *schemas.BifrostContext, _ *schemas.BifrostRequest) error { + return nil +} + // PreLLMHook intercepts requests and applies LiteLLM-compatible request normalization. func (p *CompatPlugin) PreLLMHook(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) (*schemas.BifrostRequest, *schemas.LLMPluginShortCircuit, error) { if ctx == nil || req == nil { @@ -186,4 +191,4 @@ func (p *CompatPlugin) markForConversion(ctx *schemas.BifrostContext, provider s if shouldConvert { ctx.SetValue(schemas.BifrostContextKeyChangeRequestType, targetType) } -} \ No newline at end of file +} diff --git a/plugins/governance/blocklist_test.go b/plugins/governance/blocklist_test.go deleted file mode 100644 index 4874b5a1df..0000000000 --- a/plugins/governance/blocklist_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package governance - -import ( - "testing" - - "github.com/maximhq/bifrost/core/schemas" -) - -func TestIsModelBlockedByList(t *testing.T) { - tests := []struct { - name string - blacklist schemas.BlackList - model string - want bool - }{ - { - name: "empty blacklist allows", - blacklist: schemas.BlackList{}, - model: "mistral:latest", - want: false, - }, - { - name: "wildcard blocks all", - blacklist: schemas.BlackList{"*"}, - model: "llama3.2:latest", - want: true, - }, - { - name: "bare blacklist blocks bare request", - blacklist: schemas.BlackList{"mistral:latest"}, - model: "mistral:latest", - want: true, - }, - { - name: "prefixed blacklist blocks bare request", - blacklist: schemas.BlackList{"ollama/mistral:latest"}, - model: "mistral:latest", - want: true, - }, - { - name: "bare blacklist blocks prefixed request", - blacklist: schemas.BlackList{"mistral:latest"}, - model: "ollama/mistral:latest", - want: true, - }, - { - name: "prefixed blacklist blocks prefixed request", - blacklist: schemas.BlackList{"ollama/mistral:latest"}, - model: "ollama/mistral:latest", - want: true, - }, - { - name: "different model is not blocked", - blacklist: schemas.BlackList{"mistral:latest"}, - model: "llama3.2:latest", - want: false, - }, - { - name: "case-insensitive match preserved", - blacklist: schemas.BlackList{"Ollama/Mistral:Latest"}, - model: "ollama/mistral:latest", - want: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := isModelBlockedByList(tt.blacklist, tt.model); got != tt.want { - t.Fatalf("isModelBlockedByList(%v, %q) = %v, want %v", tt.blacklist, tt.model, got, tt.want) - } - }) - } -} diff --git a/plugins/governance/complexity/analyzer.go b/plugins/governance/complexity/analyzer.go new file mode 100644 index 0000000000..c10e5deb88 --- /dev/null +++ b/plugins/governance/complexity/analyzer.go @@ -0,0 +1,203 @@ +package complexity + +import "math" + +// ComplexityAnalyzer computes complexity scores from normalized text input. +// It holds immutable tierBoundaries and matcher configuration after construction, +// so it is safe for concurrent use. +type ComplexityAnalyzer struct { + tierBoundaries TierBoundaries + matcher *compiledKeywordMatcher +} + +// NewComplexityAnalyzer creates an analyzer with built-in defaults. +func NewComplexityAnalyzer() *ComplexityAnalyzer { + return NewComplexityAnalyzerWithConfig(nil) +} + +// NewComplexityAnalyzerWithConfig creates an analyzer with runtime config. +func NewComplexityAnalyzerWithConfig(config *AnalyzerConfig) *ComplexityAnalyzer { + resolved, err := ValidateAndNormalize(config) + if err != nil || resolved == nil { + defaults := DefaultAnalyzerConfig() + resolved = &defaults + } + keywords := mergeEditableKeywordsOntoDefaults(resolved.Keywords) + return &ComplexityAnalyzer{ + tierBoundaries: resolved.TierBoundaries, + matcher: newCompiledKeywordMatcher(keywords), + } +} + +// Analyze computes complexity scores from the normalized input. +func (a *ComplexityAnalyzer) Analyze(input ComplexityInput) *ComplexityResult { + // Select scan mask based on whether conversation history is present. + lastScanMask := lastTextBaseScanMask + if len(input.PriorUserTexts) > 0 { + lastScanMask = lastTextFullScanMask + } + + // Extract lexical signals from last user message and system prompt. + lastSignals := a.matcher.analyzeText(input.LastUserText, lastScanMask) + systemSignals := a.matcher.analyzeText(input.SystemText, systemTextScanMask) + + // Score primary message signals. + userCodeScore := scoreCount(lastSignals.codeCount, 3) + reasoningScore := scoreCount(lastSignals.reasoningCount, 2) + userTechnicalScore := scoreCount(lastSignals.technicalCount, 3) + userSimpleScore := scoreCount(lastSignals.simpleCount, 2) + outputScore := scoreOutputComplexity(lastSignals) + tokenScore := scoreTokenCount(lastSignals.wordCount) + + // System prompt provides soft lexical context for code/technical/simple signals, + // but never drives reasoning override, token count, or output complexity. + systemCodeScore := scoreCount(systemSignals.codeCount, 3) + systemTechnicalScore := scoreCount(systemSignals.technicalCount, 3) + systemSimpleScore := scoreCount(systemSignals.simpleCount, 2) + + codeScore := clamp(userCodeScore+(systemCodeScore*systemPromptAssistFactor), 0.0, 1.0) + technicalScore := clamp(userTechnicalScore+(systemTechnicalScore*systemPromptAssistFactor), 0.0, 1.0) + simpleScore := clamp(userSimpleScore+(systemSimpleScore*systemPromptAssistFactor), 0.0, 1.0) + + // Conditional simple dampener: only apply full dampener on short, low-signal asks. + wordCount := lastSignals.wordCount + effectiveSimpleWeight := simpleWeight + signalCount := 0 + if userCodeScore >= 0.3 { + signalCount++ + } + if userTechnicalScore >= 0.3 { + signalCount++ + } + if reasoningScore >= 0.3 { + signalCount++ + } + if lastSignals.simpleCount > 0 && (wordCount >= 30 || signalCount >= 2) { + effectiveSimpleWeight = 0.01 + } + + codeContribution := codeScore * codeWeight + reasoningContribution := reasoningScore * reasoningWeight + technicalContribution := technicalScore * technicalWeight + simplePenalty := -(simpleScore * effectiveSimpleWeight) + tokenContribution := tokenScore * tokenCountWeight + + // Weighted sum for last message (output complexity applied separately as a score floor). + lastMsgScore := codeContribution + + reasoningContribution + + technicalContribution + + simplePenalty + + tokenContribution + lastMsgScore = clamp(lastMsgScore, 0.0, 1.0) + + // Conversation context blending (prior user turns only). + var blended float64 + var convScore float64 + if len(input.PriorUserTexts) > 0 { + convScore = a.scoreConversationContext(input.PriorUserTexts) + lastWeight := defaultLastMessageBlendWeight + contextWeight := defaultConversationBlendWeight + if isReferentialFollowup(lastSignals, lastMsgScore, convScore, wordCount) { + lastWeight = referentialLastMessageBlendWeight + contextWeight = referentialConversationBlendWeight + } + + weightedBlend := (lastMsgScore * lastWeight) + (convScore * contextWeight) + blended = math.Max(lastMsgScore, weightedBlend) + } else { + blended = lastMsgScore + } + + // Output complexity as a score floor: strong output signals set a minimum score. + outputFloorMinScore := 0.0 + if outputScore > 0.5 { + outputFloorMinScore = outputScore * 0.5 + if blended < outputFloorMinScore { + blended = outputFloorMinScore + } + } + + finalScore := clamp(blended, 0.0, 1.0) + + // Tier classification with reasoning override. + strongCount := lastSignals.strongReasoningCount + tier := a.classifyTier(finalScore) + if strongCount >= 2 { + tier = TierReasoning + } else if strongCount >= 1 && (userCodeScore > 0.5 || userTechnicalScore > 0.5) { + tier = TierReasoning + } + + return &ComplexityResult{ + Score: finalScore, + Tier: tier, + WordCount: wordCount, + } +} + +func (a *ComplexityAnalyzer) scoreConversationContext(priorUserTexts []string) float64 { + if len(priorUserTexts) == 0 { + return 0.0 + } + + texts := priorUserTexts + if len(texts) > 10 { + texts = texts[len(texts)-10:] + } + + var weightedTotal float64 + var totalWeight float64 + lastIdx := len(texts) - 1 + for idx, text := range texts { + signals := a.matcher.analyzeText(text, contextTextScanMask) + code := scoreCount(signals.codeCount, 3) + tech := scoreCount(signals.technicalCount, 3) + reasoning := scoreCount(signals.reasoningCount, 2) + msgScore := (code*codeWeight + tech*technicalWeight + reasoning*reasoningWeight) / + (codeWeight + technicalWeight + reasoningWeight) + weight := 1.0 + if lastIdx > 0 { + weight = 1.0 + (2.0 * float64(idx) / float64(lastIdx)) + } + weightedTotal += msgScore * weight + totalWeight += weight + } + + if totalWeight == 0 { + return 0.0 + } + + return math.Min(1.0, weightedTotal/totalWeight) +} + +func isReferentialFollowup(signals textSignalCounts, lastMsgScore, convScore float64, wordCount int) bool { + if wordCount == 0 || wordCount > referentialMaxWordCount { + return false + } + if lastMsgScore >= referentialMaxStandaloneScore || convScore < referentialMinContextScore { + return false + } + if signals.taskShiftCount > 0 { + return false + } + if signals.referentialPhraseCount > 0 { + return true + } + + hasReference := signals.referentialReferenceCount > 0 + hasAction := signals.referentialActionCount > 0 + return hasReference && hasAction +} + +func (a *ComplexityAnalyzer) classifyTier(score float64) string { + switch { + case score < a.tierBoundaries.SimpleMedium: + return TierSimple + case score < a.tierBoundaries.MediumComplex: + return TierMedium + case score < a.tierBoundaries.ComplexReasoning: + return TierComplex + default: + return TierReasoning + } +} diff --git a/plugins/governance/complexity/analyzer_test.go b/plugins/governance/complexity/analyzer_test.go new file mode 100644 index 0000000000..7fea167c90 --- /dev/null +++ b/plugins/governance/complexity/analyzer_test.go @@ -0,0 +1,785 @@ +package complexity + +import ( + "strings" + "testing" +) + +func TestAnalyze_Simple(t *testing.T) { + a := NewComplexityAnalyzer() + + result := a.Analyze(ComplexityInput{ + LastUserText: "What is 2+2?", + }) + + if result.Tier != "SIMPLE" { + t.Errorf("expected SIMPLE tier for 'What is 2+2?', got %s (score=%.3f)", result.Tier, result.Score) + } +} + +func TestAnalyze_CustomTierBoundaries(t *testing.T) { + defaultAnalyzer := NewComplexityAnalyzer() + cfg := DefaultAnalyzerConfig() + cfg.TierBoundaries = TierBoundaries{ + SimpleMedium: 0.05, + MediumComplex: 0.10, + ComplexReasoning: 0.20, + } + customAnalyzer := NewComplexityAnalyzerWithConfig(&cfg) + + if got := defaultAnalyzer.classifyTier(0.18); got != TierMedium { + t.Fatalf("default boundary classified 0.18 as %s, want %s", got, TierMedium) + } + if got := customAnalyzer.classifyTier(0.18); got != TierComplex { + t.Fatalf("custom boundary classified 0.18 as %s, want %s", got, TierComplex) + } +} + +func TestAnalyze_CustomReasoningKeywordsAffectOverride(t *testing.T) { + cfg := DefaultAnalyzerConfig() + cfg.Keywords.ReasoningKeywords = []string{"deepmagic"} + a := NewComplexityAnalyzerWithConfig(&cfg) + + result := a.Analyze(ComplexityInput{ + LastUserText: "deepmagic api function", + }) + + if result.Tier != TierReasoning { + t.Fatalf("expected custom reasoning keyword to promote tier to %s, got %s (score=%.3f)", TierReasoning, result.Tier, result.Score) + } +} + +func TestAnalyze_Hello(t *testing.T) { + a := NewComplexityAnalyzer() + + result := a.Analyze(ComplexityInput{ + LastUserText: "Hello, how are you?", + }) + + if result.Tier != "SIMPLE" { + t.Errorf("expected SIMPLE tier for greeting, got %s (score=%.3f)", result.Tier, result.Score) + } +} + +func TestAnalyze_CodeRequest(t *testing.T) { + a := NewComplexityAnalyzer() + + result := a.Analyze(ComplexityInput{ + LastUserText: "Write a Python quicksort function that handles arrays with duplicate elements", + }) + + if result.Tier != "MEDIUM" && result.Tier != "COMPLEX" { + t.Errorf("expected MEDIUM or COMPLEX tier for code request, got %s (score=%.3f)", result.Tier, result.Score) + } +} + +func TestAnalyze_Complex(t *testing.T) { + a := NewComplexityAnalyzer() + + result := a.Analyze(ComplexityInput{ + LastUserText: "Design a distributed authentication system using Kubernetes with encryption and load balancer", + }) + + if result.Tier != "COMPLEX" && result.Tier != "REASONING" { + t.Errorf("expected COMPLEX or REASONING tier for architecture request, got %s (score=%.3f)", result.Tier, result.Score) + } +} + +func TestAnalyze_Reasoning(t *testing.T) { + a := NewComplexityAnalyzer() + + result := a.Analyze(ComplexityInput{ + LastUserText: "Think step by step through the tradeoffs of this ML architecture and explain why one approach is better", + }) + + if result.Tier != "REASONING" { + t.Errorf("expected REASONING tier for deep reasoning request, got %s (score=%.3f)", result.Tier, result.Score) + } +} + +func TestAnalyze_OutputComplexity(t *testing.T) { + a := NewComplexityAnalyzer() + + result := a.Analyze(ComplexityInput{ + LastUserText: "List every AWS service and explain each one with examples", + }) + + if result.Tier == "SIMPLE" { + t.Errorf("expected non-SIMPLE tier for output-heavy request, got %s (score=%.3f)", result.Tier, result.Score) + } +} + +func TestAnalyze_ConversationContext(t *testing.T) { + a := NewComplexityAnalyzer() + + // Short follow-up with no context stays SIMPLE. + noCtx := a.Analyze(ComplexityInput{ + LastUserText: "Why?", + }) + + // Same follow-up with technical conversation history gets a higher score. + withCtx := a.Analyze(ComplexityInput{ + LastUserText: "Why?", + PriorUserTexts: []string{ + "How does the distributed authentication system handle encryption?", + "What about the kubernetes infrastructure for microservices?", + "Can you explain the concurrency model and mutex usage?", + }, + }) + + if withCtx.Score <= noCtx.Score { + t.Errorf("expected conversation context to raise score: noCtx=%.3f, withCtx=%.3f", + noCtx.Score, withCtx.Score) + } +} + +func TestAnalyze_ConversationContextDoesNotDiluteStrongLastMessage(t *testing.T) { + a := NewComplexityAnalyzer() + + lastTurnOnly := a.Analyze(ComplexityInput{ + LastUserText: "Design the target architecture for migrating our monolith checkout service to an event-driven system. Cover the event schema, consumer topology, idempotency strategy, and a phased data migration plan that maintains zero downtime.", + }) + + withCtx := a.Analyze(ComplexityInput{ + LastUserText: "Design the target architecture for migrating our monolith checkout service to an event-driven system. Cover the event schema, consumer topology, idempotency strategy, and a phased data migration plan that maintains zero downtime.", + PriorUserTexts: []string{ + "We're hitting scaling limits with our monolithic checkout service.", + "Current throughput is 500 TPS but we need 5,000 TPS by Q3.", + "We're considering event sourcing but worried about operational complexity.", + }, + }) + + if withCtx.Score < lastTurnOnly.Score { + t.Errorf("expected context-aware score to preserve or raise final score: lastOnly=%.3f, withCtx=%.3f", + lastTurnOnly.Score, withCtx.Score) + } +} + +func TestAnalyze_ReferentialFollowupLiftsShortTechnicalContinuation(t *testing.T) { + a := NewComplexityAnalyzer() + + result := a.Analyze(ComplexityInput{ + LastUserText: "do it", + PriorUserTexts: []string{ + "We need to refactor the retry middleware so only 429 and 408 retry.", + "Move fallback selection after request classification and keep the behavior change explicit in the PR.", + "Update the Go tests for the CEL routing rules and the governance plugin.", + }, + }) + + if result.Tier == "SIMPLE" { + t.Fatalf("expected short referential follow-up to lift above SIMPLE, got %s (score=%.3f)", result.Tier, result.Score) + } + if result.Score < simpleMediumBoundary { + t.Fatalf("expected score above SIMPLE threshold, got %.3f", result.Score) + } +} + +func TestAnalyze_ReferentialFollowupRequiresRealContext(t *testing.T) { + a := NewComplexityAnalyzer() + + result := a.Analyze(ComplexityInput{ + LastUserText: "do it", + }) + + if result.Tier != "SIMPLE" { + t.Fatalf("expected SIMPLE tier without prior context, got %s (score=%.3f)", result.Tier, result.Score) + } +} + +func TestAnalyze_TaskShiftFollowupDoesNotUseReferentialLift(t *testing.T) { + a := NewComplexityAnalyzer() + + result := a.Analyze(ComplexityInput{ + LastUserText: "translate this to spanish", + PriorUserTexts: []string{ + "We need to debug the Kubernetes deployment and fix the authentication middleware.", + "The RBAC mapping for SAML tenants is failing after the migration.", + }, + }) + + if result.Score >= mediumComplexBoundary { + t.Fatalf("expected task-shift request to stay below COMPLEX threshold, got %.3f", result.Score) + } +} + +func TestAnalyze_LimitingTaskShiftDoesNotUseReferentialLift(t *testing.T) { + a := NewComplexityAnalyzer() + + result := a.Analyze(ComplexityInput{ + LastUserText: "summarize it in one sentence", + PriorUserTexts: []string{ + "Design a multi-tenant billing ledger with metering, proration, credits, and invoice generation.", + "Include the data model and monthly aggregation flow.", + }, + }) + + if result.Score >= mediumComplexBoundary { + t.Fatalf("expected limiting summary request to stay below COMPLEX threshold, got %.3f", result.Score) + } +} + +func TestAnalyze_RecentContextOutweighsOlderContext(t *testing.T) { + a := NewComplexityAnalyzer() + + recentTech := a.Analyze(ComplexityInput{ + LastUserText: "do it", + PriorUserTexts: []string{ + "Hello there.", + "Thanks.", + "Design a distributed authentication system with RBAC, OIDC, and regional failover.", + }, + }) + + olderTech := a.Analyze(ComplexityInput{ + LastUserText: "do it", + PriorUserTexts: []string{ + "Design a distributed authentication system with RBAC, OIDC, and regional failover.", + "Hello there.", + "Thanks.", + }, + }) + + if recentTech.Score <= olderTech.Score { + t.Fatalf("expected more recent technical context to matter more: recent=%.3f older=%.3f", + recentTech.Score, olderTech.Score) + } +} + +func TestAnalyze_SystemPromptBoost(t *testing.T) { + a := NewComplexityAnalyzer() + + base := a.Analyze(ComplexityInput{ + LastUserText: "Review this code for issues", + }) + + boosted := a.Analyze(ComplexityInput{ + LastUserText: "Review this code for issues", + SystemText: "You are a security engineer responsible for RBAC, audit log reviews, and OIDC policy.", + }) + + if boosted.Score <= base.Score { + t.Errorf("expected system prompt to boost score: base=%.3f, boosted=%.3f", + base.Score, boosted.Score) + } +} + +func TestAnalyze_SystemPromptDampener(t *testing.T) { + a := NewComplexityAnalyzer() + + base := a.Analyze(ComplexityInput{ + LastUserText: "Explain how databases work", + }) + + dampened := a.Analyze(ComplexityInput{ + LastUserText: "Explain how databases work", + SystemText: "You are a beginner tutor. Keep answers simple, brief, and concise.", + }) + + if dampened.Score >= base.Score { + t.Errorf("expected system prompt to dampen score: base=%.3f, dampened=%.3f", + base.Score, dampened.Score) + } +} + +func TestAnalyze_SystemPromptLexicalAssistDoesNotOverPromoteSimpleWebhook(t *testing.T) { + a := NewComplexityAnalyzer() + + result := a.Analyze(ComplexityInput{ + LastUserText: "What is a webhook?", + SystemText: "You are responsible for RBAC, audit log controls, and OIDC integration policy.", + }) + + if result.Tier != "SIMPLE" { + t.Errorf("expected SIMPLE tier for webhook definition with technical system prompt, got %s (score=%.3f)", + result.Tier, result.Score) + } +} + +func TestAnalyze_EmptyInput(t *testing.T) { + a := NewComplexityAnalyzer() + + result := a.Analyze(ComplexityInput{}) + + if result.Tier != "SIMPLE" { + t.Errorf("expected SIMPLE tier for empty input, got %s", result.Tier) + } + if result.Score != 0.0 { + t.Errorf("expected 0.0 score for empty input, got %.3f", result.Score) + } +} + +func TestAnalyze_ReasoningOverrideNotTooEager(t *testing.T) { + a := NewComplexityAnalyzer() + + // Two weak reasoning markers should NOT force REASONING + result := a.Analyze(ComplexityInput{ + LastUserText: "Why does React re-render, and what if I use useMemo?", + }) + + if result.Tier == "REASONING" { + t.Errorf("expected non-REASONING tier for casual question with weak markers, got %s (score=%.3f)", + result.Tier, result.Score) + } +} + +func TestAnalyze_SimpleDampenerConditional(t *testing.T) { + a := NewComplexityAnalyzer() + + // "What is" + technical term should not be over-dampened + result := a.Analyze(ComplexityInput{ + LastUserText: "What is eventual consistency in distributed systems with sharding?", + }) + + if result.Tier == "SIMPLE" { + t.Errorf("expected non-SIMPLE tier for technical 'what is' question, got %s (score=%.3f)", + result.Tier, result.Score) + } +} + +func TestAnalyze_AccessVsRefreshTokens(t *testing.T) { + a := NewComplexityAnalyzer() + + result := a.Analyze(ComplexityInput{ + LastUserText: "Explain the difference between an access token and a refresh token. When would you use short-lived vs long-lived tokens?", + }) + + if result.Tier == "SIMPLE" { + t.Errorf("expected MEDIUM or higher tier for token lifecycle question, got %s (score=%.3f)", + result.Tier, result.Score) + } +} + +func TestAnalyze_OutageCustomerCommunication(t *testing.T) { + a := NewComplexityAnalyzer() + + result := a.Analyze(ComplexityInput{ + LastUserText: "Draft a short outage notification email for our enterprise customers. Our payment processing was down for 23 minutes this morning between 09:12 and 09:35 UTC. No transactions were lost but some were delayed.", + SystemText: "You are a customer success manager for a B2B SaaS platform. You help draft professional and empathetic communications to enterprise customers.", + }) + + if result.Tier == "SIMPLE" { + t.Errorf("expected MEDIUM or higher tier for outage communication prompt, got %s (score=%.3f)", + result.Tier, result.Score) + } +} + +func TestAnalyze_MultiTenantSSOArchitecture(t *testing.T) { + a := NewComplexityAnalyzer() + + result := a.Analyze(ComplexityInput{ + LastUserText: "Design a multi-tenant authentication service for a SaaS platform on Kubernetes. Requirements: RBAC with custom roles per tenant, audit logging for all auth events, regional failover across two AWS regions, and support for both SAML 2.0 and OIDC enterprise SSO. Include the data model and the request flow for a login.", + }) + + if result.Tier != "COMPLEX" && result.Tier != "REASONING" { + t.Errorf("expected COMPLEX or REASONING tier for multi-tenant SSO architecture prompt, got %s (score=%.3f)", + result.Tier, result.Score) + } +} + +func TestAnalyze_PostIncidentReconstruction(t *testing.T) { + a := NewComplexityAnalyzer() + + result := a.Analyze(ComplexityInput{ + LastUserText: "Given this partial timeline with a 15-minute telemetry gap, reconstruct the most likely sequence of failures. Why did connection pool exhaustion happen? Why didn't the ConfigMap fix work, and what should the on-call have done instead? What might have happened during the metrics blackout that we can't directly observe? Identify the weakest assumptions in your reconstruction and flag what we'd need to verify.", + PriorUserTexts: []string{ + "The outage lasted 47 minutes and affected all US-East customers. Revenue impact was approximately $180,000.", + "Timeline: 14:03 - alerts fired for elevated 5xx rates on the API gateway. 14:15 - identified database connection pool exhaustion on the primary Postgres cluster.", + "At 14:22 the on-call attempted to scale up the connection pool via a ConfigMap change, but the change didn't take effect because our pods require a restart to pick up ConfigMap changes.", + }, + SystemText: "You are leading the post-incident review for a major production outage at a multi-region SaaS company.", + }) + + if result.Tier != "COMPLEX" && result.Tier != "REASONING" { + t.Errorf("expected COMPLEX or REASONING tier for post-incident reconstruction, got %s (score=%.3f)", + result.Tier, result.Score) + } +} + +func TestAnalyze_CodingFollowupsWithTechnicalContext(t *testing.T) { + a := NewComplexityAnalyzer() + + tests := []struct { + name string + lastUserText string + prior []string + }{ + { + name: "explain_changes_for_pr", + lastUserText: "Can you explain the changes in plain English for the PR description and call out the behavior change?", + prior: []string{ + "I'm working on a Go gateway and just changed our retry middleware so it stops retrying most 4xx responses.", + "I added an allowlist so only 429 and 408 still retry, and I moved the fallback logic after the classification step.", + }, + }, + { + name: "summarize_refactor", + lastUserText: "Can you summarize the refactor for the PR in a few bullets and highlight the behavior changes?", + prior: []string{ + "I split our request parsing code into a transport-specific extractor layer and a pure analyzer package so the heuristics don't depend on raw HTTP payload shapes.", + "I also moved provider-shape branching into the governance plugin, added tests for OpenAI Responses input_text, and stopped unsupported requests from defaulting to SIMPLE.", + }, + }, + { + name: "write_commit_message", + lastUserText: "Can you write the commit message for this patch?", + prior: []string{ + "I changed the retry middleware so it stops retrying most 4xx responses.", + "I added an allowlist for retryable statuses and moved fallback selection after the classification step.", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := a.Analyze(ComplexityInput{ + LastUserText: tt.lastUserText, + PriorUserTexts: tt.prior, + }) + + if result.Tier == "SIMPLE" { + t.Errorf("expected MEDIUM or higher tier for coding follow-up, got %s (score=%.3f)", + result.Tier, result.Score) + } + }) + } +} + +func TestAnalyze_GitHubActionsWorkflow(t *testing.T) { + a := NewComplexityAnalyzer() + + result := a.Analyze(ComplexityInput{ + LastUserText: "Write a GitHub Actions workflow that detects which services changed in a PR and only runs the tests for those services.", + PriorUserTexts: []string{ + "I'm setting up CI/CD for the first time for our monorepo.", + "We use GitHub Actions and each service has its own go.mod and test suite.", + }, + }) + + if result.Tier == "SIMPLE" { + t.Errorf("expected MEDIUM or higher tier for GitHub Actions workflow request, got %s (score=%.3f)", + result.Tier, result.Score) + } +} + +func TestAnalyze_BillingLedgerPipeline(t *testing.T) { + a := NewComplexityAnalyzer() + + result := a.Analyze(ComplexityInput{ + LastUserText: "Design a usage-based billing pipeline covering metering, aggregation, proration, credits, dunning, and invoice generation. Include the data model for the ledger and the sequence flow for generating a monthly invoice.", + SystemText: "You are a staff engineer for a B2B SaaS billing platform.", + }) + + if result.Tier != "COMPLEX" && result.Tier != "REASONING" { + t.Errorf("expected COMPLEX or REASONING tier for billing ledger pipeline prompt, got %s (score=%.3f)", + result.Tier, result.Score) + } +} + +func TestAnalyze_VectorDatabaseTradeoffRecommendation(t *testing.T) { + a := NewComplexityAnalyzer() + + result := a.Analyze(ComplexityInput{ + LastUserText: "Compare self-hosted Qdrant vs managed Pinecone for a hybrid search system serving 1,000 QPS with 50M vectors. We're in a regulated industry - no data can leave our VPC, and we need SOC 2 attestation for all data stores. Weigh the tradeoffs around data residency compliance, operational burden for a 4-person infra team, query latency at scale, cost scaling characteristics, and disaster recovery options. Recommend one and explain your reasoning.", + }) + + if result.Tier != "REASONING" { + t.Errorf("expected REASONING tier for vector database tradeoff recommendation, got %s (score=%.3f)", + result.Tier, result.Score) + } +} + +func TestIsReferentialFollowup_GuardBranches(t *testing.T) { + tests := []struct { + name string + lastText string + lastMsgScore float64 + convScore float64 + wordCount int + expected bool + }{ + {"phrase_match_ok", "do it", 0.05, 0.30, 2, true}, + {"phrase_match_at_word_cap", "do it now please right away", 0.05, 0.30, 6, true}, + {"phrase_match_over_word_cap", "do it now please right away ok", 0.05, 0.30, 7, false}, + {"phrase_match_zero_words", "", 0.0, 0.30, 0, false}, + {"phrase_match_score_at_threshold", "do it", 0.15, 0.30, 2, false}, + {"phrase_match_score_just_below_threshold", "do it", 0.149, 0.30, 2, true}, + {"phrase_match_conv_just_below_threshold", "do it", 0.05, 0.199, 2, false}, + {"phrase_match_conv_at_threshold", "do it", 0.05, 0.20, 2, true}, + {"task_shift_blocks_phrase_match", "translate it", 0.05, 0.30, 2, false}, + {"task_shift_blocks_summarize", "summarize it", 0.05, 0.30, 2, false}, + {"task_shift_one_sentence_blocks", "rewrite it in one sentence", 0.05, 0.30, 5, false}, + {"multi_signal_fix_it", "fix it", 0.05, 0.30, 2, true}, + {"multi_signal_make_it_shorter", "make it shorter", 0.05, 0.30, 3, true}, + {"multi_signal_rewrite_it", "rewrite it", 0.05, 0.30, 2, true}, + {"multi_signal_use_that", "use that", 0.05, 0.30, 2, true}, + {"multi_signal_answer_previous", "answer the previous question", 0.05, 0.30, 4, true}, + {"action_only_no_deictic", "fix the race condition", 0.05, 0.30, 4, false}, + {"deictic_only_no_action", "this is great", 0.05, 0.30, 3, false}, + {"unrelated_short_text", "hello there friend", 0.05, 0.30, 3, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + matcher := newCompiledKeywordMatcher(defaultFullKeywordConfig()) + signals := matcher.analyzeText(tt.lastText, lastTextFullScanMask) + got := isReferentialFollowup(signals, tt.lastMsgScore, tt.convScore, tt.wordCount) + if got != tt.expected { + t.Errorf("isReferentialFollowup(%q, last=%.3f, conv=%.3f, words=%d) = %v, want %v", + tt.lastText, tt.lastMsgScore, tt.convScore, tt.wordCount, got, tt.expected) + } + }) + } +} + +func TestAnalyze_ReferentialMultiSignalDetection(t *testing.T) { + a := NewComplexityAnalyzer() + + techPriors := []string{ + "We need to refactor the retry middleware so only 429 and 408 retry.", + "Move fallback selection after request classification and keep the behavior change explicit in the PR.", + "Update the Go tests for the CEL routing rules and the governance plugin.", + } + + tests := []struct { + name string + lastText string + }{ + {"fix_it", "fix it"}, + {"make_it_shorter", "make it shorter"}, + {"rewrite_it", "rewrite it"}, + {"do_this", "do this"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := a.Analyze(ComplexityInput{ + LastUserText: tt.lastText, + PriorUserTexts: techPriors, + }) + if result.Tier == "SIMPLE" { + t.Fatalf("expected lift above SIMPLE for %q, got %s (score=%.3f)", + tt.lastText, result.Tier, result.Score) + } + }) + } +} + +func TestAnalyze_ReferentialPhraseDoesNotHijackStrongAsk(t *testing.T) { + a := NewComplexityAnalyzer() + + result := a.Analyze(ComplexityInput{ + LastUserText: "use option 2 to design the distributed consensus algorithm with kubernetes and rbac", + PriorUserTexts: []string{ + "We need to refactor the retry middleware so only 429 and 408 retry.", + }, + }) + + if result.Tier == "SIMPLE" { + t.Fatalf("expected high-signal message to stay above SIMPLE despite referential phrase, got %s (score=%.3f)", + result.Tier, result.Score) + } +} + +func TestAnalyze_RegressionAnchors(t *testing.T) { + a := NewComplexityAnalyzer() + + techPriors := []string{ + "We need to refactor the retry middleware so only 429 and 408 retry.", + "Move fallback selection after request classification and keep the behavior change explicit in the PR.", + "Update the Go tests for the CEL routing rules and the governance plugin.", + } + + tests := []struct { + name string + lastText string + priors []string + minTier string // tier must be at least this rank (or empty for "any") + maxTier string // tier must be at most this rank (or empty for "any") + mustNotEqualTiers []string + }{ + { + name: "do_it_after_tech_thread_lifts", + lastText: "do it", + priors: techPriors, + mustNotEqualTiers: []string{"SIMPLE"}, + }, + { + name: "try_again_after_tech_thread_lifts", + lastText: "try again", + priors: techPriors, + mustNotEqualTiers: []string{"SIMPLE"}, + }, + { + name: "translate_after_tech_thread_stays_simple", + lastText: "translate this to spanish", + priors: techPriors, + maxTier: "MEDIUM", + }, + { + name: "summarize_after_tech_thread_stays_simple", + lastText: "summarize it in one sentence", + priors: techPriors, + maxTier: "MEDIUM", + }, + { + name: "do_it_with_empty_priors_stays_simple", + lastText: "do it", + priors: nil, + maxTier: "SIMPLE", + }, + { + name: "strong_arch_ask_with_smalltalk_priors_stays_strong", + lastText: "Design a fault-tolerant distributed consensus algorithm with leader election, log replication, and snapshotting; weigh the tradeoffs between Raft and Paxos and recommend a design under the constraint of WAN replication.", + priors: []string{"hi", "thanks", "ok"}, + minTier: "COMPLEX", + }, + { + name: "translate_no_priors_stays_simple", + lastText: "translate this to spanish", + priors: nil, + maxTier: "SIMPLE", + }, + } + + tierRank := map[string]int{"SIMPLE": 0, "MEDIUM": 1, "COMPLEX": 2, "REASONING": 3} + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := a.Analyze(ComplexityInput{ + LastUserText: tt.lastText, + PriorUserTexts: tt.priors, + }) + + if tt.minTier != "" && tierRank[result.Tier] < tierRank[tt.minTier] { + t.Errorf("tier=%s, expected at least %s (score=%.3f)", result.Tier, tt.minTier, result.Score) + } + if tt.maxTier != "" && tierRank[result.Tier] > tierRank[tt.maxTier] { + t.Errorf("tier=%s, expected at most %s (score=%.3f)", result.Tier, tt.maxTier, result.Score) + } + for _, banned := range tt.mustNotEqualTiers { + if result.Tier == banned { + t.Errorf("tier=%s, must not equal %s (score=%.3f)", result.Tier, banned, result.Score) + } + } + }) + } +} + +func TestScoreConversationContext_RecencyDecay(t *testing.T) { + a := NewComplexityAnalyzer() + + // Empty list returns 0 without dividing by zero. + if got := a.scoreConversationContext(nil); got != 0.0 { + t.Errorf("empty priors should return 0.0, got %.3f", got) + } + + // Single prior message: lastIdx == 0, weight branch is the uniform fallback. + // Should not panic, should return a positive score for technical content. + single := a.scoreConversationContext([]string{ + "Design a distributed authentication system with kubernetes, rbac, and oidc.", + }) + if single <= 0 { + t.Errorf("expected positive score for single technical prior, got %.3f", single) + } + + // Linear decay: a strong technical message at the END of the list should + // produce a meaningfully higher score than the same message at the START. + recent := a.scoreConversationContext([]string{ + "hello", + "thanks", + "Design a distributed authentication system with kubernetes, rbac, and oidc.", + }) + older := a.scoreConversationContext([]string{ + "Design a distributed authentication system with kubernetes, rbac, and oidc.", + "hello", + "thanks", + }) + if recent <= older { + t.Errorf("expected recent strong message to score higher than older one: recent=%.3f older=%.3f", + recent, older) + } +} + +func TestContainsWord(t *testing.T) { + tests := []struct { + text string + word string + expected bool + }{ + {"write a function", "function", true}, + {"classification problem", "class", false}, // word boundary + {"the class is good", "class", true}, + {"debug the code", "debug", true}, + {"debug", "debug", true}, + {"nodebug", "debug", false}, + {"la securite est importante", "securite", true}, + {"la sécurité est importante", "sécurité", true}, + {"sécuritétest", "sécurité", false}, + {"", "test", false}, + {"write a function", "", false}, + } + + for _, tt := range tests { + got := containsWord(tt.text, tt.word) + if got != tt.expected { + t.Errorf("containsWord(%q, %q) = %v, want %v", tt.text, tt.word, got, tt.expected) + } + } +} + +func TestCountWordsNoAllocMatchesStringsFields(t *testing.T) { + tests := []string{ + "", + "hello world", + " multiple spaces here ", + "line one\nline two\tline three", + "unicode\u00a0space separated words", + } + + for _, text := range tests { + got := countWordsNoAlloc(text) + want := len(strings.Fields(text)) + if got != want { + t.Errorf("countWordsNoAlloc(%q) = %d, want %d", text, got, want) + } + } +} + +func TestKeywordMatchModeFor(t *testing.T) { + tests := []struct { + keyword string + want keywordMatchMode + }{ + {"function", matchModeWholeWord}, + {"sécurité", matchModeWholeWord}, + {"ci/cd", matchModeBoundarySubstring}, + {"root cause", matchModePlainSubstring}, + } + + for _, tt := range tests { + if got := keywordMatchModeFor(tt.keyword); got != tt.want { + t.Errorf("keywordMatchModeFor(%q) = %v, want %v", tt.keyword, got, tt.want) + } + } +} + +func TestBuildWordPresenceSet_UnicodeWords(t *testing.T) { + words := buildWordPresenceSet("la sécurité du réseau protège les données") + + if _, ok := words["sécurité"]; !ok { + t.Fatalf("expected unicode word to be preserved in presence set") + } + if _, ok := words["réseau"]; !ok { + t.Fatalf("expected second unicode word to be preserved in presence set") + } +} + +func TestAnalyze_PunctuatedKeywordStillMatches(t *testing.T) { + a := NewComplexityAnalyzer() + + signals := a.matcher.analyzeText("Please review our CI/CD pipeline and retry middleware behavior.", lastTextBaseScanMask) + if signals.codeCount == 0 { + t.Fatalf("expected punctuated keyword path to match code signals") + } +} diff --git a/plugins/governance/complexity/config.go b/plugins/governance/complexity/config.go new file mode 100644 index 0000000000..57bda9eb77 --- /dev/null +++ b/plugins/governance/complexity/config.go @@ -0,0 +1,140 @@ +// Package complexity provides request-complexity scoring for governance routing. +package complexity + +import "github.com/maximhq/bifrost/framework/configstore" + +// ComplexityInput is the normalized input for the analyzer. +// The caller is responsible for extracting text from request payloads. +type ComplexityInput struct { + LastUserText string // last user message text + PriorUserTexts []string // previous user message texts (up to 10) + SystemText string // concatenated system/developer prompt text +} + +// ComplexityResult holds the computed complexity scores and tier classification. +type ComplexityResult struct { + Score float64 + Tier string + WordCount int +} + +const ( + TierSimple = "SIMPLE" + TierMedium = "MEDIUM" + TierComplex = "COMPLEX" + TierReasoning = "REASONING" +) + +const ( + simpleMediumBoundary = 0.15 + mediumComplexBoundary = 0.35 + complexReasoningBoundary = 0.60 +) + +// TierBoundaries defines the score thresholds for tier classification. +type TierBoundaries = configstore.ComplexityTierBoundaries + +// EditableKeywordConfig is the user-facing subset of analyzer keyword lists. +type EditableKeywordConfig = configstore.ComplexityEditableKeywordConfig + +// AnalyzerConfig is the runtime configuration for the complexity analyzer. +type AnalyzerConfig = configstore.ComplexityAnalyzerConfig + +// KeywordConfig is the full internal keyword set used by the compiled matcher. +type KeywordConfig struct { + CodeKeywords []string + StrongReasoningKeywords []string + WeakReasoningKeywords []string + TechnicalKeywords []string + SimpleKeywords []string + EnumTriggers []string + ComprehensivenessMarkers []string + ElaborationMarkers []string + LimitingQualifiers []string + ReferentialPhrases []string + ReferentialReferenceWords []string + ReferentialActionWords []string + TaskShiftPhrases []string +} + +// DefaultTierBoundaries returns the built-in classification thresholds. +func DefaultTierBoundaries() TierBoundaries { + return TierBoundaries{ + SimpleMedium: simpleMediumBoundary, + MediumComplex: mediumComplexBoundary, + ComplexReasoning: complexReasoningBoundary, + } +} + +// DefaultEditableKeywordConfig returns the user-visible default keyword lists. +func DefaultEditableKeywordConfig() EditableKeywordConfig { + return EditableKeywordConfig{ + CodeKeywords: cloneStringSlice(codeKeywords), + ReasoningKeywords: cloneStringSlice(strongReasoningKeywords), + TechnicalKeywords: cloneStringSlice(technicalKeywords), + SimpleKeywords: cloneStringSlice(simpleKeywords), + } +} + +// DefaultAnalyzerConfig returns the built-in analyzer config. +func DefaultAnalyzerConfig() AnalyzerConfig { + return AnalyzerConfig{ + TierBoundaries: DefaultTierBoundaries(), + Keywords: DefaultEditableKeywordConfig(), + } +} + +// ValidateAndNormalize normalizes and validates analyzer config. +func ValidateAndNormalize(cfg *AnalyzerConfig) (*AnalyzerConfig, error) { + if cfg == nil { + defaults := DefaultAnalyzerConfig() + return &defaults, nil + } + normalized := cfg.Normalized() + if err := normalized.Validate(); err != nil { + return nil, err + } + return &normalized, nil +} + +func mergeEditableKeywordsOntoDefaults(editable EditableKeywordConfig) KeywordConfig { + keywords := defaultFullKeywordConfig() + if len(editable.CodeKeywords) > 0 { + keywords.CodeKeywords = cloneStringSlice(editable.CodeKeywords) + } + if len(editable.ReasoningKeywords) > 0 { + keywords.StrongReasoningKeywords = cloneStringSlice(editable.ReasoningKeywords) + } + if len(editable.TechnicalKeywords) > 0 { + keywords.TechnicalKeywords = cloneStringSlice(editable.TechnicalKeywords) + } + if len(editable.SimpleKeywords) > 0 { + keywords.SimpleKeywords = cloneStringSlice(editable.SimpleKeywords) + } + return keywords +} + +func defaultFullKeywordConfig() KeywordConfig { + return KeywordConfig{ + CodeKeywords: cloneStringSlice(codeKeywords), + StrongReasoningKeywords: cloneStringSlice(strongReasoningKeywords), + WeakReasoningKeywords: cloneStringSlice(weakReasoningKeywords), + TechnicalKeywords: cloneStringSlice(technicalKeywords), + SimpleKeywords: cloneStringSlice(simpleKeywords), + EnumTriggers: cloneStringSlice(enumTriggers), + ComprehensivenessMarkers: cloneStringSlice(comprehensivenessMarkers), + ElaborationMarkers: cloneStringSlice(elaborationMarkers), + LimitingQualifiers: cloneStringSlice(limitingQualifiers), + ReferentialPhrases: cloneStringSlice(referentialPhrases), + ReferentialReferenceWords: cloneStringSlice(referentialReferenceWords), + ReferentialActionWords: cloneStringSlice(referentialActionWords), + TaskShiftPhrases: cloneStringSlice(taskShiftPhrases), + } +} + +func cloneStringSlice(values []string) []string { + if len(values) == 0 { + return nil + } + return append([]string(nil), values...) +} diff --git a/plugins/governance/complexity/keywords.go b/plugins/governance/complexity/keywords.go new file mode 100644 index 0000000000..3f5bba5b0e --- /dev/null +++ b/plugins/governance/complexity/keywords.go @@ -0,0 +1,131 @@ +package complexity + +// --- Dimension weights --- + +const ( + codeWeight = 0.30 + reasoningWeight = 0.25 + technicalWeight = 0.25 + simpleWeight = 0.05 // dampener, subtracted + tokenCountWeight = 0.10 + systemPromptAssistFactor = 0.25 + defaultLastMessageBlendWeight = 0.60 + defaultConversationBlendWeight = 0.40 + referentialLastMessageBlendWeight = 0.35 + referentialConversationBlendWeight = 0.65 + referentialMaxStandaloneScore = 0.15 + referentialMaxWordCount = 6 + referentialMinContextScore = 0.20 + wordPresenceSetMinBytes = 8 * 1024 + // Output complexity is applied as a score floor, not a weighted dimension +) + +// --- Keyword lists --- +// CodePresence: implementation/code syntax/workflow signals +var codeKeywords = []string{ + "function", "class", "api", "database", "algorithm", "code", "implement", + "debug", "error", "syntax", "compile", "runtime", "library", "framework", + "variable", "loop", "array", "object", "method", "interface", + "regex", "deploy", "docker", "sql", "query", "schema", "endpoint", + "refactor", "bug", "parse", "async", "webhook", "migration", + "ci/cd", "pipeline", "rest", "graphql", "test", "unit test", + "python", "javascript", "typescript", "golang", "java", "ruby", + "github actions", "monorepo", "aws cli", "config rule", "config rules", + "retry", "fallback", "middleware", "patch", "diff", "pr", "pull request", + "commit", "commit message", "behavior change", + "cel", "auto-routing", "rwmutex", "goroutine", +} + +// Reasoning markers, split into strong and weak for override logic. +var strongReasoningKeywords = []string{ + "step by step", "think through", "tradeoffs", "pros and cons", + "justify", "critique", "implications", "explain why", + "root cause analysis", "reconstruct the sequence", + "reconstruct the most likely sequence", "what should have happened instead", + "explain your reasoning", "weigh the tradeoffs", "recommend a design", +} + +var weakReasoningKeywords = []string{ + "reason", "analyze", "evaluate", "compare", "assess", "consider", + "why does", "what if", "how would", "what are the", "which approach", + "think about", "design", "most likely", "reconstruct", "verify", + "assumption", "hypothesis", "compare and contrast", "weigh the options", + "recommend one", "given these constraints", "under these constraints", +} + +// TechnicalTerms: architecture/distributed/security/infrastructure signals +var technicalKeywords = []string{ + "architecture", "distributed", "encryption", "authentication", "scalability", + "microservices", "kubernetes", "infrastructure", "protocol", "latency", + "throughput", "concurrency", "optimization", "load balancer", "caching", + "sharding", "replication", "consensus", "mutex", "deadlock", + "race condition", "api gateway", "terraform", "observability", + "access token", "refresh token", "rbac", "sso", "oidc", "saml", + "tenant", "multi-tenant", "audit log", "failover", "idempotency", + "zero downtime", "incident", "outage", "postmortem", "root cause", + "telemetry", "metrics", "configmap", "connection pool", "payment processing", + "saas", "feature flag", "operational risk", "vendor lock-in", + "s3 bucket", "misconfiguration", "remediation", "oltp", "olap", + "ledger", "metering", "aggregation", "proration", "credits", "dunning", + "invoice", "invoice generation", "double-entry", "reconciliation", + "chart of accounts", "hipaa", "quarantine workflow", "retention policy", + "audit trail", "pre-signed url", "entitlements", "seat limits", + "usage quotas", "deprovisioning", "permission drift", "role mapping", + "fraud detection", "manual review", "feedback loop", + "model serving", "a/b testing", "identity resolution", + "deterministic replay", "tamper evidence", "hash chain", + "approval workflow", "vpc", "soc 2", "data residency", + "disaster recovery", "data race", "struct copy", "hybrid search", +} + +// SimpleIndicators: signals for trivial/greeting-type requests +var simpleKeywords = []string{ + "what is", "define", "hello", "hi", "thanks", "how do i spell", + "translate", "what does", "who is", "when was", "tell me about", + "good morning", "good night", "how are you", "simple", "brief", + "short", "quick", "beginner", "basic", "concise", +} + +// --- Output complexity keywords --- + +var enumTriggers = []string{ + "list every", "list all", "enumerate all", "all possible", + "every single", "show all", "name all", "give me all", +} + +var comprehensivenessMarkers = []string{ + "comprehensive", "exhaustive", "complete list", "full list", + "in detail", "detailed breakdown", "thorough", "in-depth", +} + +var elaborationMarkers = []string{ + "and what it does", "explain each", "describe each", "for each", + "with examples", "with descriptions", "along with", +} + +var limitingQualifiers = []string{ + "briefly", "top 3", "top 5", "top 10", "in one sentence", + "quickly", "summarize", "just the", "only the", "keep it short", + "tl;dr", "tldr", +} + +var referentialPhrases = []string{ + "do it", "try again", "continue", "go ahead", "proceed", + "that one", "this one", "same thing", "again", "retry", + "yes do that", "go with that", "use option 1", "use option 2", "use option 3", + "now write it", +} + +var referentialReferenceWords = []string{ + "it", "this", "that", "same", "previous", "earlier", +} + +var referentialActionWords = []string{ + "do", "retry", "continue", "proceed", "use", "fix", + "rewrite", "shorten", "clean", "adjust", "make", "give", "answer", +} + +var taskShiftPhrases = []string{ + "translate", "summarize", "in one sentence", "one sentence", + "in spanish", "in french", "in german", "more politely", "more polite", +} diff --git a/plugins/governance/complexity/matcher.go b/plugins/governance/complexity/matcher.go new file mode 100644 index 0000000000..325721e3e1 --- /dev/null +++ b/plugins/governance/complexity/matcher.go @@ -0,0 +1,247 @@ +package complexity + +import "strings" + +type compiledKeywordMask uint16 + +const ( + maskCode compiledKeywordMask = 1 << iota + maskReasoning + maskStrongReasoning + maskTechnical + maskSimple + maskEnum + maskComprehensive + maskElaboration + maskLimiter + maskReferentialPhrase + maskReferentialReference + maskReferentialAction + maskTaskShift +) + +const ( + lastTextBaseScanMask = maskCode | maskReasoning | maskStrongReasoning | maskTechnical | maskSimple | maskEnum | maskComprehensive | maskElaboration | maskLimiter + lastTextFullScanMask = lastTextBaseScanMask | maskReferentialPhrase | maskReferentialReference | maskReferentialAction | maskTaskShift + systemTextScanMask = maskCode | maskTechnical | maskSimple + contextTextScanMask = maskCode | maskReasoning | maskTechnical +) + +type keywordMatchMode uint8 + +const ( + matchModeWholeWord keywordMatchMode = iota + matchModeBoundarySubstring + matchModePlainSubstring +) + +type compiledKeyword struct { + text string + mask compiledKeywordMask + matchMode keywordMatchMode +} + +// compiledKeywordMatcher groups keywords by match strategy so request-time +// scans can skip repeated per-keyword boundary-mode decisions. +type compiledKeywordMatcher struct { + wholeWordKeywords []compiledKeyword + boundarySubstringKeywords []compiledKeyword + plainSubstringKeywords []compiledKeyword +} + +type textSignalCounts struct { + wordCount int + codeCount int + reasoningCount int + strongReasoningCount int + technicalCount int + simpleCount int + enumCount int + comprehensiveCount int + elaborationCount int + limitingQualifierCount int + referentialPhraseCount int + referentialReferenceCount int + referentialActionCount int + taskShiftCount int +} + +func newCompiledKeywordMatcher(keywords KeywordConfig) *compiledKeywordMatcher { + entries := make(map[string]compiledKeyword) + addKeywords := func(keywords []string, mask compiledKeywordMask) { + for _, kw := range keywords { + text := strings.TrimSpace(strings.ToLower(kw)) + if text == "" { + continue + } + entry, ok := entries[text] + if !ok { + entry = compiledKeyword{ + text: text, + mask: mask, + matchMode: keywordMatchModeFor(text), + } + } else { + entry.mask |= mask + } + entries[text] = entry + } + } + + addKeywords(keywords.CodeKeywords, maskCode) + addKeywords(keywords.StrongReasoningKeywords, maskReasoning|maskStrongReasoning) + addKeywords(keywords.WeakReasoningKeywords, maskReasoning) + addKeywords(keywords.TechnicalKeywords, maskTechnical) + addKeywords(keywords.SimpleKeywords, maskSimple) + addKeywords(keywords.EnumTriggers, maskEnum) + addKeywords(keywords.ComprehensivenessMarkers, maskComprehensive) + addKeywords(keywords.ElaborationMarkers, maskElaboration) + addKeywords(keywords.LimitingQualifiers, maskLimiter) + addKeywords(keywords.ReferentialPhrases, maskReferentialPhrase) + addKeywords(keywords.ReferentialReferenceWords, maskReferentialReference) + addKeywords(keywords.ReferentialActionWords, maskReferentialAction) + addKeywords(keywords.TaskShiftPhrases, maskTaskShift) + + matcher := &compiledKeywordMatcher{} + for _, entry := range entries { + switch entry.matchMode { + case matchModeWholeWord: + matcher.wholeWordKeywords = append(matcher.wholeWordKeywords, entry) + case matchModeBoundarySubstring: + matcher.boundarySubstringKeywords = append(matcher.boundarySubstringKeywords, entry) + case matchModePlainSubstring: + matcher.plainSubstringKeywords = append(matcher.plainSubstringKeywords, entry) + } + } + return matcher +} + +func keywordMatchModeFor(keyword string) keywordMatchMode { + if strings.Contains(keyword, " ") { + return matchModePlainSubstring + } + for _, r := range keyword { + if !isWordChar(r) { + return matchModeBoundarySubstring + } + } + return matchModeWholeWord +} + +// analyzeText lowercases once, then takes a cheaper whole-word lookup path for +// larger texts where a single tokenization pass beats repeated boundary scans. +func (m *compiledKeywordMatcher) analyzeText(text string, scanMask compiledKeywordMask) textSignalCounts { + if text == "" { + return textSignalCounts{} + } + + lowerText := strings.ToLower(text) + signals := textSignalCounts{ + wordCount: countWordsNoAlloc(text), + } + + if len(lowerText) >= wordPresenceSetMinBytes { + wordPresence := buildWordPresenceSet(lowerText) + for _, keyword := range m.wholeWordKeywords { + if keyword.mask&scanMask == 0 { + continue + } + if _, ok := wordPresence[keyword.text]; ok { + signals.addMask(keyword.mask) + } + } + } else { + for _, keyword := range m.wholeWordKeywords { + if keyword.mask&scanMask == 0 { + continue + } + if containsWord(lowerText, keyword.text) { + signals.addMask(keyword.mask) + } + } + } + for _, keyword := range m.boundarySubstringKeywords { + if keyword.mask&scanMask == 0 { + continue + } + if containsWord(lowerText, keyword.text) { + signals.addMask(keyword.mask) + } + } + for _, keyword := range m.plainSubstringKeywords { + if keyword.mask&scanMask == 0 { + continue + } + if strings.Contains(lowerText, keyword.text) { + signals.addMask(keyword.mask) + } + } + + return signals +} + +// addMask increments every scoring bucket a matched keyword contributes to. +func (s *textSignalCounts) addMask(mask compiledKeywordMask) { + if mask&maskCode != 0 { + s.codeCount++ + } + if mask&maskReasoning != 0 { + s.reasoningCount++ + } + if mask&maskStrongReasoning != 0 { + s.strongReasoningCount++ + } + if mask&maskTechnical != 0 { + s.technicalCount++ + } + if mask&maskSimple != 0 { + s.simpleCount++ + } + if mask&maskEnum != 0 { + s.enumCount++ + } + if mask&maskComprehensive != 0 { + s.comprehensiveCount++ + } + if mask&maskElaboration != 0 { + s.elaborationCount++ + } + if mask&maskLimiter != 0 { + s.limitingQualifierCount++ + } + if mask&maskReferentialPhrase != 0 { + s.referentialPhraseCount++ + } + if mask&maskReferentialReference != 0 { + s.referentialReferenceCount++ + } + if mask&maskReferentialAction != 0 { + s.referentialActionCount++ + } + if mask&maskTaskShift != 0 { + s.taskShiftCount++ + } +} + +// buildWordPresenceSet tokenizes large inputs once so whole-word matches become +// set lookups instead of repeated boundary-aware scans. +func buildWordPresenceSet(text string) map[string]struct{} { + words := make(map[string]struct{}, 64) + start := -1 + for i, r := range text { + if isWordChar(r) { + if start == -1 { + start = i + } + continue + } + if start != -1 { + words[text[start:i]] = struct{}{} + start = -1 + } + } + if start != -1 { + words[text[start:]] = struct{}{} + } + return words +} diff --git a/plugins/governance/complexity/utils.go b/plugins/governance/complexity/utils.go new file mode 100644 index 0000000000..b6fbd6c07f --- /dev/null +++ b/plugins/governance/complexity/utils.go @@ -0,0 +1,114 @@ +package complexity + +import ( + "math" + "strings" + "unicode" + "unicode/utf8" +) + +// containsWord checks if a word appears in text delimited by non-alphanumeric boundaries. +func containsWord(text, word string) bool { + if word == "" { + return false + } + + idx := 0 + for { + pos := strings.Index(text[idx:], word) + if pos == -1 { + return false + } + start := idx + pos + end := start + len(word) + + startOk := start == 0 || !isWordChar(lastRune(text[:start])) + endOk := end == len(text) || !isWordChar(firstRune(text[end:])) + + if startOk && endOk { + return true + } + idx = start + 1 + if idx >= len(text) { + return false + } + } +} + +func firstRune(text string) rune { + r, _ := utf8.DecodeRuneInString(text) + return r +} + +func lastRune(text string) rune { + r, _ := utf8.DecodeLastRuneInString(text) + return r +} + +func isWordChar(r rune) bool { + return unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' +} + +func countWordsNoAlloc(text string) int { + count := 0 + inWord := false + for _, r := range text { + if unicode.IsSpace(r) { + inWord = false + continue + } + if !inWord { + count++ + inWord = true + } + } + return count +} + +func scoreCount(count, capAt int) float64 { + if capAt <= 0 { + return 0.0 + } + return math.Min(1.0, float64(count)/float64(capAt)) +} + +func scoreOutputComplexity(signals textSignalCounts) float64 { + totalCount := signals.enumCount + signals.comprehensiveCount + signals.elaborationCount + if totalCount == 0 { + return 0.0 + } + + enumScore := math.Min(1.0, float64(signals.enumCount)) + compScore := math.Min(1.0, float64(signals.comprehensiveCount)) + elabScore := math.Min(1.0, float64(signals.elaborationCount)) + + rawScore := (enumScore * 0.4) + (compScore * 0.3) + (elabScore * 0.3) + if signals.limitingQualifierCount > 0 { + rawScore *= 0.3 + } + + return math.Min(1.0, rawScore) +} + +// scoreTokenCount scores based on word count of the text. +func scoreTokenCount(words int) float64 { + switch { + case words < 15: + return float64(words) / 15.0 * 0.3 + case words <= 400: + return 0.3 + float64(words-15)/385.0*0.4 + default: + extra := math.Min(0.3, float64(words-400)/600.0*0.3) + return 0.7 + extra + } +} + +func clamp(val, min, max float64) float64 { + if val < min { + return min + } + if val > max { + return max + } + return val +} diff --git a/plugins/governance/complexityextract.go b/plugins/governance/complexityextract.go new file mode 100644 index 0000000000..3516944ad3 --- /dev/null +++ b/plugins/governance/complexityextract.go @@ -0,0 +1,233 @@ +package governance + +import ( + "strings" + + "github.com/maximhq/bifrost/core/schemas" + "github.com/maximhq/bifrost/plugins/governance/complexity" +) + +// buildComplexityInput extracts text from normalized BifrostRequest values for +// complexity_tier routing. It intentionally runs after the transport converters +// have produced Bifrost's typed request shape, so governance does not duplicate +// provider-specific raw payload parsing. +func buildComplexityInput(req *schemas.BifrostRequest) (complexity.ComplexityInput, bool) { + if req == nil { + return complexity.ComplexityInput{}, false + } + + switch req.RequestType { + case schemas.ChatCompletionRequest, schemas.ChatCompletionStreamRequest: + if req.ChatRequest == nil { + return complexity.ComplexityInput{}, false + } + return extractFromChatMessages(req.ChatRequest.Input) + case schemas.TextCompletionRequest, schemas.TextCompletionStreamRequest: + if req.TextCompletionRequest == nil { + return complexity.ComplexityInput{}, false + } + return extractFromTextCompletionRequest(req.TextCompletionRequest) + case schemas.ResponsesRequest, schemas.ResponsesStreamRequest: + if req.ResponsesRequest == nil { + return complexity.ComplexityInput{}, false + } + return extractFromResponsesRequest(req.ResponsesRequest) + default: + return complexity.ComplexityInput{}, false + } +} + +// extractFromChatMessages builds a complexity input from chat messages by +// preserving system/developer context and tracking only text-only user turns. +func extractFromChatMessages(messages []schemas.ChatMessage) (complexity.ComplexityInput, bool) { + if len(messages) == 0 { + return complexity.ComplexityInput{}, false + } + + var input complexity.ComplexityInput + var userTexts []string + + for _, msg := range messages { + switch msg.Role { + case schemas.ChatMessageRoleSystem, schemas.ChatMessageRoleDeveloper: + input.SystemText = appendText(input.SystemText, extractChatText(msg.Content)) + case schemas.ChatMessageRoleUser: + text, ok := extractChatTextOnly(msg.Content) + if !ok || strings.TrimSpace(text) == "" { + return complexity.ComplexityInput{}, false + } + userTexts = append(userTexts, text) + } + } + + if len(userTexts) == 0 { + return complexity.ComplexityInput{}, false + } + + input.LastUserText = userTexts[len(userTexts)-1] + if len(userTexts) > 1 { + input.PriorUserTexts = userTexts[:len(userTexts)-1] + } + return input, true +} + +// extractFromTextCompletionRequest builds a complexity input from a single text +// completion prompt and deliberately skips batched prompt arrays. +func extractFromTextCompletionRequest(req *schemas.BifrostTextCompletionRequest) (complexity.ComplexityInput, bool) { + if req == nil || req.Input == nil || req.Input.PromptStr == nil || strings.TrimSpace(*req.Input.PromptStr) == "" { + return complexity.ComplexityInput{}, false + } + + // PromptArray represents batched completions, not one logical prompt. Do not + // synthesize a single routing input by joining unrelated batch entries. + return complexity.ComplexityInput{LastUserText: *req.Input.PromptStr}, true +} + +// extractFromResponsesRequest builds a complexity input from Responses API +// messages while combining instructions with system/developer message text. +func extractFromResponsesRequest(req *schemas.BifrostResponsesRequest) (complexity.ComplexityInput, bool) { + if req == nil || len(req.Input) == 0 { + return complexity.ComplexityInput{}, false + } + + var input complexity.ComplexityInput + if req.Params != nil && req.Params.Instructions != nil { + input.SystemText = *req.Params.Instructions + } + + var userTexts []string + for _, msg := range req.Input { + if msg.Role == nil { + continue + } + + switch *msg.Role { + case schemas.ResponsesInputMessageRoleSystem, schemas.ResponsesInputMessageRoleDeveloper: + input.SystemText = appendText(input.SystemText, extractResponsesText(msg.Content)) + case schemas.ResponsesInputMessageRoleUser: + text, ok := extractResponsesTextOnly(msg.Content) + if !ok || strings.TrimSpace(text) == "" { + return complexity.ComplexityInput{}, false + } + userTexts = append(userTexts, text) + } + } + + if len(userTexts) == 0 { + return complexity.ComplexityInput{}, false + } + + input.LastUserText = userTexts[len(userTexts)-1] + if len(userTexts) > 1 { + input.PriorUserTexts = userTexts[:len(userTexts)-1] + } + return input, true +} + +// extractChatText returns the text portions of chat content and ignores +// non-text blocks so system/developer context can still be used. +func extractChatText(content *schemas.ChatMessageContent) string { + if content == nil { + return "" + } + if content.ContentStr != nil { + return *content.ContentStr + } + + var text string + for _, block := range content.ContentBlocks { + if isChatTextBlock(block) && block.Text != nil && *block.Text != "" { + text = appendText(text, *block.Text) + } + } + return text +} + +// extractChatTextOnly returns chat content only when every block is text, +// allowing mixed-modality user prompts to opt out of complexity routing. +func extractChatTextOnly(content *schemas.ChatMessageContent) (string, bool) { + if content == nil { + return "", false + } + if content.ContentStr != nil { + return *content.ContentStr, true + } + if len(content.ContentBlocks) == 0 { + return "", false + } + + var text string + for _, block := range content.ContentBlocks { + if !isChatTextBlock(block) || block.Text == nil || *block.Text == "" { + return "", false + } + text = appendText(text, *block.Text) + } + return text, true +} + +// extractResponsesText returns the text portions of Responses content and +// ignores non-input-text blocks used by non-user context. +func extractResponsesText(content *schemas.ResponsesMessageContent) string { + if content == nil { + return "" + } + if content.ContentStr != nil { + return *content.ContentStr + } + + var text string + for _, block := range content.ContentBlocks { + if isResponsesInputTextBlock(block) && block.Text != nil && *block.Text != "" { + text = appendText(text, *block.Text) + } + } + return text +} + +// extractResponsesTextOnly returns Responses content only when every block is +// input text, avoiding synthesized prompts for mixed-modality user requests. +func extractResponsesTextOnly(content *schemas.ResponsesMessageContent) (string, bool) { + if content == nil { + return "", false + } + if content.ContentStr != nil { + return *content.ContentStr, true + } + if len(content.ContentBlocks) == 0 { + return "", false + } + + var text string + for _, block := range content.ContentBlocks { + if !isResponsesInputTextBlock(block) || block.Text == nil || *block.Text == "" { + return "", false + } + text = appendText(text, *block.Text) + } + return text, true +} + +// isChatTextBlock reports whether a chat content block is plain text, treating +// an empty type as text for compatibility with normalized request payloads. +func isChatTextBlock(block schemas.ChatContentBlock) bool { + return block.Type == "" || block.Type == schemas.ChatContentBlockTypeText +} + +// isResponsesInputTextBlock reports whether a Responses content block is input +// text, treating an empty type as text for compatibility with normalized input. +func isResponsesInputTextBlock(block schemas.ResponsesMessageContentBlock) bool { + return block.Type == "" || block.Type == schemas.ResponsesInputMessageContentBlockTypeText +} + +// appendText joins adjacent text fragments with one separating space while +// preserving empty existing or next values. +func appendText(existing, next string) string { + if next == "" { + return existing + } + if existing == "" { + return next + } + return existing + " " + next +} diff --git a/plugins/governance/complexityextract_test.go b/plugins/governance/complexityextract_test.go new file mode 100644 index 0000000000..126cda3741 --- /dev/null +++ b/plugins/governance/complexityextract_test.go @@ -0,0 +1,287 @@ +package governance + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/maximhq/bifrost/core/schemas" +) + +func TestBuildComplexityInput_ChatTextMessages(t *testing.T) { + req := &schemas.BifrostRequest{ + RequestType: schemas.ChatCompletionRequest, + ChatRequest: &schemas.BifrostChatRequest{ + Input: []schemas.ChatMessage{ + { + Role: schemas.ChatMessageRoleSystem, + Content: complexityChatString("Be concise"), + }, + { + Role: schemas.ChatMessageRoleUser, + Content: complexityChatString("Explain vector clocks"), + }, + { + Role: schemas.ChatMessageRoleAssistant, + Content: complexityChatString("Vector clocks track causal history."), + }, + { + Role: schemas.ChatMessageRoleUser, + Content: complexityChatBlocks( + complexityChatTextBlock("Compare them to Lamport clocks"), + ), + }, + }, + }, + } + + input, ok := buildComplexityInput(req) + require.True(t, ok) + assert.Equal(t, "Compare them to Lamport clocks", input.LastUserText) + assert.Equal(t, []string{"Explain vector clocks"}, input.PriorUserTexts) + assert.Equal(t, "Be concise", input.SystemText) +} + +func TestBuildComplexityInput_TextCompletionPrompt(t *testing.T) { + prompt := "Write a short summary of this changelog" + req := &schemas.BifrostRequest{ + RequestType: schemas.TextCompletionRequest, + TextCompletionRequest: &schemas.BifrostTextCompletionRequest{ + Input: &schemas.TextCompletionInput{PromptStr: &prompt}, + }, + } + + input, ok := buildComplexityInput(req) + require.True(t, ok) + assert.Equal(t, prompt, input.LastUserText) +} + +func TestBuildComplexityInput_TextCompletionPromptArraySkipped(t *testing.T) { + req := &schemas.BifrostRequest{ + RequestType: schemas.TextCompletionRequest, + TextCompletionRequest: &schemas.BifrostTextCompletionRequest{ + Input: &schemas.TextCompletionInput{ + PromptArray: []string{ + "Summarize this short changelog", + "Debug this distributed tracing timeout and propose fixes", + }, + }, + }, + } + + input, ok := buildComplexityInput(req) + require.False(t, ok) + assert.Empty(t, input.LastUserText) +} + +func TestBuildComplexityInput_ResponsesInputTextBlocks(t *testing.T) { + systemRole := schemas.ResponsesInputMessageRoleSystem + userRole := schemas.ResponsesInputMessageRoleUser + assistantRole := schemas.ResponsesInputMessageRoleAssistant + instructions := "Review carefully" + + req := &schemas.BifrostRequest{ + RequestType: schemas.ResponsesRequest, + ResponsesRequest: &schemas.BifrostResponsesRequest{ + Params: &schemas.ResponsesParameters{Instructions: &instructions}, + Input: []schemas.ResponsesMessage{ + { + Role: &systemRole, + Content: complexityResponsesString("Be concise"), + }, + { + Role: &userRole, + Content: complexityResponsesBlocks( + complexityResponsesTextBlock("I changed the retry policy and circuit breaker thresholds."), + ), + }, + { + Role: &assistantRole, + Content: complexityResponsesBlocks( + complexityResponsesOutputTextBlock("The patch retries idempotent requests and opens the breaker sooner."), + ), + }, + { + Role: &userRole, + Content: complexityResponsesBlocks( + complexityResponsesTextBlock("Can you explain the changes?"), + ), + }, + }, + }, + } + + input, ok := buildComplexityInput(req) + require.True(t, ok) + assert.Equal(t, "Can you explain the changes?", input.LastUserText) + assert.Equal(t, []string{"I changed the retry policy and circuit breaker thresholds."}, input.PriorUserTexts) + assert.Equal(t, "Review carefully Be concise", input.SystemText) +} + +func TestBuildComplexityInput_SupportsStreamingRequestTypes(t *testing.T) { + prompt := "Write a short summary of this changelog" + userRole := schemas.ResponsesInputMessageRoleUser + instructions := "Answer carefully" + + tests := []struct { + name string + req *schemas.BifrostRequest + wantLastUser string + wantSystem string + }{ + { + name: "chat_completion_stream", + req: &schemas.BifrostRequest{ + RequestType: schemas.ChatCompletionStreamRequest, + ChatRequest: &schemas.BifrostChatRequest{ + Input: []schemas.ChatMessage{ + {Role: schemas.ChatMessageRoleSystem, Content: complexityChatString("Be concise")}, + {Role: schemas.ChatMessageRoleUser, Content: complexityChatString("Explain vector clocks")}, + }, + }, + }, + wantLastUser: "Explain vector clocks", + wantSystem: "Be concise", + }, + { + name: "text_completion_stream", + req: &schemas.BifrostRequest{ + RequestType: schemas.TextCompletionStreamRequest, + TextCompletionRequest: &schemas.BifrostTextCompletionRequest{ + Input: &schemas.TextCompletionInput{PromptStr: &prompt}, + }, + }, + wantLastUser: prompt, + }, + { + name: "responses_stream", + req: &schemas.BifrostRequest{ + RequestType: schemas.ResponsesStreamRequest, + ResponsesRequest: &schemas.BifrostResponsesRequest{ + Params: &schemas.ResponsesParameters{Instructions: &instructions}, + Input: []schemas.ResponsesMessage{ + { + Role: &userRole, + Content: complexityResponsesBlocks( + complexityResponsesTextBlock("Compare Go channels and mutexes"), + ), + }, + }, + }, + }, + wantLastUser: "Compare Go channels and mutexes", + wantSystem: "Answer carefully", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + input, ok := buildComplexityInput(tt.req) + require.True(t, ok) + assert.Equal(t, tt.wantLastUser, input.LastUserText) + assert.Equal(t, tt.wantSystem, input.SystemText) + }) + } +} + +func TestBuildComplexityInput_SkipsUnsupportedRequestTypesEvenWhenTextIsPresent(t *testing.T) { + userRole := schemas.ResponsesInputMessageRoleUser + req := &schemas.BifrostRequest{ + RequestType: schemas.CountTokensRequest, + CountTokensRequest: &schemas.BifrostResponsesRequest{ + Input: []schemas.ResponsesMessage{ + { + Role: &userRole, + Content: complexityResponsesBlocks( + complexityResponsesTextBlock("How many tokens is this prompt?"), + ), + }, + }, + }, + } + + input, ok := buildComplexityInput(req) + require.False(t, ok) + assert.Empty(t, input.LastUserText) +} + +func TestBuildComplexityInput_SkipsMixedModalityUserContent(t *testing.T) { + userRole := schemas.ResponsesInputMessageRoleUser + + tests := []struct { + name string + req *schemas.BifrostRequest + }{ + { + name: "chat_text_plus_image", + req: &schemas.BifrostRequest{ + RequestType: schemas.ChatCompletionRequest, + ChatRequest: &schemas.BifrostChatRequest{ + Input: []schemas.ChatMessage{ + { + Role: schemas.ChatMessageRoleUser, + Content: complexityChatBlocks( + complexityChatTextBlock("What changed in this screenshot?"), + schemas.ChatContentBlock{Type: schemas.ChatContentBlockTypeImage}, + ), + }, + }, + }, + }, + }, + { + name: "responses_text_plus_file", + req: &schemas.BifrostRequest{ + RequestType: schemas.ResponsesRequest, + ResponsesRequest: &schemas.BifrostResponsesRequest{ + Input: []schemas.ResponsesMessage{ + { + Role: &userRole, + Content: complexityResponsesBlocks( + complexityResponsesTextBlock("Summarize this document"), + schemas.ResponsesMessageContentBlock{Type: schemas.ResponsesInputMessageContentBlockTypeFile}, + ), + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + input, ok := buildComplexityInput(tt.req) + require.False(t, ok) + assert.Empty(t, input.LastUserText) + }) + } +} + +func complexityChatString(text string) *schemas.ChatMessageContent { + return &schemas.ChatMessageContent{ContentStr: &text} +} + +func complexityChatBlocks(blocks ...schemas.ChatContentBlock) *schemas.ChatMessageContent { + return &schemas.ChatMessageContent{ContentBlocks: blocks} +} + +func complexityChatTextBlock(text string) schemas.ChatContentBlock { + return schemas.ChatContentBlock{Type: schemas.ChatContentBlockTypeText, Text: &text} +} + +func complexityResponsesString(text string) *schemas.ResponsesMessageContent { + return &schemas.ResponsesMessageContent{ContentStr: &text} +} + +func complexityResponsesBlocks(blocks ...schemas.ResponsesMessageContentBlock) *schemas.ResponsesMessageContent { + return &schemas.ResponsesMessageContent{ContentBlocks: blocks} +} + +func complexityResponsesTextBlock(text string) schemas.ResponsesMessageContentBlock { + return schemas.ResponsesMessageContentBlock{Type: schemas.ResponsesInputMessageContentBlockTypeText, Text: &text} +} + +func complexityResponsesOutputTextBlock(text string) schemas.ResponsesMessageContentBlock { + return schemas.ResponsesMessageContentBlock{Type: schemas.ResponsesOutputMessageContentTypeText, Text: &text} +} diff --git a/plugins/governance/go.mod b/plugins/governance/go.mod index 8e6d9d740d..579236985e 100644 --- a/plugins/governance/go.mod +++ b/plugins/governance/go.mod @@ -5,7 +5,6 @@ go 1.26.4 require gorm.io/gorm v1.31.1 require ( - github.com/bytedance/sonic v1.15.1 github.com/google/cel-go v0.28.1 github.com/google/uuid v1.6.0 github.com/maximhq/bifrost/core v1.5.18 @@ -20,7 +19,7 @@ require ( cloud.google.com/go/auth v0.20.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect - cloud.google.com/go/iam v1.5.3 // indirect + cloud.google.com/go/iam v1.7.0 // indirect cloud.google.com/go/monitoring v1.24.3 // indirect cloud.google.com/go/storage v1.61.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect @@ -33,13 +32,13 @@ require ( github.com/andybalholm/brotli v1.2.1 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect - github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect + github.com/aws/aws-sdk-go-v2 v1.41.12 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect github.com/aws/aws-sdk-go-v2/config v1.32.11 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect @@ -51,10 +50,11 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect - github.com/aws/smithy-go v1.25.1 // indirect + github.com/aws/smithy-go v1.27.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.2 // indirect github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic v1.15.1 // indirect github.com/bytedance/sonic/loader v0.5.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect @@ -144,7 +144,7 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.23.0 // indirect golang.org/x/crypto v0.52.0 // indirect - golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect + golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect golang.org/x/net v0.55.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sync v0.20.0 // indirect diff --git a/plugins/governance/go.sum b/plugins/governance/go.sum index e178cff45a..867a6e1bf6 100644 --- a/plugins/governance/go.sum +++ b/plugins/governance/go.sum @@ -8,8 +8,8 @@ cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIi cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= -cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= +cloud.google.com/go/iam v1.7.0 h1:JD3zh0C6LHl16aCn5Akff0+GELdp1+4hmh6ndoFLl8U= +cloud.google.com/go/iam v1.7.0/go.mod h1:tetWZW1PD/m6vcuY2Zj/aU0eCHNPuxedbnbRTyKXvdY= cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHAjA= cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZCn6hSBtKak= cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8= @@ -47,8 +47,8 @@ github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYW github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= -github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= -github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2 v1.41.12 h1:DIKX2c31ekm9RA2D9FBj1EWXx++9AdAqRw+e78Tq2Ck= +github.com/aws/aws-sdk-go-v2 v1.41.12/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= github.com/aws/aws-sdk-go-v2/config v1.32.11 h1:ftxI5sgz8jZkckuUHXfC/wMUc8u3fG1vQS0plr2F2Zs= @@ -57,10 +57,10 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8TH github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 h1:Xf2j7NdVcUKomlZ4iihOP4AZ3Fzlr8h4yKpXeP+OFPg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28/go.mod h1:O8cDo1dW63jU7ki//kRe1z+tLGcpnD1jrouitsQddDw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 h1:KqIfN9kpkKkcBqBbNpNGTIrXO6ExTUvFKvXkC+YAzVo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28/go.mod h1:uxtQiKvLtNS4iXVsH2McVD/ls8FKN/uUhe1hGxPjrw0= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 h1:clHU5fm//kWS1C2HgtgWxfQbFbx4b6rx+5jzhgX9HrI= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= @@ -83,8 +83,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6f github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= -github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= -github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8= +github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= @@ -349,8 +349,8 @@ golang.org/x/arch v0.23.0 h1:lKF64A2jF6Zd8L0knGltUnegD62JMFBiCPBmQpToHhg= golang.org/x/arch v0.23.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= -golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0= -golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= diff --git a/plugins/governance/httptransportprehook_test.go b/plugins/governance/httptransportprehook_test.go deleted file mode 100644 index c05eb95707..0000000000 --- a/plugins/governance/httptransportprehook_test.go +++ /dev/null @@ -1,703 +0,0 @@ -package governance - -import ( - "context" - "encoding/json" - "testing" - - bifrost "github.com/maximhq/bifrost/core" - "github.com/maximhq/bifrost/core/schemas" - "github.com/maximhq/bifrost/framework/configstore" - configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" - "github.com/maximhq/bifrost/framework/modelcatalog" - "github.com/stretchr/testify/require" -) - -// TestHTTPTransportPreHook_VirtualKeyReplicateRefinesNestedModel verifies that -// virtual-key provider pinning rewrites the request model to Replicate's nested provider slug. -func TestHTTPTransportPreHook_VirtualKeyReplicateRefinesNestedModel(t *testing.T) { - logger := NewMockLogger() - mc := modelcatalog.NewTestCatalog(map[string]string{ - "openai/gpt-5-nano": "gpt-5-nano", - }) - mc.UpsertModelDataForProvider(schemas.Replicate, &schemas.BifrostListModelsResponse{ - Data: []schemas.Model{ - {ID: "replicate/openai/gpt-5-nano"}, - }, - }, nil) - - virtualKey := buildVirtualKeyWithProviders( - "vk1", - "sk-bf-test", - "replicate-only", - []configstoreTables.TableVirtualKeyProviderConfig{ - buildProviderConfig("replicate", []string{"*"}), - }, - ) - store, err := NewLocalGovernanceStore(context.Background(), logger, nil, &configstore.GovernanceConfig{ - VirtualKeys: []configstoreTables.TableVirtualKey{*virtualKey}, - }, mc) - require.NoError(t, err) - - plugin, err := InitFromStore(context.Background(), &Config{IsVkMandatory: boolPtr(false)}, logger, store, nil, mc, nil, nil) - require.NoError(t, err) - defer func() { - require.NoError(t, plugin.Cleanup()) - }() - - req := schemas.AcquireHTTPRequest() - defer schemas.ReleaseHTTPRequest(req) - req.Method = "POST" - req.Path = "/v1/chat/completions" - req.Headers["Authorization"] = "Bearer sk-bf-test" - req.Headers["Content-Type"] = "application/json" - req.Body = []byte(`{"model":"gpt-5-nano","messages":[{"role":"user","content":"Hello!"}]}`) - - bfCtx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) - resp, err := plugin.HTTPTransportPreHook(bfCtx, req) - require.NoError(t, err) - require.Nil(t, resp) - - var payload struct { - Model string `json:"model"` - } - require.NoError(t, json.Unmarshal(req.Body, &payload)) - require.Equal(t, "replicate/openai/gpt-5-nano", payload.Model) -} - -func TestHTTPTransportPreHook_ModelOnlyVirtualKeySetsAvailableProviders(t *testing.T) { - logger := NewMockLogger() - - openAIConfig := buildProviderConfig("openai", []string{"gpt-4o"}) - openAIConfig.Weight = nil - anthropicConfig := buildProviderConfig("anthropic", []string{"claude-3-5-sonnet"}) - anthropicConfig.Weight = nil - - virtualKey := buildVirtualKeyWithProviders( - "vk-constraint", - "sk-bf-constraint-test", - "provider-constraint-vk", - []configstoreTables.TableVirtualKeyProviderConfig{ - openAIConfig, - anthropicConfig, - }, - ) - store, err := NewLocalGovernanceStore(context.Background(), logger, nil, &configstore.GovernanceConfig{ - VirtualKeys: []configstoreTables.TableVirtualKey{*virtualKey}, - }, nil) - require.NoError(t, err) - - plugin, err := InitFromStore(context.Background(), &Config{IsVkMandatory: boolPtr(false)}, logger, store, nil, nil, nil, nil) - require.NoError(t, err) - defer func() { - require.NoError(t, plugin.Cleanup()) - }() - - req := schemas.AcquireHTTPRequest() - defer schemas.ReleaseHTTPRequest(req) - req.Method = "POST" - req.Path = "/v1/chat/completions" - req.Headers["Authorization"] = "Bearer sk-bf-constraint-test" - req.Headers["Content-Type"] = "application/json" - req.Body = []byte(`{"model":"gpt-4o","messages":[{"role":"user","content":"Hello!"}]}`) - - bfCtx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) - resp, err := plugin.HTTPTransportPreHook(bfCtx, req) - require.NoError(t, err) - require.Nil(t, resp) - - allowedProviders, ok := bfCtx.Value(schemas.BifrostContextKeyAvailableProviders).([]schemas.ModelProvider) - require.True(t, ok, "provider constraint should be set") - require.Equal(t, []schemas.ModelProvider{schemas.OpenAI}, allowedProviders) -} - -func TestHTTPTransportPreHook_ModelOnlyVirtualKeySetsEmptyAvailableProvidersWhenNoProviderAllowsModel(t *testing.T) { - logger := NewMockLogger() - - virtualKey := buildVirtualKeyWithProviders( - "vk-empty-constraint", - "sk-bf-empty-constraint-test", - "empty-provider-constraint-vk", - []configstoreTables.TableVirtualKeyProviderConfig{ - buildProviderConfig("openai", []string{"gpt-4o"}), - }, - ) - store, err := NewLocalGovernanceStore(context.Background(), logger, nil, &configstore.GovernanceConfig{ - VirtualKeys: []configstoreTables.TableVirtualKey{*virtualKey}, - }, nil) - require.NoError(t, err) - - plugin, err := InitFromStore(context.Background(), &Config{IsVkMandatory: boolPtr(false)}, logger, store, nil, nil, nil, nil) - require.NoError(t, err) - defer func() { - require.NoError(t, plugin.Cleanup()) - }() - - req := schemas.AcquireHTTPRequest() - defer schemas.ReleaseHTTPRequest(req) - req.Method = "POST" - req.Path = "/v1/chat/completions" - req.Headers["Authorization"] = "Bearer sk-bf-empty-constraint-test" - req.Headers["Content-Type"] = "application/json" - req.Body = []byte(`{"model":"claude-3-5-sonnet","messages":[{"role":"user","content":"Hello!"}]}`) - - bfCtx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) - resp, err := plugin.HTTPTransportPreHook(bfCtx, req) - require.NoError(t, err) - require.Nil(t, resp) - - allowedProviders, ok := bfCtx.Value(schemas.BifrostContextKeyAvailableProviders).([]schemas.ModelProvider) - require.True(t, ok, "provider constraint should be set") - require.Empty(t, allowedProviders) -} - -// TestHTTPTransportPreHook_WildcardKeepsCatalogOpaqueProvider_VLLM verifies that a VK with a -// wildcard ("*") allow-list on a catalog-opaque provider (here vLLM, whose self-hosted models -// are never in the bundled catalog) keeps that provider in BifrostContextKeyAvailableProviders -// for a bare, uncatalogued model. Before the fix, loadBalanceProvider gates the provider on the -// catalog (GetProvidersForModel is empty), drops it, and publishes an empty provider set — -// dead-ending the request (issue #4122 / #3282). -func TestHTTPTransportPreHook_WildcardKeepsCatalogOpaqueProvider_VLLM(t *testing.T) { - logger := NewMockLogger() - - // Catalog knows a first-party model but has NO model list for vLLM (self-hosted). - mc := modelcatalog.NewTestCatalog(map[string]string{"openai/gpt-4o": "gpt-4o"}) - mc.UpsertModelDataForProvider(schemas.OpenAI, - &schemas.BifrostListModelsResponse{Data: []schemas.Model{{ID: "openai/gpt-4o"}}}, nil) - - // inMemoryStore must be non-nil so loadBalanceProvider takes the catalog branch. - inMem := &mockInMemoryStore{ - configuredProviders: map[schemas.ModelProvider]configstore.ProviderConfig{ - schemas.VLLM: {}, // native keyless: no CustomProviderConfig - }, - } - - virtualKey := buildVirtualKeyWithProviders( - "vk-vllm", - "sk-bf-vllm-test", - "vllm-vk", - []configstoreTables.TableVirtualKeyProviderConfig{ - buildProviderConfig("vllm", []string{"*"}), - }, - ) - store, err := NewLocalGovernanceStore(context.Background(), logger, nil, &configstore.GovernanceConfig{ - VirtualKeys: []configstoreTables.TableVirtualKey{*virtualKey}, - }, mc) - require.NoError(t, err) - - plugin, err := InitFromStore(context.Background(), &Config{IsVkMandatory: boolPtr(false)}, logger, store, nil, mc, nil, inMem) - require.NoError(t, err) - defer func() { - require.NoError(t, plugin.Cleanup()) - }() - - req := schemas.AcquireHTTPRequest() - defer schemas.ReleaseHTTPRequest(req) - req.Method = "POST" - req.Path = "/v1/chat/completions" - req.Headers["Authorization"] = "Bearer sk-bf-vllm-test" - req.Headers["Content-Type"] = "application/json" - req.Body = []byte(`{"model":"my-self-hosted-llama","messages":[{"role":"user","content":"Hi"}]}`) - - bfCtx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) - resp, err := plugin.HTTPTransportPreHook(bfCtx, req) - require.NoError(t, err) - require.Nil(t, resp) - - allowedProviders, ok := bfCtx.Value(schemas.BifrostContextKeyAvailableProviders).([]schemas.ModelProvider) - require.True(t, ok, "available providers should be set") - // PRE-PATCH: catalog has no vLLM models -> provider excluded -> [] -> FAILS. - // POST-PATCH: wildcard + catalog-opaque -> kept -> [vllm]. - require.Equal(t, []schemas.ModelProvider{schemas.VLLM}, allowedProviders) -} - -// TestHTTPTransportPreHook_MixedOpaqueAndCatalogProvider_GPT4o shows what lands in -// BifrostContextKeyAvailableProviders when a VK has catalog-known providers (openai, anthropic, -// vertex) AND a catalog-opaque vLLM (no list-models) — all under wildcard allow-lists — and the -// model is gpt-4o. Only openai (which serves gpt-4o per the catalog) and vLLM (a wildcard -// catch-all) should be available; anthropic and vertex are catalog-known but do not serve gpt-4o. -func TestHTTPTransportPreHook_MixedOpaqueAndCatalogProvider_GPT4o(t *testing.T) { - logger := NewMockLogger() - - // Catalog knows openai/gpt-4o, anthropic/claude-3-5-sonnet, vertex/gemini-1.5-pro. - // It has NO model list for vLLM (opaque). - mc := modelcatalog.NewTestCatalog(map[string]string{"openai/gpt-4o": "gpt-4o"}) - mc.UpsertModelDataForProvider(schemas.OpenAI, - &schemas.BifrostListModelsResponse{Data: []schemas.Model{{ID: "openai/gpt-4o"}}}, nil) - mc.UpsertModelDataForProvider(schemas.Anthropic, - &schemas.BifrostListModelsResponse{Data: []schemas.Model{{ID: "anthropic/claude-3-5-sonnet"}}}, nil) - mc.UpsertModelDataForProvider(schemas.Vertex, - &schemas.BifrostListModelsResponse{Data: []schemas.Model{{ID: "vertex/gemini-1.5-pro"}}}, nil) - - inMem := &mockInMemoryStore{ - configuredProviders: map[schemas.ModelProvider]configstore.ProviderConfig{ - schemas.OpenAI: {}, - schemas.Anthropic: {}, - schemas.Vertex: {}, - schemas.VLLM: {}, // opaque: no CustomProviderConfig, no catalog models - }, - } - - virtualKey := buildVirtualKeyWithProviders( - "vk-mixed", - "sk-bf-mixed-test", - "mixed-providers-vk", - []configstoreTables.TableVirtualKeyProviderConfig{ - buildProviderConfig("openai", []string{"*"}), - buildProviderConfig("anthropic", []string{"*"}), - buildProviderConfig("vertex", []string{"*"}), - buildProviderConfig("vllm", []string{"*"}), - }, - ) - store, err := NewLocalGovernanceStore(context.Background(), logger, nil, &configstore.GovernanceConfig{ - VirtualKeys: []configstoreTables.TableVirtualKey{*virtualKey}, - }, mc) - require.NoError(t, err) - - plugin, err := InitFromStore(context.Background(), &Config{IsVkMandatory: boolPtr(false)}, logger, store, nil, mc, nil, inMem) - require.NoError(t, err) - defer func() { - require.NoError(t, plugin.Cleanup()) - }() - - req := schemas.AcquireHTTPRequest() - defer schemas.ReleaseHTTPRequest(req) - req.Method = "POST" - req.Path = "/v1/chat/completions" - req.Headers["Authorization"] = "Bearer sk-bf-mixed-test" - req.Headers["Content-Type"] = "application/json" - req.Body = []byte(`{"model":"gpt-4o","messages":[{"role":"user","content":"Hi"}]}`) - - bfCtx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) - resp, err := plugin.HTTPTransportPreHook(bfCtx, req) - require.NoError(t, err) - require.Nil(t, resp) - - allowedProviders, ok := bfCtx.Value(schemas.BifrostContextKeyAvailableProviders).([]schemas.ModelProvider) - require.True(t, ok, "available providers should be set") - t.Logf("AvailableProviders for gpt-4o (VK = openai + vllm-opaque, both wildcard): %v", allowedProviders) - - // Both compete: openai matches the catalog for gpt-4o; vLLM is a wildcard catch-all. - require.ElementsMatch(t, []schemas.ModelProvider{schemas.OpenAI, schemas.VLLM}, allowedProviders) -} - -// TestHTTPTransportPreHook_VKExcludesUnlistedProviderEvenIfItServesModel shows that VK scoping -// wins: even when the catalog says BOTH openai and vertex serve gpt-4o, a VK granting access to -// only openai + vLLM yields exactly [openai, vllm] — vertex is never a candidate because it is -// not in the VK's provider configs. -func TestHTTPTransportPreHook_VKExcludesUnlistedProviderEvenIfItServesModel(t *testing.T) { - logger := NewMockLogger() - - // Catalog: BOTH openai and vertex serve gpt-4o. vLLM has no catalog models (opaque). - mc := modelcatalog.NewTestCatalog(map[string]string{"openai/gpt-4o": "gpt-4o"}) - mc.UpsertModelDataForProvider(schemas.OpenAI, - &schemas.BifrostListModelsResponse{Data: []schemas.Model{{ID: "openai/gpt-4o"}}}, nil) - mc.UpsertModelDataForProvider(schemas.Vertex, - &schemas.BifrostListModelsResponse{Data: []schemas.Model{{ID: "vertex/gpt-4o"}}}, nil) - - inMem := &mockInMemoryStore{ - configuredProviders: map[schemas.ModelProvider]configstore.ProviderConfig{ - schemas.OpenAI: {}, - schemas.Vertex: {}, - schemas.VLLM: {}, // opaque - }, - } - - // VK grants access to ONLY openai and vLLM — NOT vertex, even though vertex serves gpt-4o. - virtualKey := buildVirtualKeyWithProviders( - "vk-scoped", - "sk-bf-scoped-test", - "openai-vllm-only-vk", - []configstoreTables.TableVirtualKeyProviderConfig{ - buildProviderConfig("openai", []string{"*"}), - buildProviderConfig("vllm", []string{"*"}), - }, - ) - store, err := NewLocalGovernanceStore(context.Background(), logger, nil, &configstore.GovernanceConfig{ - VirtualKeys: []configstoreTables.TableVirtualKey{*virtualKey}, - }, mc) - require.NoError(t, err) - - plugin, err := InitFromStore(context.Background(), &Config{IsVkMandatory: boolPtr(false)}, logger, store, nil, mc, nil, inMem) - require.NoError(t, err) - defer func() { - require.NoError(t, plugin.Cleanup()) - }() - - req := schemas.AcquireHTTPRequest() - defer schemas.ReleaseHTTPRequest(req) - req.Method = "POST" - req.Path = "/v1/chat/completions" - req.Headers["Authorization"] = "Bearer sk-bf-scoped-test" - req.Headers["Content-Type"] = "application/json" - req.Body = []byte(`{"model":"gpt-4o","messages":[{"role":"user","content":"Hi"}]}`) - - bfCtx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) - resp, err := plugin.HTTPTransportPreHook(bfCtx, req) - require.NoError(t, err) - require.Nil(t, resp) - - allowedProviders, ok := bfCtx.Value(schemas.BifrostContextKeyAvailableProviders).([]schemas.ModelProvider) - require.True(t, ok, "available providers should be set") - t.Logf("AvailableProviders for gpt-4o (catalog: openai+vertex serve it; VK = openai + vllm only): %v", allowedProviders) - - // Vertex serves gpt-4o per the catalog but is NOT in the VK, so it must be absent. - require.ElementsMatch(t, []schemas.ModelProvider{schemas.OpenAI, schemas.VLLM}, allowedProviders) -} - -// TestHTTPTransportPreHook_WildcardOpaqueProviderRespectsBlacklist guards the ordering in -// loadBalanceProvider: the blacklist pre-pass must exclude a provider before the wildcard + -// catalog-opaque shortcut applies, so a blacklisted model on an opaque provider is dropped -// from BifrostContextKeyAvailableProviders even under a ["*"] allow-list. -func TestHTTPTransportPreHook_WildcardOpaqueProviderRespectsBlacklist(t *testing.T) { - logger := NewMockLogger() - - mc := modelcatalog.NewTestCatalog(nil) // no vLLM models -> opaque - - inMem := &mockInMemoryStore{ - configuredProviders: map[schemas.ModelProvider]configstore.ProviderConfig{ - schemas.VLLM: {}, - }, - } - - vllmConfig := buildProviderConfig("vllm", []string{"*"}) - vllmConfig.BlacklistedModels = schemas.BlackList{"my-self-hosted-llama"} - - virtualKey := buildVirtualKeyWithProviders( - "vk-vllm-bl", - "sk-bf-vllm-bl-test", - "vllm-bl-vk", - []configstoreTables.TableVirtualKeyProviderConfig{vllmConfig}, - ) - store, err := NewLocalGovernanceStore(context.Background(), logger, nil, &configstore.GovernanceConfig{ - VirtualKeys: []configstoreTables.TableVirtualKey{*virtualKey}, - }, mc) - require.NoError(t, err) - - plugin, err := InitFromStore(context.Background(), &Config{IsVkMandatory: boolPtr(false)}, logger, store, nil, mc, nil, inMem) - require.NoError(t, err) - defer func() { - require.NoError(t, plugin.Cleanup()) - }() - - req := schemas.AcquireHTTPRequest() - defer schemas.ReleaseHTTPRequest(req) - req.Method = "POST" - req.Path = "/v1/chat/completions" - req.Headers["Authorization"] = "Bearer sk-bf-vllm-bl-test" - req.Headers["Content-Type"] = "application/json" - req.Body = []byte(`{"model":"my-self-hosted-llama","messages":[{"role":"user","content":"Hi"}]}`) - - bfCtx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) - resp, err := plugin.HTTPTransportPreHook(bfCtx, req) - require.NoError(t, err) - require.Nil(t, resp) - - allowedProviders, ok := bfCtx.Value(schemas.BifrostContextKeyAvailableProviders).([]schemas.ModelProvider) - require.True(t, ok, "available providers should be set") - // Blacklisted model is excluded even though the provider is catalog-opaque under ["*"]. - require.Empty(t, allowedProviders) -} - -// TestHTTPTransportPreHook_GenAIRoutingRulePreservesTarget verifies that when a routing rule -// matches on the /genai path, governance load balancing does not override the routing-rule target -// with a provider from the VK pool (regression test for issue #2516). -func TestHTTPTransportPreHook_GenAIRoutingRulePreservesTarget(t *testing.T) { - logger := NewMockLogger() - - routingRule := configstoreTables.TableRoutingRule{ - ID: "rule-genai-1", - Name: "genai-repro-rule", - Enabled: bifrost.Ptr(true), - CelExpression: `model == "probe-genai-model" && provider == ""`, - Targets: []configstoreTables.TableRoutingTarget{ - { - RuleID: "rule-genai-1", - Provider: bifrost.Ptr("repro-openai-a"), - Model: bifrost.Ptr("error-test"), - Weight: 1.0, - }, - }, - Scope: "global", - Priority: 1, - } - - // VK with repro-openai-b at weight=1 — this is what governance LB would wrongly select without the fix - virtualKey := buildVirtualKeyWithProviders( - "vk-genai", - "sk-bf-genai-test", - "genai-repro-vk", - []configstoreTables.TableVirtualKeyProviderConfig{ - buildProviderConfig("repro-openai-b", []string{"*"}), - }, - ) - - store, err := NewLocalGovernanceStore(context.Background(), logger, nil, &configstore.GovernanceConfig{ - VirtualKeys: []configstoreTables.TableVirtualKey{*virtualKey}, - RoutingRules: []configstoreTables.TableRoutingRule{routingRule}, - }, nil) - require.NoError(t, err) - - plugin, err := InitFromStore(context.Background(), &Config{IsVkMandatory: boolPtr(false)}, logger, store, nil, nil, nil, nil) - require.NoError(t, err) - defer func() { - require.NoError(t, plugin.Cleanup()) - }() - - req := schemas.AcquireHTTPRequest() - defer schemas.ReleaseHTTPRequest(req) - req.Method = "POST" - req.Path = "/genai/v1beta/models/probe-genai-model:generateContent" - req.PathParams["model"] = "probe-genai-model:generateContent" - req.Headers["Authorization"] = "Bearer sk-bf-genai-test" - req.Headers["Content-Type"] = "application/json" - req.Body = []byte(`{"contents":[{"role":"user","parts":[{"text":"hi"}]}]}`) - - bfCtx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) - resp, err := plugin.HTTPTransportPreHook(bfCtx, req) - require.NoError(t, err) - require.Nil(t, resp) - - // Routing rule matched and set context model to "repro-openai-a/error-test:generateContent". - // Governance LB must NOT override this with "repro-openai-b/probe-genai-model:generateContent". - ctxModel, ok := bfCtx.Value("model").(string) - require.True(t, ok, "context model should be set") - require.Equal(t, "repro-openai-a/error-test:generateContent", ctxModel) -} - -// TestHTTPTransportPreHook_GenAIRoutingRulePreservesTarget_WithStore is a production-like variant -// of TestHTTPTransportPreHook_GenAIRoutingRulePreservesTarget that passes a non-nil inMemoryStore -// containing the routing-rule provider, confirming the fix holds when p.inMemoryStore != nil -// and the provider IS present in GetConfiguredProviders (the normal production code path). -func TestHTTPTransportPreHook_GenAIRoutingRulePreservesTarget_WithStore(t *testing.T) { - logger := NewMockLogger() - - routingRule := configstoreTables.TableRoutingRule{ - ID: "rule-genai-ws-1", - Name: "genai-repro-rule-with-store", - Enabled: bifrost.Ptr(true), - CelExpression: `model == "probe-genai-model" && provider == ""`, - Targets: []configstoreTables.TableRoutingTarget{ - { - RuleID: "rule-genai-ws-1", - Provider: bifrost.Ptr("repro-openai-a"), - Model: bifrost.Ptr("error-test"), - Weight: 1.0, - }, - }, - Scope: "global", - Priority: 1, - } - - virtualKey := buildVirtualKeyWithProviders( - "vk-genai-ws", - "sk-bf-genai-ws-test", - "genai-repro-vk-with-store", - []configstoreTables.TableVirtualKeyProviderConfig{ - buildProviderConfig("repro-openai-b", []string{"*"}), - }, - ) - - store, err := NewLocalGovernanceStore(context.Background(), logger, nil, &configstore.GovernanceConfig{ - VirtualKeys: []configstoreTables.TableVirtualKey{*virtualKey}, - RoutingRules: []configstoreTables.TableRoutingRule{routingRule}, - }, nil) - require.NoError(t, err) - - // Register the fake provider so ParseModelString can split "repro-openai-a/model" - // the same way it would for a real provider in production. - schemas.RegisterKnownProvider("repro-openai-a") - t.Cleanup(func() { schemas.UnregisterKnownProvider("repro-openai-a") }) - - // Use a non-nil inMemoryStore that recognises the routing-rule provider, - // mirroring production where configured providers are always registered in the store. - inMemStore := &mockInMemoryStore{ - configuredProviders: map[schemas.ModelProvider]configstore.ProviderConfig{ - "repro-openai-a": {}, - }, - } - - plugin, err := InitFromStore(context.Background(), &Config{IsVkMandatory: boolPtr(false)}, logger, store, nil, nil, nil, inMemStore) - require.NoError(t, err) - defer func() { - require.NoError(t, plugin.Cleanup()) - }() - - req := schemas.AcquireHTTPRequest() - defer schemas.ReleaseHTTPRequest(req) - req.Method = "POST" - req.Path = "/genai/v1beta/models/probe-genai-model:generateContent" - req.PathParams["model"] = "probe-genai-model:generateContent" - req.Headers["Authorization"] = "Bearer sk-bf-genai-ws-test" - req.Headers["Content-Type"] = "application/json" - req.Body = []byte(`{"contents":[{"role":"user","parts":[{"text":"hi"}]}]}`) - - bfCtx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) - resp, err := plugin.HTTPTransportPreHook(bfCtx, req) - require.NoError(t, err) - require.Nil(t, resp) - - ctxModel, ok := bfCtx.Value("model").(string) - require.True(t, ok, "context model should be set") - require.Equal(t, "repro-openai-a/error-test:generateContent", ctxModel) -} - -// TestHTTPTransportPreHook_GenAINoRoutingRuleStillLoadBalances verifies that when no routing rule -// matches on the /genai path, governance load balancing still selects a provider from the VK pool. -func TestHTTPTransportPreHook_GenAINoRoutingRuleStillLoadBalances(t *testing.T) { - logger := NewMockLogger() - - // VK with repro-openai-b at weight=1 — LB should select this - virtualKey := buildVirtualKeyWithProviders( - "vk-genai-lb", - "sk-bf-genai-lb-test", - "genai-lb-vk", - []configstoreTables.TableVirtualKeyProviderConfig{ - buildProviderConfig("repro-openai-b", []string{"*"}), - }, - ) - - store, err := NewLocalGovernanceStore(context.Background(), logger, nil, &configstore.GovernanceConfig{ - VirtualKeys: []configstoreTables.TableVirtualKey{*virtualKey}, - // No routing rules — governance LB should run normally - }, nil) - require.NoError(t, err) - - plugin, err := InitFromStore(context.Background(), &Config{IsVkMandatory: boolPtr(false)}, logger, store, nil, nil, nil, nil) - require.NoError(t, err) - defer func() { - require.NoError(t, plugin.Cleanup()) - }() - - req := schemas.AcquireHTTPRequest() - defer schemas.ReleaseHTTPRequest(req) - req.Method = "POST" - req.Path = "/genai/v1beta/models/probe-genai-model:generateContent" - req.PathParams["model"] = "probe-genai-model:generateContent" - req.Headers["Authorization"] = "Bearer sk-bf-genai-lb-test" - req.Headers["Content-Type"] = "application/json" - req.Body = []byte(`{"contents":[{"role":"user","parts":[{"text":"hi"}]}]}`) - - bfCtx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) - resp, err := plugin.HTTPTransportPreHook(bfCtx, req) - require.NoError(t, err) - require.Nil(t, resp) - - // No routing rule: governance LB must still run and select repro-openai-b from the VK pool - ctxModel, ok := bfCtx.Value("model").(string) - require.True(t, ok, "context model should be set by governance LB") - require.Equal(t, "repro-openai-b/probe-genai-model:generateContent", ctxModel) -} - -// TestHTTPTransportPreHook_BedrockRoutingRulePreservesTarget verifies that when a routing rule -// matches on the /bedrock path, governance load balancing does not override the routing-rule target -// (regression test mirroring the GenAI fix for the Bedrock integration). -func TestHTTPTransportPreHook_BedrockRoutingRulePreservesTarget(t *testing.T) { - logger := NewMockLogger() - - routingRule := configstoreTables.TableRoutingRule{ - ID: "rule-bedrock-1", - Name: "bedrock-repro-rule", - Enabled: bifrost.Ptr(true), - CelExpression: `model == "probe-bedrock-model" && provider == ""`, - Targets: []configstoreTables.TableRoutingTarget{ - { - RuleID: "rule-bedrock-1", - Provider: bifrost.Ptr("repro-openai-a"), - Model: bifrost.Ptr("error-test"), - Weight: 1.0, - }, - }, - Scope: "global", - Priority: 1, - } - - // VK with repro-openai-b at weight=1 — this is what governance LB would wrongly select without the fix - virtualKey := buildVirtualKeyWithProviders( - "vk-bedrock", - "sk-bf-bedrock-test", - "bedrock-repro-vk", - []configstoreTables.TableVirtualKeyProviderConfig{ - buildProviderConfig("repro-openai-b", []string{"*"}), - }, - ) - - store, err := NewLocalGovernanceStore(context.Background(), logger, nil, &configstore.GovernanceConfig{ - VirtualKeys: []configstoreTables.TableVirtualKey{*virtualKey}, - RoutingRules: []configstoreTables.TableRoutingRule{routingRule}, - }, nil) - require.NoError(t, err) - - plugin, err := InitFromStore(context.Background(), &Config{IsVkMandatory: boolPtr(false)}, logger, store, nil, nil, nil, nil) - require.NoError(t, err) - defer func() { - require.NoError(t, plugin.Cleanup()) - }() - - req := schemas.AcquireHTTPRequest() - defer schemas.ReleaseHTTPRequest(req) - req.Method = "POST" - req.Path = "/bedrock/model/probe-bedrock-model/converse" - req.PathParams["modelId"] = "probe-bedrock-model" - req.Headers["Authorization"] = "Bearer sk-bf-bedrock-test" - req.Headers["Content-Type"] = "application/json" - req.Body = []byte(`{"messages":[{"role":"user","content":[{"text":"hi"}]}]}`) - - bfCtx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) - resp, err := plugin.HTTPTransportPreHook(bfCtx, req) - require.NoError(t, err) - require.Nil(t, resp) - - // Routing rule matched and set context modelId to "repro-openai-a/error-test". - // Governance LB must NOT override this with "repro-openai-b/probe-bedrock-model". - ctxModelID, ok := bfCtx.Value("modelId").(string) - require.True(t, ok, "context modelId should be set") - require.Equal(t, "repro-openai-a/error-test", ctxModelID) -} - -// TestHTTPTransportPreHook_BedrockNoRoutingRuleStillLoadBalances verifies that when no routing rule -// matches on the /bedrock path, governance load balancing still selects a provider from the VK pool. -func TestHTTPTransportPreHook_BedrockNoRoutingRuleStillLoadBalances(t *testing.T) { - logger := NewMockLogger() - - // VK with repro-openai-b at weight=1 — LB should select this - virtualKey := buildVirtualKeyWithProviders( - "vk-bedrock-lb", - "sk-bf-bedrock-lb-test", - "bedrock-lb-vk", - []configstoreTables.TableVirtualKeyProviderConfig{ - buildProviderConfig("repro-openai-b", []string{"*"}), - }, - ) - - store, err := NewLocalGovernanceStore(context.Background(), logger, nil, &configstore.GovernanceConfig{ - VirtualKeys: []configstoreTables.TableVirtualKey{*virtualKey}, - // No routing rules — governance LB should run normally - }, nil) - require.NoError(t, err) - - plugin, err := InitFromStore(context.Background(), &Config{IsVkMandatory: boolPtr(false)}, logger, store, nil, nil, nil, nil) - require.NoError(t, err) - defer func() { - require.NoError(t, plugin.Cleanup()) - }() - - req := schemas.AcquireHTTPRequest() - defer schemas.ReleaseHTTPRequest(req) - req.Method = "POST" - req.Path = "/bedrock/model/probe-bedrock-model/converse" - req.PathParams["modelId"] = "probe-bedrock-model" - req.Headers["Authorization"] = "Bearer sk-bf-bedrock-lb-test" - req.Headers["Content-Type"] = "application/json" - req.Body = []byte(`{"messages":[{"role":"user","content":[{"text":"hi"}]}]}`) - - bfCtx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) - resp, err := plugin.HTTPTransportPreHook(bfCtx, req) - require.NoError(t, err) - require.Nil(t, resp) - - // No routing rule: governance LB must still run and select repro-openai-b from the VK pool - ctxModelID, ok := bfCtx.Value("modelId").(string) - require.True(t, ok, "context modelId should be set by governance LB") - require.Equal(t, "repro-openai-b/probe-bedrock-model", ctxModelID) -} diff --git a/plugins/governance/main.go b/plugins/governance/main.go index e2c61db700..c30a8a3185 100644 --- a/plugins/governance/main.go +++ b/plugins/governance/main.go @@ -6,22 +6,20 @@ import ( "errors" "fmt" "math/rand/v2" - "net/url" "sort" "strings" "sync" + "sync/atomic" "time" - "github.com/bytedance/sonic" "github.com/google/uuid" bifrost "github.com/maximhq/bifrost/core" - "github.com/maximhq/bifrost/core/network" - "github.com/maximhq/bifrost/core/providers/gemini" "github.com/maximhq/bifrost/core/schemas" "github.com/maximhq/bifrost/framework/configstore" configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" "github.com/maximhq/bifrost/framework/mcpcatalog" "github.com/maximhq/bifrost/framework/modelcatalog" + "github.com/maximhq/bifrost/plugins/governance/complexity" ) // PluginName is the name of the governance plugin @@ -88,6 +86,8 @@ type GovernancePlugin struct { requiredHeaders *[]string // pointer to live config slice; lowercased at check time isEnterprise bool disableAutoToolInject *bool + + complexityAnalyzer atomic.Pointer[complexity.ComplexityAnalyzer] } // Init initializes and returns a governance plugin instance. @@ -236,6 +236,7 @@ func Init( disableAutoToolInject: disableAutoToolInject, inMemoryStore: inMemoryStore, } + plugin.storeComplexityAnalyzerConfig(resolveAnalyzerConfigFromStoreOrArg(ctx, logger, configStore, governanceConfig)) return plugin, nil } @@ -330,6 +331,7 @@ func InitFromStore( isEnterprise: config != nil && config.IsEnterprise, disableAutoToolInject: disableAutoToolInject, } + plugin.storeComplexityAnalyzerConfig(resolveAnalyzerConfigFromStoreOrArg(ctx, logger, configStore, nil)) return plugin, nil } @@ -338,338 +340,117 @@ func (p *GovernancePlugin) GetName() string { return PluginName } -// UpdateEnforceAuthOnInference updates the enforce auth on inference config -func (p *GovernancePlugin) UpdateEnforceAuthOnInference(enforceAuthOnInference bool) { - p.cfgMutex.Lock() - defer p.cfgMutex.Unlock() - p.isVkMandatory = new(enforceAuthOnInference) +// ReloadComplexityAnalyzerConfig swaps the analyzer used by complexity_tier routing. +func (p *GovernancePlugin) ReloadComplexityAnalyzerConfig(config *complexity.AnalyzerConfig) { + p.storeComplexityAnalyzerConfig(config) } -// HTTPTransportPreHook intercepts requests before they are processed (governance decision point) -// It modifies the request in-place and returns nil to continue, or an HTTPResponse to short-circuit. -// Optimized to skip unnecessary operations: only unmarshals/marshals when needed -func (p *GovernancePlugin) HTTPTransportPreHook(ctx *schemas.BifrostContext, req *schemas.HTTPRequest) (*schemas.HTTPResponse, error) { - virtualKeyValue := parseVirtualKeyFromHTTPRequest(req) - hasRoutingRules := p.store.HasRoutingRules(ctx) - - if strings.Contains(req.Path, "passthrough") { - return nil, nil - } - - // If no virtual key and no routing rules configured, skip all processing - if virtualKeyValue == nil && !hasRoutingRules { - return nil, nil - } - - // If no body, check if the request carries a model via query params (e.g. realtime - // WebSocket upgrades: GET /v1/realtime?model=... or Azure preview ?deployment=...) - // or if large payload mode is active. - // For query-param-based models we build a synthetic payload so routing rules and VK - // load-balancing can rewrite provider/model, then propagate changes back to the query. - if len(req.Body) == 0 { - if modelParam := realtimeModelQueryParam(req); modelParam != "" { - return p.governRealtimeQueryParam(ctx, req, virtualKeyValue, hasRoutingRules) - } - isLargePayload, _ := ctx.Value(schemas.BifrostContextKeyLargePayloadMode).(bool) - if !isLargePayload { - return nil, nil - } - return p.governLargePayload(ctx, req, virtualKeyValue, hasRoutingRules) - } - - // Only unmarshal if we have VK or routing rules - var payload map[string]any - var virtualKey *configstoreTables.TableVirtualKey - var ok bool - var needsMarshal bool - - contentType := req.CaseInsensitiveHeaderLookup("Content-Type") - lowerCT := strings.ToLower(contentType) - // Strip parameters (e.g., "; charset=utf-8") for clean media type comparison - mediaType := lowerCT - if idx := strings.IndexByte(mediaType, ';'); idx >= 0 { - mediaType = strings.TrimSpace(mediaType[:idx]) - } - isMultipart := strings.HasPrefix(mediaType, "multipart/form-data") - isJSON := mediaType == "" || mediaType == "application/json" || strings.HasSuffix(mediaType, "+json") - - if !isMultipart && !isJSON { - // Non-parseable body (e.g., application/sdp for WebRTC signaling) — skip governance - return nil, nil - } - - var err error - if isMultipart { - payload, err = network.ParseMultipartFormFields(contentType, req.Body) - if err != nil { - p.logger.Warn("failed to parse multipart form in governance plugin: %v", err) - return nil, nil - } - } else { - err = sonic.Unmarshal(req.Body, &payload) - if err != nil { - p.logger.Error("failed to unmarshal request body: %v", err) - return nil, nil - } - } - - // Process virtual key if provided - if virtualKeyValue != nil { - virtualKey, ok = p.store.GetVirtualKey(ctx, *virtualKeyValue) - if !ok || virtualKey == nil || !virtualKey.IsActiveValue() { - return nil, nil - } - } - - // Attaching team and customer based on the virtual key - if virtualKey != nil { - if virtualKey.TeamID != nil { - ctx.SetValue(schemas.BifrostContextKeyGovernanceTeamID, *virtualKey.TeamID) - } - if virtualKey.Team != nil { - ctx.SetValue(schemas.BifrostContextKeyGovernanceTeamName, virtualKey.Team.Name) - } - if virtualKey.CustomerID != nil { - ctx.SetValue(schemas.BifrostContextKeyGovernanceCustomerID, *virtualKey.CustomerID) - } - if virtualKey.Customer != nil { - ctx.SetValue(schemas.BifrostContextKeyGovernanceCustomerName, virtualKey.Customer.Name) +func (p *GovernancePlugin) storeComplexityAnalyzerConfig(config *complexity.AnalyzerConfig) { + resolved, err := complexity.ValidateAndNormalize(config) + if err != nil { + if p.logger != nil { + p.logger.Warn("invalid complexity analyzer config, using defaults: %v", err) } + defaults := complexity.DefaultAnalyzerConfig() + resolved = &defaults } + p.complexityAnalyzer.Store(complexity.NewComplexityAnalyzerWithConfig(resolved)) +} - //1. Apply routing rules only if we have rules or matched decision - var routingDecision *RoutingDecision - if hasRoutingRules { - var err error - payload, routingDecision, err = p.applyRoutingRules(ctx, req, payload, virtualKey) +func resolveAnalyzerConfigFromStoreOrArg( + ctx context.Context, + logger schemas.Logger, + configStore configstore.ConfigStore, + governanceConfig *configstore.GovernanceConfig, +) *complexity.AnalyzerConfig { + if governanceConfig != nil && governanceConfig.ComplexityAnalyzerConfig != nil { + cfg, err := complexity.ValidateAndNormalize(governanceConfig.ComplexityAnalyzerConfig) if err != nil { - return nil, err - } - // Mark for marshal if a routing rule matched - if routingDecision != nil { - needsMarshal = true + if logger != nil { + logger.Warn("invalid complexity analyzer config from governance config: %v", err) + } + } else if cfg != nil { + return cfg } } - - // Process virtual key if provided - if virtualKey != nil { - //2. Load balance provider - payload, err = p.loadBalanceProvider(ctx, req, payload, virtualKey) + if configStore != nil { + cfg, err := configStore.GetComplexityAnalyzerConfig(ctx) if err != nil { - return nil, err - } - //3. Add MCP tools only when auto-inject is enabled and header not already set by the caller - p.cfgMutex.RLock() - autoInjectDisabled := p.disableAutoToolInject != nil && *p.disableAutoToolInject - p.cfgMutex.RUnlock() - if !autoInjectDisabled { - // Treat an explicitly-present (even empty) x-bf-mcp-include-tools header as "present" - // so that callers can block auto-injection by sending an empty header value. - headerPresent := false - for k := range req.Headers { - if strings.EqualFold(k, "x-bf-mcp-include-tools") { - headerPresent = true - break - } - } - if !headerPresent { - req.Headers, err = p.addMCPIncludeTools(req.Headers, virtualKey) - if err != nil { - p.logger.Error("failed to add MCP include tools: %v", err) - return nil, nil - } + if logger != nil { + logger.Warn("failed to load complexity analyzer config from store: %v", err) } + } else if cfg != nil { + return cfg } - needsMarshal = true } + return nil +} - // Only marshal if something changed (VK processing or routing decision matched) - if needsMarshal { - if err := network.SerializePayloadToRequest(req, payload, isMultipart, contentType); err != nil { - p.logger.Error("failed to serialize request body in governance plugin: %v", err) - return nil, nil - } - } +// UpdateEnforceAuthOnInference updates the enforce auth on inference config +func (p *GovernancePlugin) UpdateEnforceAuthOnInference(enforceAuthOnInference bool) { + p.cfgMutex.Lock() + defer p.cfgMutex.Unlock() + p.isVkMandatory = new(enforceAuthOnInference) +} +// HTTPTransportPreHook is retained as a no-op so governance still satisfies the +// HTTPTransportPlugin interface (used by the enterprise wrapper's 503 gate delegation). +// All routing now flows through PreRequestHook: body-having requests via handleRequest, +// large-payload requests via PreRequestHook reading LargePayloadMetadata, and realtime WS +// upgrades via the realtime handler's explicit RunPreRequestHooks call. +func (p *GovernancePlugin) HTTPTransportPreHook(ctx *schemas.BifrostContext, req *schemas.HTTPRequest) (*schemas.HTTPResponse, error) { return nil, nil } -// governLargePayload handles read-only governance for large payload requests. -// The request body is streaming and cannot be modified, so we build a synthetic payload -// from pre-extracted metadata and run VK validation, routing rules, and load balancing. -// Any model changes are propagated via the metadata in context (not body rewriting). -func (p *GovernancePlugin) governLargePayload(ctx *schemas.BifrostContext, req *schemas.HTTPRequest, virtualKeyValue *string, hasRoutingRules bool) (*schemas.HTTPResponse, error) { - metadata, _ := ctx.Value(schemas.BifrostContextKeyLargePayloadMetadata).(*schemas.LargePayloadMetadata) - if metadata == nil || metadata.Model == "" { - return nil, nil - } - - // Build synthetic payload from metadata — only the model field is needed - payload := map[string]any{ - "model": metadata.Model, +// runPreRequestRouting wraps a model string in a synthetic BifrostRequest, runs the same +// applyRoutingRules + loadBalanceProvider helpers used by the main PreRequestHook path, and +// returns the resolved model (provider-prefixed when a provider was selected, plain model +// otherwise). Used by PreRequestHook's large-payload branch where req.Model is empty because +// the body wasn't parsed. +func (p *GovernancePlugin) runPreRequestRouting(ctx *schemas.BifrostContext, virtualKey *configstoreTables.TableVirtualKey, hasRoutingRules bool, modelIn string, requestType schemas.RequestType) (string, error) { + // Parse a provider-prefixed model string the same way the transport does for + // body-having requests, so an explicit prefix like "openai/gpt-4o" lands in + // ChatRequest.Provider and load balancing honors the caller's routing intent. + providerIn, parsedModel := schemas.ParseModelString(modelIn, "") + synthetic := &schemas.BifrostRequest{ + RequestType: requestType, + ChatRequest: &schemas.BifrostChatRequest{Provider: providerIn, Model: parsedModel}, } - originalModel := metadata.Model - // Process virtual key if provided - var virtualKey *configstoreTables.TableVirtualKey - if virtualKeyValue != nil { - vk, ok := p.store.GetVirtualKey(ctx, *virtualKeyValue) - if !ok || vk == nil || !vk.IsActiveValue() { - return nil, nil + if hasRoutingRules { + if _, err := p.applyRoutingRules(ctx, synthetic, virtualKey); err != nil { + return modelIn, err } - virtualKey = vk } - // Attaching team and customer based on the virtual key if virtualKey != nil { - if virtualKey.TeamID != nil { - ctx.SetValue(schemas.BifrostContextKeyGovernanceTeamID, *virtualKey.TeamID) - } - if virtualKey.Team != nil { - ctx.SetValue(schemas.BifrostContextKeyGovernanceTeamName, virtualKey.Team.Name) - } - if virtualKey.CustomerID != nil { - ctx.SetValue(schemas.BifrostContextKeyGovernanceCustomerID, *virtualKey.CustomerID) - } - if virtualKey.Customer != nil { - ctx.SetValue(schemas.BifrostContextKeyGovernanceCustomerName, virtualKey.Customer.Name) + if err := p.loadBalanceProvider(ctx, synthetic, virtualKey); err != nil { + return modelIn, err } - } - // Apply routing rules (read-only: decisions still affect downstream evaluation) - if hasRoutingRules { - var err error - payload, _, err = p.applyRoutingRules(ctx, req, payload, virtualKey) - if err != nil { - return nil, err - } - } + // A caller-provided include-tools list can only narrow the virtual key's + // tool grant, never expand it — prune entries the key does not allow. + includeToolsProvided := p.pruneMCPIncludeToolsFromContext(ctx, virtualKey) - // Process virtual key: load balance + MCP tool headers - if virtualKey != nil { - var err error - payload, err = p.loadBalanceProvider(ctx, req, payload, virtualKey) - if err != nil { - return nil, err - } - // MCP tool headers — apply the same auto-inject guard as the normal path: - // skip when DisableAutoToolInject is set or the caller already sent the header. p.cfgMutex.RLock() autoInjectDisabled := p.disableAutoToolInject != nil && *p.disableAutoToolInject p.cfgMutex.RUnlock() - if !autoInjectDisabled { - headerPresent := false - for k := range req.Headers { - if strings.EqualFold(k, "x-bf-mcp-include-tools") { - headerPresent = true - break - } - } - if !headerPresent { - req.Headers, err = p.addMCPIncludeTools(req.Headers, virtualKey) - if err != nil { - p.logger.Error("failed to add MCP include tools: %v", err) - return nil, nil - } + // An include-clients filter opts the request into tool injection even when + // auto-injection is disabled (see ParseAndAddToolsToRequest in core/mcp), so + // the key's allowlist must be stamped on every path where injection can run. + includeClientsPresent := ctx.Value(schemas.MCPContextKeyIncludeClients) != nil + if !includeToolsProvided && (!autoInjectDisabled || includeClientsPresent) { + if tools := p.computeMCPIncludeTools(virtualKey); tools != nil { + ctx.SetValue(schemas.MCPContextKeyIncludeTools, tools) } } } - // Propagate model changes to metadata so downstream hydration picks up - // the load-balanced/routed model (e.g., provider prefix added by LB). - if newModel, ok := payload["model"].(string); ok && newModel != originalModel { - metadata.Model = newModel - } - - // No body serialization — large payload body streams through unchanged - return nil, nil -} - -// realtimeModelQueryParam returns the query parameter used as the realtime model selector. -// Azure preview realtime uses `deployment`, while GA/OpenAI-compatible paths use `model`. -func realtimeModelQueryParam(req *schemas.HTTPRequest) string { - if req == nil || req.Query == nil { - return "" - } - if modelParam := req.Query["model"]; modelParam != "" { - return modelParam + provider, model, _ := synthetic.GetRequestFields() + if provider != "" { + return string(provider) + "/" + model, nil } - return req.Query["deployment"] -} - -// governRealtimeQueryParam handles governance for bodyless realtime requests -// (e.g. WebSocket upgrade GET /v1/realtime?model=... or Azure preview -// /realtime?deployment=...) where the model lives in a query parameter instead -// of the JSON body. We build a synthetic payload so routing rules and VK -// load-balancing can evaluate normally, then propagate any model rewrite back -// to the original query param for the downstream handler to pick up. -func (p *GovernancePlugin) governRealtimeQueryParam(ctx *schemas.BifrostContext, req *schemas.HTTPRequest, virtualKeyValue *string, hasRoutingRules bool) (*schemas.HTTPResponse, error) { - modelQueryKey := "model" - modelParam := req.Query[modelQueryKey] - if modelParam == "" { - modelQueryKey = "deployment" - modelParam = req.Query[modelQueryKey] - } - if modelParam == "" { - return nil, nil - } - - payload := map[string]any{ - "model": modelParam, - } - originalModel := modelParam - - // Process virtual key if provided - var virtualKey *configstoreTables.TableVirtualKey - if virtualKeyValue != nil { - vk, ok := p.store.GetVirtualKey(ctx, *virtualKeyValue) - if !ok || vk == nil || !vk.IsActiveValue() { - return nil, nil - } - virtualKey = vk - } - - // Attaching team and customer based on the virtual key - if virtualKey != nil { - if virtualKey.TeamID != nil { - ctx.SetValue(schemas.BifrostContextKeyGovernanceTeamID, *virtualKey.TeamID) - } - if virtualKey.Team != nil { - ctx.SetValue(schemas.BifrostContextKeyGovernanceTeamName, virtualKey.Team.Name) - } - if virtualKey.CustomerID != nil { - ctx.SetValue(schemas.BifrostContextKeyGovernanceCustomerID, *virtualKey.CustomerID) - } - if virtualKey.Customer != nil { - ctx.SetValue(schemas.BifrostContextKeyGovernanceCustomerName, virtualKey.Customer.Name) - } - } - - // Apply routing rules - if hasRoutingRules { - var err error - payload, _, err = p.applyRoutingRules(ctx, req, payload, virtualKey) - if err != nil { - return nil, err - } - } - - // Process virtual key: load balance provider - if virtualKey != nil { - var err error - payload, err = p.loadBalanceProvider(ctx, req, payload, virtualKey) - if err != nil { - return nil, err - } - } - - // Propagate model changes back to the original query param so the downstream - // realtime handler sees the routed/load-balanced model. - if newModel, ok := payload["model"].(string); ok && newModel != originalModel { - req.Query[modelQueryKey] = newModel - } - - return nil, nil + return model, nil } // HTTPTransportPostHook intercepts requests after they are processed (governance decision point) @@ -683,77 +464,18 @@ func (p *GovernancePlugin) HTTPTransportStreamChunkHook(ctx *schemas.BifrostCont return chunk, nil } -// loadBalanceProvider loads balances the provider for the request -// Parameters: -// - req: The HTTP request -// - body: The request body -// - virtualKey: The virtual key configuration -// -// Returns: -// - map[string]any: The updated request body -// - error: Any error that occurred during processing -func (p *GovernancePlugin) loadBalanceProvider(ctx *schemas.BifrostContext, req *schemas.HTTPRequest, body map[string]any, virtualKey *configstoreTables.TableVirtualKey) (map[string]any, error) { - // Check if the request has a model field - modelValue, hasModel := body["model"] - isGeminiPath := strings.Contains(req.Path, "/genai") - isBedrockPath := strings.Contains(req.Path, "/bedrock") - if !hasModel { - // For genai integration, model is present in URL path instead of the request body - if isGeminiPath { - // Prefer context value set by a routing rule (format: "provider/model:suffix") - if ctxModel, ok := ctx.Value("model").(string); ok && ctxModel != "" { - modelValue = ctxModel - } else { - modelValue = req.CaseInsensitivePathParamLookup("model") - } - } else if isBedrockPath { - // For bedrock integration, model is present in URL path as modelId - // Prefer context value set by a routing rule (format: "provider/model") - if ctxModelID, ok := ctx.Value("modelId").(string); ok && ctxModelID != "" { - modelValue = ctxModelID - } else { - rawModelID := req.CaseInsensitivePathParamLookup("modelId") - if rawModelID == "" { - return body, nil - } - // URL-decode the modelId (Bedrock model IDs may be URL-encoded, e.g. anthropic%2Fclaude-3-5-sonnet) - decoded, err := url.PathUnescape(rawModelID) - if err != nil { - decoded = rawModelID - } - modelValue = decoded - } - } else { - return body, nil - } - } - modelStr, ok := modelValue.(string) - if !ok || modelStr == "" { - return body, nil - } - var genaiRequestSuffix string - // Remove Google GenAI API endpoint suffixes if present - if isGeminiPath { - for _, sfx := range gemini.GeminiRequestSuffixPaths { - if before, ok := strings.CutSuffix(modelStr, sfx); ok { - modelStr = before - genaiRequestSuffix = sfx - break - } - } +// loadBalanceProvider picks a weighted provider from the VK's configs for req.Model +// and mutates req.Provider/req.Model with the refined provider/model. Also populates req.Fallbacks +// from the remaining weighted providers if no fallbacks were configured by the caller. +func (p *GovernancePlugin) loadBalanceProvider(ctx *schemas.BifrostContext, req *schemas.BifrostRequest, virtualKey *configstoreTables.TableVirtualKey) error { + provider, modelStr, existingFallbacks := req.GetRequestFields() + if modelStr == "" { + return nil } - // Check if model already has provider prefix (contains "/") - if strings.Contains(modelStr, "/") { - provider, _ := schemas.ParseModelString(modelStr, "") - // Checking valid provider when store is available; if store is nil, - // assume the prefixed model should be left unchanged. - if p.inMemoryStore != nil { - if _, ok := p.inMemoryStore.GetConfiguredProviders()[provider]; ok { - return body, nil - } - } else { - return body, nil - } + + if provider != "" { + ctx.AppendRoutingEngineLog(schemas.RoutingEngineGovernance, schemas.LogLevelInfo, fmt.Sprintf("Skipping load balancing for model %s: provider %s already set", modelStr, provider)) + return nil } ctx.AppendRoutingEngineLog(schemas.RoutingEngineGovernance, schemas.LogLevelInfo, fmt.Sprintf("Load balancing provider for model %s", modelStr)) @@ -761,10 +483,9 @@ func (p *GovernancePlugin) loadBalanceProvider(ctx *schemas.BifrostContext, req // Get provider configs for this virtual key providerConfigs := virtualKey.ProviderConfigs if len(providerConfigs) == 0 { - ctx.SetValue(schemas.BifrostContextKeyAvailableProviders, []schemas.ModelProvider{}) ctx.AppendRoutingEngineLog(schemas.RoutingEngineGovernance, schemas.LogLevelWarn, fmt.Sprintf("No provider configs on virtual key %s for model %s, skipping load balancing", virtualKey.Name, modelStr)) // No provider configs, continue without modification - return body, nil + return nil } var configuredProviders []string @@ -777,7 +498,7 @@ func (p *GovernancePlugin) loadBalanceProvider(ctx *schemas.BifrostContext, req // Pre-pass: if any config for a provider blacklists the model, that provider is fully blocked. blacklistedProviders := make(map[string]bool) for _, config := range providerConfigs { - if isModelBlockedByList(config.BlacklistedModels, modelStr) { + if config.BlacklistedModels.IsBlocked(modelStr) { blacklistedProviders[config.Provider] = true } } @@ -825,12 +546,9 @@ func (p *GovernancePlugin) loadBalanceProvider(ctx *schemas.BifrostContext, req } var allowedProviders []string - allowedModelProviders := make([]schemas.ModelProvider, 0, len(allowedProviderConfigs)) for _, pc := range allowedProviderConfigs { allowedProviders = append(allowedProviders, pc.Provider) - allowedModelProviders = append(allowedModelProviders, schemas.ModelProvider(pc.Provider)) } - ctx.SetValue(schemas.BifrostContextKeyAvailableProviders, allowedModelProviders) p.logger.Debug("[Governance] Allowed providers after filtering: %v", allowedProviders) ctx.AppendRoutingEngineLog(schemas.RoutingEngineGovernance, schemas.LogLevelInfo, fmt.Sprintf("Allowed providers after filtering: %v", allowedProviders)) @@ -838,9 +556,9 @@ func (p *GovernancePlugin) loadBalanceProvider(ctx *schemas.BifrostContext, req ctx.AppendRoutingEngineLog(schemas.RoutingEngineGovernance, schemas.LogLevelInfo, fmt.Sprintf("No eligible providers remaining after filtering for model %s, skipping load balancing", modelStr)) // TODO: Send proper error if (overall VK budget/rate limit) or (all provider budgets/rate limits) are violated // No allowed provider configs, continue without modification - return body, nil + return nil } - // Separate providers with weight set (participate in routing) from those without (nil weight = excluded from routing) + weightedConfigs := make([]configstoreTables.TableVirtualKeyProviderConfig, 0, len(allowedProviderConfigs)) for _, config := range allowedProviderConfigs { if config.Weight != nil { @@ -848,268 +566,240 @@ func (p *GovernancePlugin) loadBalanceProvider(ctx *schemas.BifrostContext, req } } - var selectedProvider schemas.ModelProvider + if len(weightedConfigs) == 0 { + // All allowed configs survived the model-allowance / budget / rate-limit filters, + // but none of them have a Weight set — there's nothing to feed weighted selection. + // Emit an explicit log so the routing trail explains why governance stops here + // instead of trailing off after "Allowed providers after filtering: [...]". + ctx.AppendRoutingEngineLog(schemas.RoutingEngineGovernance, schemas.LogLevelInfo, fmt.Sprintf("No weighted configs for model %s — none of the allowed VK provider configs have a weight assigned; skipping load balancing", modelStr)) + return nil + } - if len(weightedConfigs) > 0 { - // Weighted random selection from providers that have weight set - totalWeight := 0.0 - for _, config := range weightedConfigs { - totalWeight += getWeight(config.Weight) - } - // Generate random number between 0 and totalWeight - randomValue := rand.Float64() * totalWeight - // Select provider based on weighted random selection - currentWeight := 0.0 - for _, config := range weightedConfigs { - currentWeight += getWeight(config.Weight) - if randomValue <= currentWeight { - selectedProvider = schemas.ModelProvider(config.Provider) - break - } - } - // Fallback: if no provider was selected (shouldn't happen but guard against FP issues) - if selectedProvider == "" { - selectedProvider = schemas.ModelProvider(weightedConfigs[0].Provider) + var selectedProvider schemas.ModelProvider + totalWeight := 0.0 + for _, config := range weightedConfigs { + totalWeight += getWeight(config.Weight) + } + // Generate random number between 0 and totalWeight + randomValue := rand.Float64() * totalWeight + // Select provider based on weighted random selection + currentWeight := 0.0 + for _, config := range weightedConfigs { + currentWeight += getWeight(config.Weight) + if randomValue <= currentWeight { + selectedProvider = schemas.ModelProvider(config.Provider) + break } - } else { - // No providers have weight set - return body, nil + } + // Fallback: if no provider was selected (shouldn't happen but guard against FP issues) + if selectedProvider == "" { + selectedProvider = schemas.ModelProvider(weightedConfigs[0].Provider) } p.logger.Debug("[governance] Selected provider: %s", selectedProvider) ctx.AppendRoutingEngineLog(schemas.RoutingEngineGovernance, schemas.LogLevelInfo, fmt.Sprintf("Selected provider %s for model %s (from %d eligible: %v)", selectedProvider, modelStr, len(allowedProviderConfigs), allowedProviders)) - // For genai integration, model is present in URL path instead of the request body - if isGeminiPath { - newModelWithRequestSuffix := string(selectedProvider) + "/" + modelStr + genaiRequestSuffix - ctx.SetValue("model", newModelWithRequestSuffix) - } else if isBedrockPath { - // For bedrock integration, model is present in URL path as modelId - ctx.SetValue("modelId", string(selectedProvider)+"/"+modelStr) - } else { + refinedModel := modelStr + // Refine the model for the selected provider + if p.modelCatalog != nil { var err error - refinedModel := modelStr - // Refine the model for the selected provider - if p.modelCatalog != nil { - refinedModel, err = p.modelCatalog.RefineModelForProvider(selectedProvider, modelStr) - if err != nil { - return body, err - } + refinedModel, err = p.modelCatalog.RefineModelForProvider(selectedProvider, modelStr) + if err != nil { + return err } - // Update the model field in the request body - body["model"] = string(selectedProvider) + "/" + refinedModel } - // Append governance to routing engines used + + req.SetProvider(selectedProvider) + req.SetModel(refinedModel) + schemas.AppendToContextList(ctx, schemas.BifrostContextKeyRoutingEnginesUsed, schemas.RoutingEngineGovernance) - // Check if fallbacks field is already present - _, hasFallbacks := body["fallbacks"] - // Use the same candidate set that was used for primary selection - fallbackConfigs := weightedConfigs - if !hasFallbacks && len(fallbackConfigs) > 1 { - // Sort fallback configs by weight (descending) + if len(existingFallbacks) == 0 && len(weightedConfigs) > 1 { + fallbackConfigs := append([]configstoreTables.TableVirtualKeyProviderConfig(nil), weightedConfigs...) sort.Slice(fallbackConfigs, func(i, j int) bool { return getWeight(fallbackConfigs[i].Weight) > getWeight(fallbackConfigs[j].Weight) }) // Filter out the selected provider and create fallbacks array - fallbacks := make([]string, 0, len(fallbackConfigs)-1) + fallbacks := make([]schemas.Fallback, 0, len(fallbackConfigs)-1) for _, config := range fallbackConfigs { - if config.Provider != string(selectedProvider) { - var err error - refinedModel := modelStr - if p.modelCatalog != nil { - refinedModel, err = p.modelCatalog.RefineModelForProvider(schemas.ModelProvider(config.Provider), modelStr) - if err != nil { - // Skip fallback if model refinement fails - p.logger.Warn("failed to refine model for fallback, skipping fallback in governance plugin: %v", err) - continue - } + if config.Provider == string(selectedProvider) { + continue + } + fbProvider := schemas.ModelProvider(config.Provider) + fbModel := modelStr + if p.modelCatalog != nil { + refined, err := p.modelCatalog.RefineModelForProvider(fbProvider, modelStr) + if err != nil { + p.logger.Warn("failed to refine model for fallback, skipping fallback in governance plugin: %v", err) + ctx.AppendRoutingEngineLog(schemas.RoutingEngineGovernance, schemas.LogLevelWarn, fmt.Sprintf("Fallback provider %s skipped: failed to refine model %s for this provider", fbProvider, modelStr)) + continue } - fallbacks = append(fallbacks, string(schemas.ModelProvider(config.Provider))+"/"+refinedModel) + fbModel = refined } + fallbacks = append(fallbacks, schemas.Fallback{Provider: fbProvider, Model: fbModel}) } - - // Add fallbacks to request body - body["fallbacks"] = fallbacks - ctx.AppendRoutingEngineLog(schemas.RoutingEngineGovernance, schemas.LogLevelInfo, fmt.Sprintf("Added %d fallback providers: %v", len(fallbacks), fallbacks)) + req.SetFallbacks(fallbacks) + ctx.AppendRoutingEngineLog(schemas.RoutingEngineGovernance, schemas.LogLevelInfo, fmt.Sprintf("Added %d fallback providers", len(fallbacks))) } - return body, nil + return nil } -// applyRoutingRules evaluates routing rules and returns both the modified payload AND the routing decision -// This allows the caller to determine if marshaling is necessary (only if decision != nil or payload changed) -// Parameters: -// - ctx: Bifrost context -// - req: HTTP request -// - body: Request body (may be modified if routing rule matches) -// - virtualKey: Virtual key configuration (may be nil) +// publishRoutingAllowlist records, for downstream routing layers, which of the VK's configured +// providers permit modelStr according to the VK's own allowed_models / blocked_models. It is a +// coarse provider gate (BifrostContextKeyRoutingAllowedProviders) layered on top of the model +// catalog checks those layers already run — its purpose is to stop a later routing layer (load +// balancing, model-catalog resolution) from selecting a provider the VK forbids for this model, +// even when governance itself couldn't pick one. An empty slice means "no provider is permitted" +// (fail-closed via the empty-provider validation in handleRequest); a nil VK publishes nothing. // -// Returns: -// - map[string]any: The potentially modified request body -// - *RoutingDecision: The matched routing decision (nil if no rule matched) -// - error: Any error that occurred during evaluation -func (p *GovernancePlugin) applyRoutingRules(ctx *schemas.BifrostContext, req *schemas.HTTPRequest, body map[string]any, virtualKey *configstoreTables.TableVirtualKey) (map[string]any, *RoutingDecision, error) { - // Check if the request has a model field - modelValue, hasModel := body["model"] - isGeminiPath := strings.Contains(req.Path, "/genai") - isBedrockPath := strings.Contains(req.Path, "/bedrock") - if !hasModel { - // For genai integration, model is present in URL path - if isGeminiPath { - modelValue = req.CaseInsensitivePathParamLookup("model") - } else if isBedrockPath { - // For bedrock integration, model is present in URL path as modelId - rawModelID := req.CaseInsensitivePathParamLookup("modelId") - if rawModelID == "" { - return body, nil, nil - } - // URL-decode the modelId (Bedrock model IDs may be URL-encoded) - decoded, err := url.PathUnescape(rawModelID) - if err != nil { - decoded = rawModelID - } - modelValue = decoded - } else { - return body, nil, nil - } - } +// Provider prefixes on the request model are already split into req.Provider + bare model at the +// HTTP layer (resolveModelAndProvider), so VK allowed_models / blocked_models are matched against +// bare names and plain membership checks are sufficient here. +func (p *GovernancePlugin) publishRoutingAllowlist(ctx *schemas.BifrostContext, virtualKey *configstoreTables.TableVirtualKey, modelStr string) { + if virtualKey == nil { + return + } + allowed := make([]schemas.ModelProvider, 0, len(virtualKey.ProviderConfigs)) + for _, pc := range virtualKey.ProviderConfigs { + // No model to filter on → keep the provider so we don't over-restrict. + if modelStr == "" || + (pc.AllowedModels.IsAllowed(modelStr) && !pc.BlacklistedModels.IsBlocked(modelStr)) { + allowed = append(allowed, schemas.ModelProvider(pc.Provider)) + } + } + ctx.SetValue(schemas.BifrostContextKeyRoutingAllowedProviders, allowed) +} - modelStr, ok := modelValue.(string) - if !ok || modelStr == "" { - return body, nil, nil +// applyRoutingRules evaluates routing rules against req and mutates +// req.Provider/req.Model/req.Fallbacks when a rule matches. Returns the matched RoutingDecision +// (nil if no rule matched). Integrations normalize req.Model (and Provider when applicable) before +// the BifrostRequest reaches this point. +func (p *GovernancePlugin) applyRoutingRules(ctx *schemas.BifrostContext, req *schemas.BifrostRequest, virtualKey *configstoreTables.TableVirtualKey) (*RoutingDecision, error) { + provider, model, _ := req.GetRequestFields() + if model == "" { + return nil, nil } - var genaiRequestSuffix string - if strings.Contains(req.Path, "/genai") { - for _, sfx := range gemini.GeminiRequestSuffixPaths { - if before, ok := strings.CutSuffix(modelStr, sfx); ok { - modelStr = before - genaiRequestSuffix = sfx - break - } - } - } + requestType := string(req.RequestType) + headers, _ := ctx.Value(schemas.BifrostContextKeyRequestHeaders).(map[string]string) + queryParams, _ := ctx.Value(schemas.BifrostContextKeyRequestQuery).(map[string]string) - // Parse provider and model from modelStr (format: "provider/model" or just "model") - provider, model := schemas.ParseModelString(modelStr, "") + // Set up lazy complexity computation; only runs if a rule references complexity_tier. + var computeComplexity func() *complexity.ComplexityResult + if analyzer := p.complexityAnalyzer.Load(); analyzer != nil { + computeComplexity = func() *complexity.ComplexityResult { + input, ok := buildComplexityInput(req) + if !ok { + if p.logger != nil { + p.logger.Debug("[Governance] Complexity analysis skipped: unsupported request type") + } + ctx.AppendRoutingEngineLog(schemas.RoutingEngineRoutingRule, schemas.LogLevelInfo, "Complexity analysis skipped: no supported text-bearing input detected") + return nil + } - // Extract normalized request type from context (set by HTTP middleware) - requestType := "" - if val := ctx.Value(schemas.BifrostContextKeyHTTPRequestType); val != nil { - if requestTypeEnum, ok := val.(schemas.RequestType); ok { - requestType = string(requestTypeEnum) - } else if requestTypeStr, ok := val.(string); ok { - requestType = requestTypeStr + result := analyzer.Analyze(input) + if p.logger != nil { + p.logger.Debug( + "[Governance] Complexity analysis details: tier=%s score=%.2f words=%d", + result.Tier, + result.Score, + result.WordCount, + ) + } + ctx.AppendRoutingEngineLog( + schemas.RoutingEngineRoutingRule, + schemas.LogLevelInfo, + fmt.Sprintf("Complexity: tier=%s score=%.2f words=%d", result.Tier, result.Score, result.WordCount), + ) + return result } } - // Build routing context routingCtx := &RoutingContext{ VirtualKey: virtualKey, Provider: provider, Model: model, RequestType: requestType, - Headers: req.Headers, - QueryParams: req.Query, + Headers: headers, + QueryParams: queryParams, BudgetAndRateLimitStatus: p.store.GetBudgetAndRateLimitStatus(ctx, model, provider, virtualKey, nil, nil, nil), + computeComplexity: computeComplexity, } - p.logger.Debug("[HTTPTransport] Built routing context: provider=%s, model=%s, requestType=%s, vk=%v, headerCount=%d, paramCount=%d", - provider, model, requestType, virtualKey != nil, len(req.Headers), len(req.Query)) + p.logger.Debug("[PreRequestHook] Built routing context: provider=%s, model=%s, requestType=%s, vk=%v", + provider, model, requestType, virtualKey != nil) // Evaluate routing rules decision, err := p.engine.EvaluateRoutingRules(ctx, routingCtx) if err != nil { p.logger.Error("failed to evaluate routing rules: %v", err) ctx.AppendRoutingEngineLog(schemas.RoutingEngineRoutingRule, schemas.LogLevelError, fmt.Sprintf("Routing rule evaluation error: %v", err)) - return body, nil, nil + return nil, nil + } + if decision == nil { + return nil, nil } - // If a routing rule matched, apply the decision - if decision != nil { - p.logger.Debug("[Governance] Routing rule matched: %s", decision.MatchedRuleName) + p.logger.Debug("[Governance] Routing rule matched: %s", decision.MatchedRuleName) - // Update model in request body - if strings.Contains(req.Path, "/genai") { - // For genai, model is in URL path - newModel := decision.Model + genaiRequestSuffix - // Add provider prefix if present (because there can be other routing rules down stream that can add the provider) - if decision.Provider != "" { - newModel = decision.Provider + "/" + newModel - } - ctx.SetValue("model", newModel) - } else if isBedrockPath { - // For bedrock, model is in URL path as modelId - // Set new modelId in context so bedrockPreCallback picks it up via ctx.UserValue("modelId") - newModel := decision.Model - if decision.Provider != "" { - newModel = decision.Provider + "/" + newModel - } - ctx.SetValue("modelId", newModel) - } else { - // For regular requests, update in body - newModel := decision.Model - // Add provider prefix if present (because there can be other routing rules down stream that can add the provider) - if decision.Provider != "" { - newModel = decision.Provider + "/" + newModel + if decision.Provider != "" { + req.SetProvider(schemas.ModelProvider(decision.Provider)) + } + if decision.Model != "" { + req.SetModel(decision.Model) + } + + schemas.AppendToContextList(ctx, schemas.BifrostContextKeyRoutingEnginesUsed, schemas.RoutingEngineRoutingRule) + + // Add fallbacks if present; fill in the incoming model for fallbacks that omit it + if len(decision.Fallbacks) > 0 { + resolvedFallbacks := make([]schemas.Fallback, 0, len(decision.Fallbacks)) + for _, fb := range decision.Fallbacks { + fbProvider, fbModel := schemas.ParseModelString(fb, "") + trimmedFbProvider := strings.TrimSpace(string(fbProvider)) + trimmedFbModel := strings.TrimSpace(fbModel) + if trimmedFbProvider == "" { + continue } - body["model"] = newModel - } - // Append routing-rule to routing engines used - schemas.AppendToContextList(ctx, schemas.BifrostContextKeyRoutingEnginesUsed, schemas.RoutingEngineRoutingRule) - - // Add fallbacks if present; fill in the incoming model for fallbacks that omit it - if len(decision.Fallbacks) > 0 { - resolvedFallbacks := make([]string, 0, len(decision.Fallbacks)) - for _, fb := range decision.Fallbacks { - fbProvider, fbModel := schemas.ParseModelString(fb, "") - trimmedFbProvider := strings.TrimSpace(string(fbProvider)) - trimmedFbModel := strings.TrimSpace(fbModel) - if trimmedFbProvider == "" { - continue - } - if trimmedFbModel == "" && model != "" { - resolvedFallbacks = append(resolvedFallbacks, trimmedFbProvider+"/"+model) - } else { - resolvedFallbacks = append(resolvedFallbacks, trimmedFbProvider+"/"+trimmedFbModel) - } + if trimmedFbModel == "" && model != "" { + trimmedFbModel = model } - body["fallbacks"] = resolvedFallbacks + resolvedFallbacks = append(resolvedFallbacks, schemas.Fallback{ + Provider: schemas.ModelProvider(trimmedFbProvider), + Model: trimmedFbModel, + }) } - - // Pin specific API key by ID if the routing rule specifies one - if decision.KeyID != "" { - ctx.SetValue(schemas.BifrostContextKeyAPIKeyID, decision.KeyID) - } - - p.logger.Debug("[Governance] Applied routing decision: provider=%s, model=%s, keyID=%s, fallbacks=%v", decision.Provider, decision.Model, decision.KeyID, decision.Fallbacks) + req.SetFallbacks(resolvedFallbacks) } - return body, decision, nil -} - -// addMCPIncludeTools adds the x-bf-mcp-include-tools header to the request headers -// Parameters: -// - headers: The request headers -// - virtualKey: The virtual key configuration -// -// Returns: -// - map[string]string: The updated request headers -// - error: Any error that occurred during processing -func (p *GovernancePlugin) addMCPIncludeTools(headers map[string]string, virtualKey *configstoreTables.TableVirtualKey) (map[string]string, error) { - if headers == nil { - headers = make(map[string]string) + // Pin specific API key by ID if the routing rule specifies one + if decision.KeyID != "" { + ctx.SetValue(schemas.BifrostContextKeyAPIKeyID, decision.KeyID) } - executeOnlyTools := make([]string, 0) + p.logger.Debug("[Governance] Applied routing decision: provider=%s, model=%s, keyID=%s, fallbacks=%v", decision.Provider, decision.Model, decision.KeyID, decision.Fallbacks) + return decision, nil +} - // Build a lookup of AllowOnAllVirtualKeys clients: clientID -> clientName +// computeMCPIncludeTools builds the MCP include-tools list for a virtual key. Returns the list +// directly; callers store it via ctx.SetValue(schemas.MCPContextKeyIncludeTools, ...). VK-specific +// MCP configs take precedence over AllowOnAllVirtualKeys clients. +func (p *GovernancePlugin) computeMCPIncludeTools(virtualKey *configstoreTables.TableVirtualKey) []string { var allowAllVKsClients map[string]string if p.inMemoryStore != nil { allowAllVKsClients = p.inMemoryStore.GetMCPClientsAllowingAllVirtualKeys() } + return p.computeMCPIncludeToolsWith(virtualKey, allowAllVKsClients) +} + +// computeMCPIncludeToolsWith is the computeMCPIncludeTools variant taking a pre-fetched +// AllowOnAllVirtualKeys map (clientID → clientName), so callers that make multiple +// grant decisions per request can evaluate them all against one consistent snapshot. +func (p *GovernancePlugin) computeMCPIncludeToolsWith(virtualKey *configstoreTables.TableVirtualKey, allowAllVKsClients map[string]string) []string { + executeOnlyTools := make([]string, 0) + if allowAllVKsClients == nil { allowAllVKsClients = make(map[string]string) } @@ -1145,39 +835,68 @@ func (p *GovernancePlugin) addMCPIncludeTools(headers map[string]string, virtual } } - // Set even when empty to exclude tools when no tools are present in the virtual key config - headers["x-bf-mcp-include-tools"] = strings.Join(executeOnlyTools, ",") - - return headers, nil + return executeOnlyTools } -// validateRequiredHeaders checks that all configured required headers are present in the request. -// Headers are compared case-insensitively (both sides lowercased). -// Returns a BifrostError with status 400 if any required headers are missing, or nil if all present. -func (p *GovernancePlugin) validateRequiredHeaders(ctx *schemas.BifrostContext) *schemas.BifrostError { - if p.requiredHeaders == nil || len(*p.requiredHeaders) == 0 { - return nil +// pruneMCPIncludeToolsFromContext narrows a caller-provided include-tools list (stamped on ctx +// from the x-bf-mcp-include-tools header in lib/ctx.go) down to the tools the virtual key +// allows, and writes the pruned list back to ctx. Returns true when a caller list was present, +// regardless of how many entries survived. Entries the key does not grant are dropped; a +// "client-*" wildcard is kept only when the key itself is unrestricted for that client, +// otherwise it is replaced by the key's specific grants for that client (passing the wildcard +// through would read downstream as "all tools of this client"). +func (p *GovernancePlugin) pruneMCPIncludeToolsFromContext(ctx *schemas.BifrostContext, virtualKey *configstoreTables.TableVirtualKey) bool { + existing := ctx.Value(schemas.MCPContextKeyIncludeTools) + if existing == nil { + return false + } + requested, _ := existing.([]string) + + // Fetch the AllowOnAllVirtualKeys snapshot once so the wildcard checks (via vkSet) + // and the per-tool checks (via isMCPToolAllowedByVKWith) can't observe different + // states across a concurrent config reload. + var allowAllClients map[string]string + if p.inMemoryStore != nil { + allowAllClients = p.inMemoryStore.GetMCPClientsAllowingAllVirtualKeys() } - headers, _ := ctx.Value(schemas.BifrostContextKeyRequestHeaders).(map[string]string) - if headers == nil { - headers = map[string]string{} + + vkTools := p.computeMCPIncludeToolsWith(virtualKey, allowAllClients) + vkSet := make(map[string]struct{}, len(vkTools)) + for _, tool := range vkTools { + vkSet[tool] = struct{}{} } - var missing []string - for _, h := range *p.requiredHeaders { - if _, ok := headers[strings.ToLower(h)]; !ok { - missing = append(missing, h) + + pruned := make([]string, 0, len(requested)) + seen := make(map[string]struct{}, len(requested)) + add := func(tool string) { + if _, dup := seen[tool]; !dup { + seen[tool] = struct{}{} + pruned = append(pruned, tool) } } - if len(missing) > 0 { - return &schemas.BifrostError{ - Type: bifrost.Ptr("missing_required_headers"), - StatusCode: bifrost.Ptr(400), - Error: &schemas.ErrorField{ - Message: fmt.Sprintf("missing required headers: %s", strings.Join(missing, ", ")), - }, + for _, pattern := range requested { + if pattern == "" { + continue + } + if clientName, isWildcard := strings.CutSuffix(pattern, "-*"); isWildcard { + if _, ok := vkSet[pattern]; ok { + add(pattern) + continue + } + for _, tool := range vkTools { + if strings.HasPrefix(tool, clientName+"-") { + add(tool) + } + } + continue + } + if p.isMCPToolAllowedByVKWith(virtualKey, pattern, allowAllClients) { + add(pattern) } } - return nil + + ctx.SetValue(schemas.MCPContextKeyIncludeTools, pruned) + return true } // EvaluateGovernanceRequest is a common function that handles virtual key validation @@ -1440,6 +1159,93 @@ func (p *GovernancePlugin) isMCPToolAllowedByVKWith(vk *configstoreTables.TableV return false } +// PreRequestHook is the per-request governance phase. It runs for both normal body-having +// requests (route on req.Model) and large-payload streaming requests (route on +// LargePayloadMetadata.Model from ctx — the body is opaque mid-stream, so routing is +// constrained to same-protocol-family targets that the upstream provider can hydrate +// from the rewritten metadata). +// +// Realtime + generic streaming bypass handleRequest (see core/bifrost.go +// RunRealtimeTurnPreHooks / RunStreamPreHooks) and are still handled at HTTPTransportPreHook. +func (p *GovernancePlugin) PreRequestHook(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) error { + if req.RequestType == schemas.PassthroughRequest || req.RequestType == schemas.PassthroughStreamRequest { + return nil + } + + virtualKeyValue := bifrost.GetStringFromContext(ctx, schemas.BifrostContextKeyVirtualKey) + hasRoutingRules := p.store.HasRoutingRules(ctx) + if virtualKeyValue == "" && !hasRoutingRules { + return nil + } + + var virtualKey *configstoreTables.TableVirtualKey + if virtualKeyValue != "" { + var ok bool + virtualKey, ok = p.store.GetVirtualKey(ctx, virtualKeyValue) + if !ok || virtualKey == nil || !virtualKey.IsActiveValue() { + return nil + } + } + + stampGovernanceCtxFromVK(ctx, virtualKey) + + // Large-payload mode: the body streams to the provider unparsed, so req.Model is + // empty for routes where the model lives in the body (OpenAI/Anthropic chat, + // responses, etc.). Route on LargePayloadMetadata.Model — the provider's + // streaming body rewriter (ApplyLargePayloadRequestBodyWithModelNormalization) + // reads metadata.Model when it rewrites the model field in the body prefix, so + // mutating it here is what propagates the routing decision to the upstream call. + if metadata, _ := ctx.Value(schemas.BifrostContextKeyLargePayloadMetadata).(*schemas.LargePayloadMetadata); metadata != nil && metadata.Model != "" { + newModel, err := p.runPreRequestRouting(ctx, virtualKey, hasRoutingRules, metadata.Model, req.RequestType) + if err != nil { + return err + } + if newModel != "" && newModel != metadata.Model { + metadata.Model = newModel + } + _, routedModel := schemas.ParseModelString(metadata.Model, "") + p.publishRoutingAllowlist(ctx, virtualKey, routedModel) + return nil + } + + if hasRoutingRules { + if _, err := p.applyRoutingRules(ctx, req, virtualKey); err != nil { + return err + } + } + + // Publish the VK provider allowlist for the (post routing-rules) model so downstream routing + // layers (load balancing, model-catalog resolution) and core enforcement intersect their + // candidates with it — a later layer must not select a provider the VK forbids for this model. + _, routedModel, _ := req.GetRequestFields() + p.publishRoutingAllowlist(ctx, virtualKey, routedModel) + + if virtualKey != nil { + if err := p.loadBalanceProvider(ctx, req, virtualKey); err != nil { + return err + } + + // A caller-provided include-tools list can only narrow the virtual key's + // tool grant, never expand it — prune entries the key does not allow. + includeToolsProvided := p.pruneMCPIncludeToolsFromContext(ctx, virtualKey) + + p.cfgMutex.RLock() + autoInjectDisabled := p.disableAutoToolInject != nil && *p.disableAutoToolInject + p.cfgMutex.RUnlock() + // An include-clients filter opts the request into tool injection even when + // auto-injection is disabled (see ParseAndAddToolsToRequest in core/mcp), so + // the key's allowlist must be stamped on every path where injection can run. + includeClientsPresent := ctx.Value(schemas.MCPContextKeyIncludeClients) != nil + if !includeToolsProvided && (!autoInjectDisabled || includeClientsPresent) { + if tools := p.computeMCPIncludeTools(virtualKey); tools != nil { + ctx.SetValue(schemas.MCPContextKeyIncludeTools, tools) + } + } + } + + return nil +} + // PreLLMHook intercepts requests before they are processed (governance decision point) // Parameters: // - ctx: The Bifrost context diff --git a/plugins/governance/prerequesthookcomplexity_test.go b/plugins/governance/prerequesthookcomplexity_test.go new file mode 100644 index 0000000000..b8b17e115d --- /dev/null +++ b/plugins/governance/prerequesthookcomplexity_test.go @@ -0,0 +1,132 @@ +package governance + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/maximhq/bifrost/core/schemas" + "github.com/maximhq/bifrost/framework/configstore" + configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" +) + +func TestPreRequestHook_ComplexityAnalyzerFeedsCELVariable(t *testing.T) { + logger := NewMockLogger() + provider := "openai" + model := "gpt-4o-mini" + + plugin, err := Init( + context.Background(), + &Config{IsVkMandatory: boolPtr(false)}, + logger, + nil, + &configstore.GovernanceConfig{ + RoutingRules: []configstoreTables.TableRoutingRule{ + { + ID: "rule-1", + Name: "Complexity Available", + CelExpression: `complexity_tier != ""`, + Targets: []configstoreTables.TableRoutingTarget{ + {Provider: &provider, Model: &model, Weight: 1.0}, + }, + Enabled: schemas.Ptr(true), + Scope: "global", + Priority: 0, + }, + }, + }, + nil, + nil, + nil, + ) + require.NoError(t, err) + defer func() { + require.NoError(t, plugin.Cleanup()) + }() + + req := &schemas.BifrostRequest{ + RequestType: schemas.ChatCompletionRequest, + ChatRequest: &schemas.BifrostChatRequest{ + Provider: schemas.OpenAI, + Model: "gpt-4o", + Input: []schemas.ChatMessage{ + { + Role: schemas.ChatMessageRoleUser, + Content: complexityChatString("What is a vector database?"), + }, + }, + }, + } + + bfCtx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + require.NoError(t, plugin.PreRequestHook(bfCtx, req)) + + engines, ok := bfCtx.Value(schemas.BifrostContextKeyRoutingEnginesUsed).([]string) + require.True(t, ok, "routing engines used should be tracked") + require.Contains(t, engines, schemas.RoutingEngineRoutingRule) + + providerOut, modelOut, _ := req.GetRequestFields() + require.Equal(t, schemas.OpenAI, providerOut) + require.Equal(t, "gpt-4o-mini", modelOut) +} + +func TestPreRequestHook_ComplexitySkippedWhenNoRulesReferenceIt(t *testing.T) { + logger := NewMockLogger() + provider := "openai" + model := "gpt-4o-mini" + + plugin, err := Init( + context.Background(), + &Config{IsVkMandatory: boolPtr(false)}, + logger, + nil, + &configstore.GovernanceConfig{ + RoutingRules: []configstoreTables.TableRoutingRule{ + { + ID: "rule-1", + Name: "Always match", + CelExpression: "true", + Targets: []configstoreTables.TableRoutingTarget{ + {Provider: &provider, Model: &model, Weight: 1.0}, + }, + Enabled: schemas.Ptr(true), + Scope: "global", + Priority: 0, + }, + }, + }, + nil, + nil, + nil, + ) + require.NoError(t, err) + defer func() { + require.NoError(t, plugin.Cleanup()) + }() + + req := &schemas.BifrostRequest{ + RequestType: schemas.ChatCompletionRequest, + ChatRequest: &schemas.BifrostChatRequest{ + Provider: schemas.OpenAI, + Model: "gpt-4o", + Input: []schemas.ChatMessage{ + { + Role: schemas.ChatMessageRoleUser, + Content: complexityChatString("Hello"), + }, + }, + }, + } + + bfCtx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + require.NoError(t, plugin.PreRequestHook(bfCtx, req)) + + logs := bfCtx.GetRoutingEngineLogs() + for _, entry := range logs { + if entry.Engine == schemas.RoutingEngineRoutingRule && strings.Contains(entry.Message, "Complexity") { + t.Fatalf("expected no complexity logs when no rules reference complexity_tier, got: %s", entry.Message) + } + } +} diff --git a/plugins/governance/prerequesthookmcp_test.go b/plugins/governance/prerequesthookmcp_test.go new file mode 100644 index 0000000000..112ce95df9 --- /dev/null +++ b/plugins/governance/prerequesthookmcp_test.go @@ -0,0 +1,408 @@ +// This suite covers PreRequestHook's MCP include-tools stamping: how a caller-provided +// x-bf-mcp-include-tools list is pruned against the virtual key's tool grant, and when +// the grant itself is stamped onto the context for downstream injection. The rules under +// test, for both the normal request path and the large-payload branch: +// +// - A caller include-tools list is always pruned to the grant (it can only narrow, +// never expand), regardless of the DisableAutoToolInject setting. +// - When no caller list is present, the grant is stamped if auto-injection is enabled +// OR an include-clients filter is present — the latter because any explicit MCP +// filter opts the request into injection downstream even when auto-inject is off. +// - With no filters and auto-injection disabled, nothing is stamped and no injection +// occurs. +// - Deny-all is always an explicit empty list; an unset key means "no filtering" and +// would expose every available tool. +package governance + +import ( + "context" + "fmt" + "testing" + + "github.com/maximhq/bifrost/core/schemas" + "github.com/maximhq/bifrost/framework/configstore" + configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const mcpTestVKValue = "sk-bf-mcp-test" + +// buildVKForMCPStamping returns an active VK with an openai provider config (so load +// balancing has a provider pool) and an explicit sentry MCP config granting the given +// tools. Passing nil yields a VK with no MCP configs at all, which is semantically +// different from passing an empty slice (an explicit deny-all config for the client). +func buildVKForMCPStamping(tools []string) *configstoreTables.TableVirtualKey { + vk := buildVirtualKeyWithProviders( + "vk-mcp-stamp", + mcpTestVKValue, + "mcp-stamp-vk", + []configstoreTables.TableVirtualKeyProviderConfig{ + buildProviderConfig("openai", []string{"*"}), + }, + ) + if tools != nil { + vk.MCPConfigs = []configstoreTables.TableVirtualKeyMCPConfig{ + { + MCPClient: configstoreTables.TableMCPClient{ClientID: "client-1", Name: "sentry"}, + ToolsToExecute: tools, + }, + } + } + return vk +} + +// newPluginForMCPStamping builds a governance plugin around a single VK with the +// given DisableAutoToolInject setting. +func newPluginForMCPStamping(t *testing.T, vk *configstoreTables.TableVirtualKey, disableAutoToolInject bool) *GovernancePlugin { + t.Helper() + logger := NewMockLogger() + store, err := NewLocalGovernanceStore(context.Background(), logger, nil, &configstore.GovernanceConfig{ + VirtualKeys: []configstoreTables.TableVirtualKey{*vk}, + }, nil) + require.NoError(t, err) + + plugin, err := InitFromStore(context.Background(), &Config{ + IsVkMandatory: boolPtr(false), + DisableAutoToolInject: boolPtr(disableAutoToolInject), + }, logger, store, nil, nil, nil, nil) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, plugin.Cleanup()) }) + return plugin +} + +// newPreRequestCtx returns a ctx carrying the test VK plus optional caller MCP filters, +// mirroring what lib/ctx.go stamps from the x-bf-mcp-include-tools and +// x-bf-mcp-include-clients request headers. +func newPreRequestCtx(includeTools, includeClients []string) *schemas.BifrostContext { + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + ctx.SetValue(schemas.BifrostContextKeyVirtualKey, mcpTestVKValue) + if includeTools != nil { + ctx.SetValue(schemas.MCPContextKeyIncludeTools, includeTools) + } + if includeClients != nil { + ctx.SetValue(schemas.MCPContextKeyIncludeClients, includeClients) + } + return ctx +} + +// newChatRequest returns a chat request with the provider already resolved, so +// PreRequestHook's load-balancing step is a no-op and only MCP stamping is exercised. +func newChatRequest() *schemas.BifrostRequest { + return &schemas.BifrostRequest{ + RequestType: schemas.ChatCompletionRequest, + ChatRequest: &schemas.BifrostChatRequest{Provider: schemas.OpenAI, Model: "gpt-4o"}, + } +} + +// stampedIncludeTools runs PreRequestHook and returns the resulting include-tools ctx +// value (nil when nothing was stamped). +func stampedIncludeTools(t *testing.T, p *GovernancePlugin, ctx *schemas.BifrostContext, req *schemas.BifrostRequest) []string { + t.Helper() + require.NoError(t, p.PreRequestHook(ctx, req)) + value := ctx.Value(schemas.MCPContextKeyIncludeTools) + if value == nil { + return nil + } + tools, ok := value.([]string) + require.True(t, ok, "include-tools ctx value should be a []string") + return tools +} + +// ============================================================================ +// Decision matrix: include-tools present/absent × include-clients present/absent +// × DisableAutoToolInject on/off +// ============================================================================ + +// Baseline auto-injection: no caller filters, auto-injection enabled. Governance stamps +// the key's full tool grant (every granted tool, client-prefixed) onto the context, and +// downstream injection attaches exactly these tools to the outgoing request. +func TestPreRequestHookMCP_AutoInjectOn_NoFilters_StampsGrant(t *testing.T) { + p := newPluginForMCPStamping(t, buildVKForMCPStamping([]string{"tool_a", "tool_b"}), false) + ctx := newPreRequestCtx(nil, nil) + + tools := stampedIncludeTools(t, p, ctx, newChatRequest()) + assert.Equal(t, []string{"sentry-tool_a", "sentry-tool_b"}, tools) +} + +// The caller filters by client only and sends no include-tools list. Governance still +// stamps the full grant: the grant is the tool-level ceiling, and the client-level +// narrowing is applied downstream where the include-clients filter intersects with it. +func TestPreRequestHookMCP_AutoInjectOn_IncludeClientsOnly_StampsGrant(t *testing.T) { + p := newPluginForMCPStamping(t, buildVKForMCPStamping([]string{"tool_a", "tool_b"}), false) + ctx := newPreRequestCtx(nil, []string{"sentry"}) + + tools := stampedIncludeTools(t, p, ctx, newChatRequest()) + assert.Equal(t, []string{"sentry-tool_a", "sentry-tool_b"}, tools) +} + +// The caller sends an include-tools list with one granted and one ungranted tool. +// Governance prunes the list in place instead of replacing it: the granted entry +// survives, the ungranted entry is dropped, and the full grant is NOT stamped over +// the caller's narrower selection. +func TestPreRequestHookMCP_AutoInjectOn_IncludeToolsPresent_Prunes(t *testing.T) { + p := newPluginForMCPStamping(t, buildVKForMCPStamping([]string{"tool_a", "tool_b"}), false) + ctx := newPreRequestCtx([]string{"sentry-tool_a", "sentry-tool_c"}, nil) + + tools := stampedIncludeTools(t, p, ctx, newChatRequest()) + assert.Equal(t, []string{"sentry-tool_a"}, tools, + "granted entry survives, ungranted entry is pruned, grant is not re-expanded") +} + +// With auto-injection disabled and no caller filters, the request has not opted into +// MCP tools in any way: governance must leave the include-tools key unset so downstream +// injection sees neither filter and skips entirely. This is the contract of the +// DisableAutoToolInject setting. +func TestPreRequestHookMCP_AutoInjectOff_NoFilters_StampsNothing(t *testing.T) { + p := newPluginForMCPStamping(t, buildVKForMCPStamping([]string{"tool_a", "tool_b"}), true) + ctx := newPreRequestCtx(nil, nil) + + tools := stampedIncludeTools(t, p, ctx, newChatRequest()) + assert.Nil(t, tools, "no filters + auto-inject disabled must leave include-tools unset") +} + +// With auto-injection disabled, an include-clients filter still opts the request into +// injection downstream (any explicit MCP filter counts as opt-in). The grant must +// therefore be stamped even though auto-inject is off — left unset, the request would +// be injected with every tool of the included client, bypassing the key's grant. +func TestPreRequestHookMCP_AutoInjectOff_IncludeClientsOnly_StampsGrant(t *testing.T) { + p := newPluginForMCPStamping(t, buildVKForMCPStamping([]string{"tool_a", "tool_b"}), true) + ctx := newPreRequestCtx(nil, []string{"sentry"}) + + tools := stampedIncludeTools(t, p, ctx, newChatRequest()) + assert.Equal(t, []string{"sentry-tool_a", "sentry-tool_b"}, tools, + "include-clients triggers injection downstream, so the VK ceiling must be stamped") +} + +// Pruning is independent of the auto-inject setting: a caller include-tools list is +// narrowed against the grant even when auto-injection is disabled, because the list +// itself opts the request into injection downstream. +func TestPreRequestHookMCP_AutoInjectOff_IncludeToolsPresent_Prunes(t *testing.T) { + p := newPluginForMCPStamping(t, buildVKForMCPStamping([]string{"tool_a", "tool_b"}), true) + ctx := newPreRequestCtx([]string{"sentry-tool_b", "sentry-tool_c"}, nil) + + tools := stampedIncludeTools(t, p, ctx, newChatRequest()) + assert.Equal(t, []string{"sentry-tool_b"}, tools) +} + +// When the caller sends both filters, the pruned include-tools list is the final value: +// the presence of include-clients must not cause the full grant to overwrite the +// caller's narrower selection. Verified under both DisableAutoToolInject values. +func TestPreRequestHookMCP_BothFilters_PrunedListWins(t *testing.T) { + for _, disabled := range []bool{false, true} { + t.Run(fmt.Sprintf("disableAutoToolInject=%v", disabled), func(t *testing.T) { + p := newPluginForMCPStamping(t, buildVKForMCPStamping([]string{"tool_a", "tool_b"}), disabled) + ctx := newPreRequestCtx([]string{"sentry-tool_a"}, []string{"sentry"}) + + tools := stampedIncludeTools(t, p, ctx, newChatRequest()) + assert.Equal(t, []string{"sentry-tool_a"}, tools, + "pruned caller list must not be overwritten by the grant") + }) + } +} + +// ============================================================================ +// Grant-shape edge cases +// ============================================================================ + +// A key with no MCP configs at all has an empty effective grant. When include-clients +// opts the request into injection, governance must stamp an explicit empty list +// (deny-all) rather than leave the key unset — an unset key reads downstream as +// "no filtering" and would inject every available tool of the included client. +func TestPreRequestHookMCP_NoGrants_IncludeClients_StampsDenyAll(t *testing.T) { + p := newPluginForMCPStamping(t, buildVKForMCPStamping(nil), true) + ctx := newPreRequestCtx(nil, []string{"sentry"}) + + tools := stampedIncludeTools(t, p, ctx, newChatRequest()) + require.NotNil(t, tools, "deny-all must be an empty list, not an unset key") + assert.Empty(t, tools) +} + +// Same deny-all outcome as the no-configs case, but through a different path in +// computeMCPIncludeTools: here the key has an explicit MCP config for the client whose +// tools list is empty, exercising the ToolsToExecute.IsEmpty guard that skips the +// client without emitting any grant entries. +func TestPreRequestHookMCP_ExplicitEmptyGrant_IncludeClients_StampsDenyAll(t *testing.T) { + p := newPluginForMCPStamping(t, buildVKForMCPStamping([]string{}), true) + ctx := newPreRequestCtx(nil, []string{"sentry"}) + + tools := stampedIncludeTools(t, p, ctx, newChatRequest()) + require.NotNil(t, tools, "deny-all must be an empty list, not an unset key") + assert.Empty(t, tools) +} + +// An unrestricted ("*") grant is stamped as the client-scoped wildcard "sentry-*", +// which downstream filtering reads as "all tools of this client" — the grant never +// expands to other clients. +func TestPreRequestHookMCP_UnrestrictedGrant_StampsWildcard(t *testing.T) { + p := newPluginForMCPStamping(t, buildVKForMCPStamping([]string{"*"}), false) + ctx := newPreRequestCtx(nil, nil) + + tools := stampedIncludeTools(t, p, ctx, newChatRequest()) + assert.Equal(t, []string{"sentry-*"}, tools) +} + +// The caller requests a tool of a client the key has no grants for. Pruning drops it +// and stamps an explicit empty list, so the request cannot reach another client's +// tools just by naming them in the header. +func TestPreRequestHookMCP_IncludeToolsForUngrantedClient_PrunesToDenyAll(t *testing.T) { + p := newPluginForMCPStamping(t, buildVKForMCPStamping([]string{"tool_a"}), false) + ctx := newPreRequestCtx([]string{"github-list_repos"}, nil) + + tools := stampedIncludeTools(t, p, ctx, newChatRequest()) + require.NotNil(t, tools) + assert.Empty(t, tools) +} + +// An empty x-bf-mcp-include-tools header value reaches ctx as [""] (see lib/ctx.go). +// Pruning drops the empty entry and stamps deny-all: a caller can suppress tool +// injection for a single request, but cannot gain access through the empty value. +func TestPreRequestHookMCP_EmptyIncludeToolsHeader_DenyAll(t *testing.T) { + p := newPluginForMCPStamping(t, buildVKForMCPStamping([]string{"tool_a", "tool_b"}), false) + ctx := newPreRequestCtx([]string{""}, nil) + + tools := stampedIncludeTools(t, p, ctx, newChatRequest()) + require.NotNil(t, tools) + assert.Empty(t, tools) +} + +// ============================================================================ +// Paths that must NOT stamp +// ============================================================================ + +// Without a virtual key on ctx (and no routing rules configured), PreRequestHook +// returns before any MCP handling: there is no grant to enforce, so caller filters +// pass through untouched for downstream layers to interpret. +func TestPreRequestHookMCP_NoVirtualKey_NoStamping(t *testing.T) { + p := newPluginForMCPStamping(t, buildVKForMCPStamping([]string{"tool_a"}), false) + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + ctx.SetValue(schemas.MCPContextKeyIncludeClients, []string{"sentry"}) + + require.NoError(t, p.PreRequestHook(ctx, newChatRequest())) + assert.Nil(t, ctx.Value(schemas.MCPContextKeyIncludeTools)) +} + +// An inactive key is treated as absent: PreRequestHook returns before MCP handling +// and stamps nothing. +func TestPreRequestHookMCP_InactiveVK_NoStamping(t *testing.T) { + vk := buildVKForMCPStamping([]string{"tool_a"}) + inactive := false + vk.IsActive = &inactive + p := newPluginForMCPStamping(t, vk, false) + ctx := newPreRequestCtx(nil, []string{"sentry"}) + + require.NoError(t, p.PreRequestHook(ctx, newChatRequest())) + assert.Nil(t, ctx.Value(schemas.MCPContextKeyIncludeTools)) +} + +// Passthrough request types skip governance entirely, including MCP stamping. +func TestPreRequestHookMCP_PassthroughRequest_NoStamping(t *testing.T) { + p := newPluginForMCPStamping(t, buildVKForMCPStamping([]string{"tool_a"}), false) + ctx := newPreRequestCtx(nil, []string{"sentry"}) + req := &schemas.BifrostRequest{RequestType: schemas.PassthroughRequest} + + require.NoError(t, p.PreRequestHook(ctx, req)) + assert.Nil(t, ctx.Value(schemas.MCPContextKeyIncludeTools)) +} + +// ============================================================================ +// Large-payload branch (runPreRequestRouting) applies the same stamping rules +// ============================================================================ + +// newLargePayloadCtx returns a ctx that routes PreRequestHook through its large-payload +// branch: the body streams to the provider unparsed, so the model comes from +// LargePayloadMetadata instead of the request. The metadata pointer is returned so +// tests can assert the routed model is propagated (the streaming body rewriter +// consumes metadata.Model when rewriting the body prefix). +func newLargePayloadCtx(includeTools, includeClients []string) (*schemas.BifrostContext, *schemas.LargePayloadMetadata) { + ctx := newPreRequestCtx(includeTools, includeClients) + metadata := &schemas.LargePayloadMetadata{Model: "openai/gpt-4o"} + ctx.SetValue(schemas.BifrostContextKeyLargePayloadMetadata, metadata) + return ctx, metadata +} + +// Large-payload counterpart of the include-clients opt-in case: with auto-injection +// disabled, the grant must still be stamped because include-clients triggers injection +// downstream. The provider-prefixed model must survive routing unchanged. +func TestPreRequestHookMCP_LargePayload_AutoInjectOff_IncludeClients_StampsGrant(t *testing.T) { + p := newPluginForMCPStamping(t, buildVKForMCPStamping([]string{"tool_a", "tool_b"}), true) + ctx, metadata := newLargePayloadCtx(nil, []string{"sentry"}) + + tools := stampedIncludeTools(t, p, ctx, newChatRequest()) + assert.Equal(t, []string{"sentry-tool_a", "sentry-tool_b"}, tools) + assert.Equal(t, "openai/gpt-4o", metadata.Model, "provider-prefixed model must survive routing unchanged") +} + +// Large-payload counterpart of the baseline auto-injection case: no caller filters, +// auto-injection enabled, full grant stamped. +func TestPreRequestHookMCP_LargePayload_AutoInjectOn_NoFilters_StampsGrant(t *testing.T) { + p := newPluginForMCPStamping(t, buildVKForMCPStamping([]string{"tool_a", "tool_b"}), false) + ctx, metadata := newLargePayloadCtx(nil, nil) + + tools := stampedIncludeTools(t, p, ctx, newChatRequest()) + assert.Equal(t, []string{"sentry-tool_a", "sentry-tool_b"}, tools) + assert.Equal(t, "openai/gpt-4o", metadata.Model, "provider-prefixed model must survive routing unchanged") +} + +// Large-payload counterpart of the include-clients-only case with auto-injection +// enabled: the full grant is stamped as the tool-level ceiling. +func TestPreRequestHookMCP_LargePayload_AutoInjectOn_IncludeClientsOnly_StampsGrant(t *testing.T) { + p := newPluginForMCPStamping(t, buildVKForMCPStamping([]string{"tool_a", "tool_b"}), false) + ctx, metadata := newLargePayloadCtx(nil, []string{"sentry"}) + + tools := stampedIncludeTools(t, p, ctx, newChatRequest()) + assert.Equal(t, []string{"sentry-tool_a", "sentry-tool_b"}, tools) + assert.Equal(t, "openai/gpt-4o", metadata.Model, "provider-prefixed model must survive routing unchanged") +} + +// Large-payload pruning with auto-injection disabled: the caller's include-tools list +// is narrowed against the grant — the toggle never disables grant enforcement. +func TestPreRequestHookMCP_LargePayload_AutoInjectOff_IncludeToolsPresent_Prunes(t *testing.T) { + p := newPluginForMCPStamping(t, buildVKForMCPStamping([]string{"tool_a", "tool_b"}), true) + ctx, metadata := newLargePayloadCtx([]string{"sentry-tool_a", "sentry-tool_c"}, nil) + + tools := stampedIncludeTools(t, p, ctx, newChatRequest()) + assert.Equal(t, []string{"sentry-tool_a"}, tools) + assert.Equal(t, "openai/gpt-4o", metadata.Model, "provider-prefixed model must survive routing unchanged") +} + +// Large-payload pruning with auto-injection enabled: granted entries survive, +// ungranted entries are dropped, and the grant is not re-stamped over the caller's +// narrower selection. +func TestPreRequestHookMCP_LargePayload_IncludeToolsPresent_Prunes(t *testing.T) { + p := newPluginForMCPStamping(t, buildVKForMCPStamping([]string{"tool_a", "tool_b"}), false) + ctx, metadata := newLargePayloadCtx([]string{"sentry-tool_a", "sentry-tool_c"}, nil) + + tools := stampedIncludeTools(t, p, ctx, newChatRequest()) + assert.Equal(t, []string{"sentry-tool_a"}, tools) + assert.Equal(t, "openai/gpt-4o", metadata.Model, "provider-prefixed model must survive routing unchanged") +} + +// Large-payload counterpart of the both-filters case: the pruned include-tools list is +// the final value and is not overwritten by the grant. Verified under both +// DisableAutoToolInject values. +func TestPreRequestHookMCP_LargePayload_BothFilters_PrunedListWins(t *testing.T) { + for _, disabled := range []bool{false, true} { + t.Run(fmt.Sprintf("disableAutoToolInject=%v", disabled), func(t *testing.T) { + p := newPluginForMCPStamping(t, buildVKForMCPStamping([]string{"tool_a", "tool_b"}), disabled) + ctx, metadata := newLargePayloadCtx([]string{"sentry-tool_a"}, []string{"sentry"}) + + tools := stampedIncludeTools(t, p, ctx, newChatRequest()) + assert.Equal(t, []string{"sentry-tool_a"}, tools, + "pruned caller list must not be overwritten by the grant") + assert.Equal(t, "openai/gpt-4o", metadata.Model, "provider-prefixed model must survive routing unchanged") + }) + } +} + +// Large-payload counterpart of the disabled-toggle baseline: no filters and +// auto-injection disabled leaves the include-tools key unset, so downstream injection +// is skipped entirely. +func TestPreRequestHookMCP_LargePayload_AutoInjectOff_NoFilters_StampsNothing(t *testing.T) { + p := newPluginForMCPStamping(t, buildVKForMCPStamping([]string{"tool_a", "tool_b"}), true) + ctx, metadata := newLargePayloadCtx(nil, nil) + + tools := stampedIncludeTools(t, p, ctx, newChatRequest()) + assert.Nil(t, tools) + assert.Equal(t, "openai/gpt-4o", metadata.Model, "provider-prefixed model must survive routing unchanged") +} diff --git a/plugins/governance/prerequestrouting_test.go b/plugins/governance/prerequestrouting_test.go new file mode 100644 index 0000000000..bea8e09e3e --- /dev/null +++ b/plugins/governance/prerequestrouting_test.go @@ -0,0 +1,74 @@ +package governance + +import ( + "context" + "testing" + + "github.com/maximhq/bifrost/core/schemas" + "github.com/maximhq/bifrost/framework/configstore" + configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newPreRequestRoutingPlugin(t *testing.T, vk *configstoreTables.TableVirtualKey) *GovernancePlugin { + t.Helper() + logger := NewMockLogger() + store, err := NewLocalGovernanceStore(context.Background(), logger, nil, &configstore.GovernanceConfig{ + VirtualKeys: []configstoreTables.TableVirtualKey{*vk}, + }, nil) + require.NoError(t, err) + return &GovernancePlugin{ + logger: logger, + store: store, + resolver: NewBudgetResolver(store, nil, logger, nil), + } +} + +// TestRunPreRequestRouting_ExplicitProviderPrefixSkipsLoadBalancing covers the +// large-payload path: metadata.Model arrives provider-prefixed and unparsed, and +// the explicit prefix must win over VK load balancing even when multiple weighted +// providers allow the model. +func TestRunPreRequestRouting_ExplicitProviderPrefixSkipsLoadBalancing(t *testing.T) { + vk := buildVirtualKeyWithProviders("vk1", "sk-bf-lb", "LB VK", []configstoreTables.TableVirtualKeyProviderConfig{ + buildProviderConfig("openai", []string{"*"}), + buildProviderConfig("anthropic", []string{"*"}), + }) + p := newPreRequestRoutingPlugin(t, vk) + + for range 20 { + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + got, err := p.runPreRequestRouting(ctx, vk, false, "openai/gpt-4o", schemas.ChatCompletionRequest) + require.NoError(t, err) + assert.Equal(t, "openai/gpt-4o", got) + } +} + +// TestRunPreRequestRouting_UnprefixedModelLoadBalances verifies that a bare model +// string still goes through VK load balancing and comes back provider-prefixed. +func TestRunPreRequestRouting_UnprefixedModelLoadBalances(t *testing.T) { + vk := buildVirtualKeyWithProviders("vk1", "sk-bf-lb", "LB VK", []configstoreTables.TableVirtualKeyProviderConfig{ + buildProviderConfig("openai", []string{"*"}), + }) + p := newPreRequestRoutingPlugin(t, vk) + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + + got, err := p.runPreRequestRouting(ctx, vk, false, "gpt-4o", schemas.ChatCompletionRequest) + require.NoError(t, err) + assert.Equal(t, "openai/gpt-4o", got) +} + +// TestRunPreRequestRouting_UnknownPrefixIsTreatedAsModelNamespace verifies that a +// "/" prefix that is not a known provider (e.g. a HuggingFace-style namespace) is +// kept as part of the model name and load balancing still applies. +func TestRunPreRequestRouting_UnknownPrefixIsTreatedAsModelNamespace(t *testing.T) { + vk := buildVirtualKeyWithProviders("vk1", "sk-bf-lb", "LB VK", []configstoreTables.TableVirtualKeyProviderConfig{ + buildProviderConfig("groq", []string{"*"}), + }) + p := newPreRequestRoutingPlugin(t, vk) + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + + got, err := p.runPreRequestRouting(ctx, vk, false, "meta-llama/llama-3.1-8b-instant", schemas.ChatCompletionRequest) + require.NoError(t, err) + assert.Equal(t, "groq/meta-llama/llama-3.1-8b-instant", got) +} diff --git a/plugins/governance/prunemcpincludetools_test.go b/plugins/governance/prunemcpincludetools_test.go new file mode 100644 index 0000000000..d849146160 --- /dev/null +++ b/plugins/governance/prunemcpincludetools_test.go @@ -0,0 +1,187 @@ +package governance + +import ( + "context" + "testing" + + "github.com/maximhq/bifrost/core/schemas" + configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newCtxWithIncludeTools returns a BifrostContext pre-stamped with a caller-provided +// include-tools list, mirroring what lib/ctx.go does for the x-bf-mcp-include-tools header. +func newCtxWithIncludeTools(tools []string) *schemas.BifrostContext { + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + ctx.SetValue(schemas.MCPContextKeyIncludeTools, tools) + return ctx +} + +// includeToolsFromCtx reads back the (possibly pruned) include-tools list from ctx. +func includeToolsFromCtx(t *testing.T, ctx *schemas.BifrostContext) []string { + t.Helper() + value := ctx.Value(schemas.MCPContextKeyIncludeTools) + require.NotNil(t, value, "include-tools ctx value should be set") + tools, ok := value.([]string) + require.True(t, ok, "include-tools ctx value should be a []string") + return tools +} + +// No caller-provided list on ctx → returns false and leaves ctx untouched. +func TestPruneMCPIncludeTools_NoCallerList(t *testing.T) { + p := newPluginWithInMemoryStore(&mockInMemoryStore{}) + vk := buildVKWithMCPConfigs("client-1", "sentry", []string{"find_projects"}) + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + + assert.False(t, p.pruneMCPIncludeToolsFromContext(ctx, vk)) + assert.Nil(t, ctx.Value(schemas.MCPContextKeyIncludeTools), + "ctx should remain unset when the caller provided no list") +} + +// A tool the VK does not grant is dropped; the result is an empty (deny-all) list. +func TestPruneMCPIncludeTools_DisallowedToolDropped(t *testing.T) { + p := newPluginWithInMemoryStore(&mockInMemoryStore{}) + vk := buildVKWithMCPConfigs("client-1", "sentry", []string{"find_projects"}) + ctx := newCtxWithIncludeTools([]string{"sentry-search_tools"}) + + assert.True(t, p.pruneMCPIncludeToolsFromContext(ctx, vk)) + assert.Empty(t, includeToolsFromCtx(t, ctx), + "a tool outside the VK grant must be pruned, leaving a deny-all list") +} + +// A tool the VK explicitly grants survives pruning. +func TestPruneMCPIncludeTools_GrantedToolKept(t *testing.T) { + p := newPluginWithInMemoryStore(&mockInMemoryStore{}) + vk := buildVKWithMCPConfigs("client-1", "sentry", []string{"find_projects", "search_issues"}) + ctx := newCtxWithIncludeTools([]string{"sentry-find_projects", "sentry-search_tools"}) + + assert.True(t, p.pruneMCPIncludeToolsFromContext(ctx, vk)) + assert.Equal(t, []string{"sentry-find_projects"}, includeToolsFromCtx(t, ctx), + "granted entries survive, ungranted entries are dropped") +} + +// A specific tool requested under an unrestricted ("*") VK grant survives — the +// header narrows within the wildcard grant. +func TestPruneMCPIncludeTools_SpecificToolUnderUnrestrictedGrant(t *testing.T) { + p := newPluginWithInMemoryStore(&mockInMemoryStore{}) + vk := buildVKWithMCPConfigs("client-1", "sentry", []string{"*"}) + ctx := newCtxWithIncludeTools([]string{"sentry-search_tools"}) + + assert.True(t, p.pruneMCPIncludeToolsFromContext(ctx, vk)) + assert.Equal(t, []string{"sentry-search_tools"}, includeToolsFromCtx(t, ctx), + "specific request should be allowed by the client's unrestricted grant") +} + +// A caller wildcard is kept verbatim only when the VK itself is unrestricted for that client. +func TestPruneMCPIncludeTools_WildcardKeptWhenVKUnrestricted(t *testing.T) { + p := newPluginWithInMemoryStore(&mockInMemoryStore{}) + vk := buildVKWithMCPConfigs("client-1", "sentry", []string{"*"}) + ctx := newCtxWithIncludeTools([]string{"sentry-*"}) + + assert.True(t, p.pruneMCPIncludeToolsFromContext(ctx, vk)) + assert.Equal(t, []string{"sentry-*"}, includeToolsFromCtx(t, ctx)) +} + +// A caller wildcard against a specific VK grant is narrowed to the grant's entries — +// passing the wildcard through would read downstream as "all tools of this client". +func TestPruneMCPIncludeTools_WildcardNarrowedToSpecificGrants(t *testing.T) { + p := newPluginWithInMemoryStore(&mockInMemoryStore{}) + vk := buildVKWithMCPConfigs("client-1", "sentry", []string{"find_projects", "search_issues"}) + ctx := newCtxWithIncludeTools([]string{"sentry-*"}) + + assert.True(t, p.pruneMCPIncludeToolsFromContext(ctx, vk)) + assert.Equal(t, []string{"sentry-find_projects", "sentry-search_issues"}, includeToolsFromCtx(t, ctx)) +} + +// A caller wildcard for a client the VK does not grant at all yields nothing. +func TestPruneMCPIncludeTools_WildcardForUngrantedClientDropped(t *testing.T) { + p := newPluginWithInMemoryStore(&mockInMemoryStore{}) + vk := buildVKWithMCPConfigs("client-1", "sentry", []string{"find_projects"}) + ctx := newCtxWithIncludeTools([]string{"github-*"}) + + assert.True(t, p.pruneMCPIncludeToolsFromContext(ctx, vk)) + assert.Empty(t, includeToolsFromCtx(t, ctx)) +} + +// An empty header value (parsed as [""] by lib/ctx.go) prunes to a deny-all list, +// letting callers suppress tool injection for a request. +func TestPruneMCPIncludeTools_EmptyHeaderOptOut(t *testing.T) { + p := newPluginWithInMemoryStore(&mockInMemoryStore{}) + vk := buildVKWithMCPConfigs("client-1", "sentry", []string{"*"}) + ctx := newCtxWithIncludeTools([]string{""}) + + assert.True(t, p.pruneMCPIncludeToolsFromContext(ctx, vk)) + assert.Empty(t, includeToolsFromCtx(t, ctx)) +} + +// Wildcard expansion plus an overlapping specific request must not produce duplicates. +func TestPruneMCPIncludeTools_DedupAcrossWildcardAndSpecific(t *testing.T) { + p := newPluginWithInMemoryStore(&mockInMemoryStore{}) + vk := buildVKWithMCPConfigs("client-1", "sentry", []string{"find_projects", "search_issues"}) + ctx := newCtxWithIncludeTools([]string{"sentry-*", "sentry-find_projects"}) + + assert.True(t, p.pruneMCPIncludeToolsFromContext(ctx, vk)) + assert.Equal(t, []string{"sentry-find_projects", "sentry-search_issues"}, includeToolsFromCtx(t, ctx)) +} + +// AllowOnAllVirtualKeys client with no explicit VK config: both specific and wildcard +// requests survive (the implicit grant is client-wide). +func TestPruneMCPIncludeTools_AllowOnAllVirtualKeysClient(t *testing.T) { + p := newPluginWithInMemoryStore(&mockInMemoryStore{ + allowAllClients: map[string]string{"client-1": "youtube"}, + }) + vk := buildVKNoMCPConfigs() + ctx := newCtxWithIncludeTools([]string{"youtube-search", "youtube-*", "github-list_repos"}) + + assert.True(t, p.pruneMCPIncludeToolsFromContext(ctx, vk)) + assert.Equal(t, []string{"youtube-search", "youtube-*"}, includeToolsFromCtx(t, ctx), + "AllowOnAllVirtualKeys grants the whole client; other clients are still pruned") +} + +// An explicit empty VK config (deny-all) overrides the client's AllowOnAllVirtualKeys flag. +func TestPruneMCPIncludeTools_ExplicitEmptyConfigOverridesAllowAll(t *testing.T) { + p := newPluginWithInMemoryStore(&mockInMemoryStore{ + allowAllClients: map[string]string{"client-1": "youtube"}, + }) + vk := buildVKWithMCPConfigs("client-1", "youtube", []string{}) + ctx := newCtxWithIncludeTools([]string{"youtube-search", "youtube-*"}) + + assert.True(t, p.pruneMCPIncludeToolsFromContext(ctx, vk)) + assert.Empty(t, includeToolsFromCtx(t, ctx)) +} + +// Pruning spans multiple VK clients independently. +func TestPruneMCPIncludeTools_MultipleClients(t *testing.T) { + p := newPluginWithInMemoryStore(&mockInMemoryStore{}) + vk := &configstoreTables.TableVirtualKey{ + ID: "vk-multi", + Name: "test-vk-multi", + MCPConfigs: []configstoreTables.TableVirtualKeyMCPConfig{ + { + MCPClient: configstoreTables.TableMCPClient{ClientID: "client-1", Name: "sentry"}, + ToolsToExecute: []string{"find_projects"}, + }, + { + MCPClient: configstoreTables.TableMCPClient{ClientID: "client-2", Name: "github"}, + ToolsToExecute: []string{"*"}, + }, + }, + } + ctx := newCtxWithIncludeTools([]string{"sentry-find_projects", "sentry-search_issues", "github-list_repos", "github-*"}) + + assert.True(t, p.pruneMCPIncludeToolsFromContext(ctx, vk)) + assert.Equal(t, []string{"sentry-find_projects", "github-list_repos", "github-*"}, includeToolsFromCtx(t, ctx)) +} + +// A ctx value of the wrong type fails closed: treated as a present-but-empty caller list. +func TestPruneMCPIncludeTools_WrongTypeFailsClosed(t *testing.T) { + p := newPluginWithInMemoryStore(&mockInMemoryStore{}) + vk := buildVKWithMCPConfigs("client-1", "sentry", []string{"*"}) + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + ctx.SetValue(schemas.MCPContextKeyIncludeTools, "sentry-search_tools") + + assert.True(t, p.pruneMCPIncludeToolsFromContext(ctx, vk)) + assert.Empty(t, includeToolsFromCtx(t, ctx), + "a malformed ctx value must prune to deny-all, not pass through") +} diff --git a/plugins/governance/resolver.go b/plugins/governance/resolver.go index 84dd9e207b..e420034431 100644 --- a/plugins/governance/resolver.go +++ b/plugins/governance/resolver.go @@ -363,7 +363,7 @@ func (r *BudgetResolver) isModelAllowed(vk *configstoreTables.TableVirtualKey, p // Pass 1: if any matching provider config blacklists the model, block immediately. for _, pc := range vk.ProviderConfigs { - if pc.Provider == string(provider) && isModelBlockedByList(pc.BlacklistedModels, model) { + if pc.Provider == string(provider) && pc.BlacklistedModels.IsBlocked(model) { return false } } diff --git a/plugins/governance/resolver_test.go b/plugins/governance/resolver_test.go index f3135a1bdc..86fb7a4041 100644 --- a/plugins/governance/resolver_test.go +++ b/plugins/governance/resolver_test.go @@ -9,7 +9,6 @@ import ( "github.com/maximhq/bifrost/core/schemas" "github.com/maximhq/bifrost/framework/configstore" configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" - "github.com/maximhq/bifrost/framework/modelcatalog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -36,87 +35,6 @@ func TestBudgetResolver_EvaluateRequest_AllowedRequest(t *testing.T) { assertVirtualKeyFound(t, result) } -// TestBudgetResolver_EvaluateRequest_WildcardAllowsCatalogOpaqueProvider verifies that a -// wildcard ("*") allow-list permits any model on a catalog-opaque provider (vLLM, whose -// self-hosted models are never in the bundled catalog), while leaving catalog-known providers -// (openai) fully intact — i.e. wildcard is still catalog-cross-checked for them. -func TestBudgetResolver_EvaluateRequest_WildcardAllowsCatalogOpaqueProvider(t *testing.T) { - logger := NewMockLogger() - - // Catalog knows openai/gpt-4o but has NO model list for vLLM. - mc := modelcatalog.NewTestCatalog(map[string]string{"openai/gpt-4o": "gpt-4o"}) - mc.UpsertModelDataForProvider(schemas.OpenAI, - &schemas.BifrostListModelsResponse{Data: []schemas.Model{{ID: "openai/gpt-4o"}}}, nil) - - // Non-nil inMemoryStore so isModelAllowed takes the catalog branch. - inMem := &mockInMemoryStore{ - configuredProviders: map[schemas.ModelProvider]configstore.ProviderConfig{ - schemas.VLLM: {}, - schemas.OpenAI: {}, - }, - } - - vk := buildVirtualKey("vk1", "sk-bf-test", "Test VK", true) - vk.ProviderConfigs = []configstoreTables.TableVirtualKeyProviderConfig{ - buildProviderConfig("vllm", []string{"*"}), - buildProviderConfig("openai", []string{"*"}), - } - store, err := NewLocalGovernanceStore(context.Background(), logger, nil, &configstore.GovernanceConfig{ - VirtualKeys: []configstoreTables.TableVirtualKey{*vk}, - }, mc) - require.NoError(t, err) - - resolver := NewBudgetResolver(store, mc, logger, inMem) - ctx := &schemas.BifrostContext{} - - // vLLM (catalog-opaque) + ["*"] + uncatalogued model -> allowed (the fix). - result := resolver.EvaluateVirtualKeyRequest(ctx, "sk-bf-test", schemas.VLLM, "my-self-hosted-llama", schemas.ChatCompletionRequest, false) - assertDecision(t, DecisionAllow, result) - - // openai intact: a real catalog model under ["*"] is still allowed. - result = resolver.EvaluateVirtualKeyRequest(ctx, "sk-bf-test", schemas.OpenAI, "gpt-4o", schemas.ChatCompletionRequest, false) - assertDecision(t, DecisionAllow, result) - - // openai intact: an unknown model under ["*"] is still catalog-cross-checked and blocked. - result = resolver.EvaluateVirtualKeyRequest(ctx, "sk-bf-test", schemas.OpenAI, "not-a-real-model", schemas.ChatCompletionRequest, false) - assertDecision(t, DecisionModelBlocked, result) -} - -// TestBudgetResolver_EvaluateRequest_WildcardOpaqueProviderRespectsBlacklist guards the ordering -// in isModelAllowed: the blacklist pass must run before the wildcard + catalog-opaque shortcut, -// so a blacklisted model is blocked on an opaque provider even under a ["*"] allow-list. -func TestBudgetResolver_EvaluateRequest_WildcardOpaqueProviderRespectsBlacklist(t *testing.T) { - logger := NewMockLogger() - - mc := modelcatalog.NewTestCatalog(nil) // catalog has no vLLM models -> opaque - inMem := &mockInMemoryStore{ - configuredProviders: map[schemas.ModelProvider]configstore.ProviderConfig{ - schemas.VLLM: {}, - }, - } - - vllmConfig := buildProviderConfig("vllm", []string{"*"}) - vllmConfig.BlacklistedModels = schemas.BlackList{"my-self-hosted-llama"} - - vk := buildVirtualKey("vk1", "sk-bf-test", "Test VK", true) - vk.ProviderConfigs = []configstoreTables.TableVirtualKeyProviderConfig{vllmConfig} - store, err := NewLocalGovernanceStore(context.Background(), logger, nil, &configstore.GovernanceConfig{ - VirtualKeys: []configstoreTables.TableVirtualKey{*vk}, - }, mc) - require.NoError(t, err) - - resolver := NewBudgetResolver(store, mc, logger, inMem) - ctx := &schemas.BifrostContext{} - - // Blacklisted model on the opaque provider is blocked despite the ["*"] allow-list. - result := resolver.EvaluateVirtualKeyRequest(ctx, "sk-bf-test", schemas.VLLM, "my-self-hosted-llama", schemas.ChatCompletionRequest, false) - assertDecision(t, DecisionModelBlocked, result) - - // A different (non-blacklisted) model on the same opaque provider is still allowed. - result = resolver.EvaluateVirtualKeyRequest(ctx, "sk-bf-test", schemas.VLLM, "another-local-model", schemas.ChatCompletionRequest, false) - assertDecision(t, DecisionAllow, result) -} - // TestBudgetResolver_EvaluateRequest_VirtualKeyNotFound tests missing VK func TestBudgetResolver_EvaluateRequest_VirtualKeyNotFound(t *testing.T) { logger := NewMockLogger() diff --git a/plugins/governance/routing.go b/plugins/governance/routing.go index bfccd8ec4f..0853a5b333 100644 --- a/plugins/governance/routing.go +++ b/plugins/governance/routing.go @@ -8,8 +8,10 @@ import ( "sync" "github.com/google/cel-go/cel" + "github.com/google/cel-go/common/types" "github.com/maximhq/bifrost/core/schemas" configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" + "github.com/maximhq/bifrost/plugins/governance/complexity" ) // DefaultRoutingChainMaxDepth is the default maximum depth for routing rule chain evaluation. @@ -35,14 +37,15 @@ type RoutingDecision struct { // RoutingContext holds all data needed for routing rule evaluation // Reuses existing configstore table types for VirtualKey, Team, Customer type RoutingContext struct { - VirtualKey *configstoreTables.TableVirtualKey // nil if no VK - Provider schemas.ModelProvider // Current provider - Model string // Current model - RequestType string // Normalized request type (e.g., "chat_completion", "embedding") from HTTP context - Fallbacks []string // Fallback chain: ["provider/model", ...] - Headers map[string]string // Request headers for dynamic routing - QueryParams map[string]string // Query parameters for dynamic routing - BudgetAndRateLimitStatus *BudgetAndRateLimitStatus // Budget and rate limit status by provider/model + VirtualKey *configstoreTables.TableVirtualKey // nil if no VK + Provider schemas.ModelProvider // Current provider + Model string // Current model + RequestType string // Normalized request type (e.g., "chat_completion", "embedding") from HTTP context + Fallbacks []string // Fallback chain: ["provider/model", ...] + Headers map[string]string // Request headers for dynamic routing + QueryParams map[string]string // Query parameters for dynamic routing + BudgetAndRateLimitStatus *BudgetAndRateLimitStatus // Budget and rate limit status by provider/model + computeComplexity func() *complexity.ComplexityResult // Lazy complexity computation; called at most once when a rule references "complexity_tier" } type RoutingEngine struct { @@ -122,6 +125,8 @@ func (re *RoutingEngine) EvaluateRoutingRules(ctx *schemas.BifrostContext, routi ctx.AppendRoutingEngineLog(schemas.RoutingEngineRoutingRule, schemas.LogLevelInfo, fmt.Sprintf("Scope chain: %v", scopeChainToStrings(scopeChain))) var finalDecision *RoutingDecision + var complexityResult *complexity.ComplexityResult + computeComplexity := routingCtx.computeComplexity for chainStep := 0; ; chainStep++ { // TERMINATION 4: Chain exceeded configured max depth. @@ -150,6 +155,9 @@ func (re *RoutingEngine) EvaluateRoutingRules(ctx *schemas.BifrostContext, routi ctx.AppendRoutingEngineLog(schemas.RoutingEngineRoutingRule, schemas.LogLevelError, fmt.Sprintf("Failed to extract routing variables: %v", err)) return nil, fmt.Errorf("failed to extract routing variables: %w", err) } + if complexityResult != nil { + variables["complexity_tier"] = complexityResult.Tier + } re.logger.Debug("[RoutingEngine] Chain Step: %d", chainStep) @@ -180,6 +188,17 @@ func (re *RoutingEngine) EvaluateRoutingRules(ctx *schemas.BifrostContext, routi } re.logger.Debug("[RoutingEngine] Evaluating rule: name=%s, expression=%s", rule.Name, rule.CelExpression) + referencesComplexity := celExpressionReferencesIdentifier(rule.CelExpression, "complexity_tier") + + // Lazy complexity: compute only when a rule references complexity and it hasn't been computed yet + if complexityResult == nil && computeComplexity != nil && referencesComplexity { + complexityResult = computeComplexity() + computeComplexity = nil // compute at most once + if complexityResult != nil { + variables["complexity_tier"] = complexityResult.Tier + } + } + program, err := re.store.GetRoutingProgram(ctx, rule) if err != nil { re.logger.Warn("[RoutingEngine] Failed to compile rule %s: %v", rule.Name, err) @@ -187,7 +206,12 @@ func (re *RoutingEngine) EvaluateRoutingRules(ctx *schemas.BifrostContext, routi continue } - matched, err := evaluateCELExpression(program, variables) + var unknowns []*cel.AttributePatternType + if referencesComplexity && complexityResult == nil { + unknowns = append(unknowns, cel.AttributePattern("complexity_tier")) + } + + matched, err := evaluateCELExpression(program, variables, unknowns...) if err != nil { re.logger.Warn("[RoutingEngine] Failed to evaluate rule %s: %v", rule.Name, err) ctx.AppendRoutingEngineLog(schemas.RoutingEngineRoutingRule, schemas.LogLevelError, fmt.Sprintf("Rule '%s' skipped: eval error: %v", rule.Name, err)) @@ -369,13 +393,22 @@ func buildScopeChain(virtualKey *configstoreTables.TableVirtualKey) []ScopeLevel } // evaluateCELExpression evaluates a compiled CEL program with given variables -func evaluateCELExpression(program cel.Program, variables map[string]any) (bool, error) { +func evaluateCELExpression(program cel.Program, variables map[string]any, unknowns ...*cel.AttributePatternType) (bool, error) { if program == nil { return false, fmt.Errorf("CEL program is nil") } + activation := any(variables) + if len(unknowns) > 0 { + partial, err := cel.PartialVars(variables, unknowns...) + if err != nil { + return false, fmt.Errorf("CEL partial activation error: %w", err) + } + activation = partial + } + // Evaluate the program - out, _, err := program.Eval(variables) + out, _, err := program.Eval(activation) if err != nil { // Gracefully handle "no such key" errors - when a header/param is missing, treat as non-match if strings.Contains(err.Error(), "no such key") { @@ -384,6 +417,13 @@ func evaluateCELExpression(program cel.Program, variables map[string]any) (bool, return false, fmt.Errorf("CEL evaluation error: %w", err) } + // Unknown means the expression depends on a value that is unavailable for + // this request. For routing safety, treat it as a no-match rather than + // allowing sentinels like complexity_tier == "" to leak into product logic. + if types.IsUnknown(out) { + return false, nil + } + // Convert result to boolean matched, ok := out.Value().(bool) if !ok { @@ -474,6 +514,11 @@ func extractRoutingVariables(ctx *RoutingContext) (map[string]interface{}, error variables["request"] = 0.0 } + // Placeholder only: EvaluateRoutingRules fills this lazily when a rule + // actually references complexity_tier. If complexity is unavailable, it is + // evaluated as a CEL unknown so negative predicates do not accidentally match. + variables["complexity_tier"] = "" + return variables, nil } @@ -576,5 +621,9 @@ func createCELEnvironment() (*cel.Env, error) { cel.Variable("tokens_used", cel.DoubleType), cel.Variable("request", cel.DoubleType), cel.Variable("budget_used", cel.DoubleType), + + // Complexity tier. When analysis is unavailable, evaluation marks this + // variable as CEL unknown so complexity-dependent predicates do not match. + cel.Variable("complexity_tier", cel.StringType), ) } diff --git a/plugins/governance/routingcelrefs.go b/plugins/governance/routingcelrefs.go new file mode 100644 index 0000000000..06a011533d --- /dev/null +++ b/plugins/governance/routingcelrefs.go @@ -0,0 +1,147 @@ +package governance + +import ( + "sync" + + "github.com/google/cel-go/cel" + "github.com/google/cel-go/common" + celast "github.com/google/cel-go/common/ast" + "github.com/google/cel-go/parser" +) + +// Most routing variables are cheap: createCELEnvironment declares them, +// extractRoutingVariables populates them, and evaluateCELExpression passes them +// to CEL for evaluation. complexity_tier is different because populating it +// means extracting text from the request body and running the complexity +// analyzer (we dont have the value yet without these steps). Keep that work lazy by +// first checking whether a CEL rule actually references the identifier. + +// Walk the parsed CEL AST instead of using strings.Contains so string literals +// like "complexity_tier" and scoped macro variables do not accidentally trigger +// analysis. The same check is used during program compilation so only +// complexity-aware rules enable partial evaluation for the unavailable/unknown +// complexity_tier path. + +var celExpressionIdentifierRefCache sync.Map + +func celExpressionReferencesIdentifier(expr string, identifier string) bool { + if expr == "" || identifier == "" { + return false + } + + cacheKey := identifier + "\x00" + expr + if cached, ok := celExpressionIdentifierRefCache.Load(cacheKey); ok { + if result, ok := cached.(bool); ok { + return result + } + } + + result := false + p, err := parser.NewParser(parser.Macros(parser.AllMacros...)) + if err != nil { + celExpressionIdentifierRefCache.Store(cacheKey, result) + return result + } + + parsed, errs := p.Parse(common.NewTextSource(expr)) + if errs != nil && len(errs.GetErrors()) > 0 { + celExpressionIdentifierRefCache.Store(cacheKey, result) + return result + } + if parsed != nil { + result = celExprReferencesIdentifier(parsed.Expr(), identifier, nil) + } + + celExpressionIdentifierRefCache.Store(cacheKey, result) + return result +} + +func celASTReferencesIdentifier(ast *cel.Ast, identifier string) bool { + if ast == nil || ast.NativeRep() == nil || identifier == "" { + return false + } + return celExprReferencesIdentifier(ast.NativeRep().Expr(), identifier, nil) +} + +func celExprReferencesIdentifier(expr celast.Expr, identifier string, scopedIdents map[string]int) bool { + if expr == nil { + return false + } + + switch expr.Kind() { + case celast.IdentKind: + return expr.AsIdent() == identifier && scopedIdents[identifier] == 0 + case celast.CallKind: + call := expr.AsCall() + if celExprReferencesIdentifier(call.Target(), identifier, scopedIdents) { + return true + } + for _, arg := range call.Args() { + if celExprReferencesIdentifier(arg, identifier, scopedIdents) { + return true + } + } + case celast.ComprehensionKind: + comp := expr.AsComprehension() + if celExprReferencesIdentifier(comp.IterRange(), identifier, scopedIdents) { + return true + } + + scoped := addScopedCELIdentifiers(scopedIdents, comp.IterVar(), comp.IterVar2(), comp.AccuVar()) + if celExprReferencesIdentifier(comp.AccuInit(), identifier, scoped) { + return true + } + if celExprReferencesIdentifier(comp.LoopCondition(), identifier, scoped) { + return true + } + if celExprReferencesIdentifier(comp.LoopStep(), identifier, scoped) { + return true + } + if celExprReferencesIdentifier(comp.Result(), identifier, scoped) { + return true + } + case celast.ListKind: + for _, elem := range expr.AsList().Elements() { + if celExprReferencesIdentifier(elem, identifier, scopedIdents) { + return true + } + } + case celast.MapKind: + for _, entry := range expr.AsMap().Entries() { + if entry.Kind() != celast.MapEntryKind { + continue + } + mapEntry := entry.AsMapEntry() + if celExprReferencesIdentifier(mapEntry.Key(), identifier, scopedIdents) || + celExprReferencesIdentifier(mapEntry.Value(), identifier, scopedIdents) { + return true + } + } + case celast.SelectKind: + return celExprReferencesIdentifier(expr.AsSelect().Operand(), identifier, scopedIdents) + case celast.StructKind: + for _, field := range expr.AsStruct().Fields() { + if field.Kind() != celast.StructFieldKind { + continue + } + if celExprReferencesIdentifier(field.AsStructField().Value(), identifier, scopedIdents) { + return true + } + } + } + + return false +} + +func addScopedCELIdentifiers(parent map[string]int, identifiers ...string) map[string]int { + scoped := make(map[string]int, len(parent)+len(identifiers)) + for identifier, count := range parent { + scoped[identifier] = count + } + for _, identifier := range identifiers { + if identifier != "" { + scoped[identifier]++ + } + } + return scoped +} diff --git a/plugins/governance/routingcomplexity_test.go b/plugins/governance/routingcomplexity_test.go new file mode 100644 index 0000000000..b793ae3277 --- /dev/null +++ b/plugins/governance/routingcomplexity_test.go @@ -0,0 +1,385 @@ +package governance + +import ( + "context" + "testing" + "time" + + "github.com/google/cel-go/cel" + "github.com/maximhq/bifrost/core/schemas" + "github.com/maximhq/bifrost/framework/configstore" + configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" + "github.com/maximhq/bifrost/plugins/governance/complexity" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCELExpressionReferencesComplexityTierIdentifierOnly(t *testing.T) { + tests := []struct { + name string + expression string + expected bool + }{ + { + name: "direct identifier", + expression: `complexity_tier == "SIMPLE"`, + expected: true, + }, + { + name: "identifier in in-list", + expression: `complexity_tier in ["COMPLEX", "REASONING"]`, + expected: true, + }, + { + name: "string literal only", + expression: `model == "complexity_tier"`, + expected: false, + }, + { + name: "unrelated identifier containing name", + expression: `my_complexity_tier == true`, + expected: false, + }, + { + name: "map key string", + expression: `headers["complexity_tier"] == "SIMPLE"`, + expected: false, + }, + { + name: "field selection", + expression: `metadata.complexity_tier == "SIMPLE"`, + expected: false, + }, + { + name: "comprehension local shadows identifier", + expression: `["SIMPLE"].exists(complexity_tier, complexity_tier == "SIMPLE")`, + expected: false, + }, + { + name: "comprehension references outer identifier", + expression: `["SIMPLE"].exists(tier, complexity_tier == tier)`, + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, celExpressionReferencesIdentifier(tt.expression, "complexity_tier")) + }) + } +} + +// TestCELComplexityTierVariable proves that CEL supports the flat complexity_tier string variable. +// This is the foundation for expressions like complexity_tier == "COMPLEX". +func TestCELComplexityTierVariable(t *testing.T) { + env, err := cel.NewEnv( + cel.Variable("complexity_tier", cel.StringType), + ) + require.NoError(t, err, "failed to create CEL environment") + + tests := []struct { + name string + expression string + variables map[string]interface{} + expected bool + }{ + { + name: "tier equals COMPLEX", + expression: `complexity_tier == "COMPLEX"`, + variables: map[string]interface{}{ + "complexity_tier": "COMPLEX", + }, + expected: true, + }, + { + name: "tier equals SIMPLE", + expression: `complexity_tier == "SIMPLE"`, + variables: map[string]interface{}{ + "complexity_tier": "SIMPLE", + }, + expected: true, + }, + { + name: "tier equals REASONING", + expression: `complexity_tier == "REASONING"`, + variables: map[string]interface{}{ + "complexity_tier": "REASONING", + }, + expected: true, + }, + { + name: "tier mismatch", + expression: `complexity_tier == "COMPLEX"`, + variables: map[string]interface{}{ + "complexity_tier": "MEDIUM", + }, + expected: false, + }, + { + name: "tier not equals", + expression: `complexity_tier != "SIMPLE"`, + variables: map[string]interface{}{ + "complexity_tier": "COMPLEX", + }, + expected: true, + }, + { + name: "tier in list", + expression: `complexity_tier in ["COMPLEX", "REASONING"]`, + variables: map[string]interface{}{ + "complexity_tier": "COMPLEX", + }, + expected: true, + }, + { + name: "tier not in list", + expression: `!(complexity_tier in ["SIMPLE", "MEDIUM"])`, + variables: map[string]interface{}{ + "complexity_tier": "COMPLEX", + }, + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ast, issues := env.Compile(tt.expression) + require.NoError(t, issues.Err(), "compilation failed for: %s", tt.expression) + + program, err := env.Program(ast) + require.NoError(t, err, "program creation failed for: %s", tt.expression) + + out, _, err := program.Eval(tt.variables) + require.NoError(t, err, "evaluation failed for: %s", tt.expression) + + result, ok := out.Value().(bool) + assert.True(t, ok, "expected boolean result") + assert.Equal(t, tt.expected, result, "unexpected result for: %s", tt.expression) + }) + } +} + +// TestCELComplexityWithFullEnvironment tests complexity_tier alongside all existing CEL variables. +func TestCELComplexityWithFullEnvironment(t *testing.T) { + env, err := createCELEnvironment() + require.NoError(t, err, "failed to create full CEL environment") + + expression := `complexity_tier == "SIMPLE" && budget_used > 60.0` + ast, issues := env.Compile(expression) + require.NoError(t, issues.Err(), "compilation failed") + + program, err := env.Program(ast) + require.NoError(t, err, "program creation failed") + + variables := map[string]interface{}{ + "model": "gpt-4o", + "provider": "openai", + "request_type": "chat_completion", + "headers": map[string]string{}, + "params": map[string]string{}, + "virtual_key_id": "", + "virtual_key_name": "", + "team_id": "", + "team_name": "", + "customer_id": "", + "customer_name": "", + "tokens_used": 0.0, + "request": 0.0, + "budget_used": 75.0, + "complexity_tier": "SIMPLE", + } + + out, _, err := program.Eval(variables) + require.NoError(t, err, "evaluation failed") + + result, ok := out.Value().(bool) + assert.True(t, ok, "expected boolean result") + assert.True(t, result, "expected complexity_tier == SIMPLE && budget_used > 60 to match") +} + +func TestEvaluateCELExpression_ComplexityTierUnknown(t *testing.T) { + tests := []struct { + name string + expression string + budgetUsed float64 + expected bool + }{ + { + name: "not equals depends on unavailable complexity", + expression: `complexity_tier != "SIMPLE"`, + expected: false, + }, + { + name: "not in depends on unavailable complexity", + expression: `!(complexity_tier in ["SIMPLE"])`, + expected: false, + }, + { + name: "or short-circuits when non-complexity side is true", + expression: `budget_used > 90.0 || complexity_tier != "SIMPLE"`, + budgetUsed: 95.0, + expected: true, + }, + { + name: "or is no match when only unavailable complexity can decide", + expression: `budget_used > 90.0 || complexity_tier != "SIMPLE"`, + budgetUsed: 40.0, + expected: false, + }, + } + + env, err := createCELEnvironment() + require.NoError(t, err) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ast, issues := env.Compile(tt.expression) + require.NoError(t, issues.Err()) + + program, err := env.Program(ast, cel.EvalOptions(cel.OptPartialEval)) + require.NoError(t, err) + + variables := complexityRoutingVariables() + variables["budget_used"] = tt.budgetUsed + + matched, err := evaluateCELExpression(program, variables, cel.AttributePattern("complexity_tier")) + require.NoError(t, err) + assert.Equal(t, tt.expected, matched) + }) + } +} + +func TestEvaluateRoutingRules_ComplexityUnavailableNegativePredicatesDoNotMatch(t *testing.T) { + tests := []struct { + name string + expression string + }{ + { + name: "not equals", + expression: `complexity_tier != "SIMPLE"`, + }, + { + name: "not in", + expression: `!(complexity_tier in ["SIMPLE"])`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + store, err := NewLocalGovernanceStore(ctx, NewMockLogger(), nil, &configstore.GovernanceConfig{}, nil) + require.NoError(t, err) + + rule := complexityRoutingRule("complexity-unavailable-"+tt.name, tt.expression) + require.NoError(t, store.UpdateRoutingRuleInMemory(ctx, rule)) + + engine, err := NewRoutingEngine(store, NewMockLogger(), schemas.Ptr(10)) + require.NoError(t, err) + + computeCalls := 0 + decision, err := engine.EvaluateRoutingRules(schemas.NewBifrostContext(ctx, time.Now()), &RoutingContext{ + Provider: schemas.OpenAI, + Model: "gpt-4o", + RequestType: "chat_completion", + computeComplexity: func() *complexity.ComplexityResult { + computeCalls++ + return nil + }, + }) + require.NoError(t, err) + + assert.Nil(t, decision) + assert.Equal(t, 1, computeCalls) + }) + } +} + +func TestEvaluateRoutingRules_ComplexityTierLiteralDoesNotComputeComplexity(t *testing.T) { + ctx := context.Background() + store, err := NewLocalGovernanceStore(ctx, NewMockLogger(), nil, &configstore.GovernanceConfig{}, nil) + require.NoError(t, err) + + rule := complexityRoutingRule("complexity-tier-literal", `model == "complexity_tier"`) + require.NoError(t, store.UpdateRoutingRuleInMemory(ctx, rule)) + + engine, err := NewRoutingEngine(store, NewMockLogger(), schemas.Ptr(10)) + require.NoError(t, err) + + computeCalls := 0 + decision, err := engine.EvaluateRoutingRules(schemas.NewBifrostContext(ctx, time.Now()), &RoutingContext{ + Provider: schemas.OpenAI, + Model: "complexity_tier", + RequestType: "chat_completion", + computeComplexity: func() *complexity.ComplexityResult { + computeCalls++ + return &complexity.ComplexityResult{Tier: "SIMPLE"} + }, + }) + require.NoError(t, err) + require.NotNil(t, decision) + + assert.Equal(t, 0, computeCalls) + assert.Equal(t, "anthropic", decision.Provider) + assert.Equal(t, "claude-3-5-sonnet", decision.Model) +} + +func TestEvaluateRoutingRules_ComplexityNegativePredicateMatchesAvailableTier(t *testing.T) { + ctx := context.Background() + store, err := NewLocalGovernanceStore(ctx, NewMockLogger(), nil, &configstore.GovernanceConfig{}, nil) + require.NoError(t, err) + + rule := complexityRoutingRule("complexity-available-not-simple", `complexity_tier != "SIMPLE"`) + require.NoError(t, store.UpdateRoutingRuleInMemory(ctx, rule)) + + engine, err := NewRoutingEngine(store, NewMockLogger(), schemas.Ptr(10)) + require.NoError(t, err) + + decision, err := engine.EvaluateRoutingRules(schemas.NewBifrostContext(ctx, time.Now()), &RoutingContext{ + Provider: schemas.OpenAI, + Model: "gpt-4o", + RequestType: "chat_completion", + computeComplexity: func() *complexity.ComplexityResult { + return &complexity.ComplexityResult{Tier: "COMPLEX"} + }, + }) + require.NoError(t, err) + require.NotNil(t, decision) + assert.Equal(t, "anthropic", decision.Provider) + assert.Equal(t, "claude-3-5-sonnet", decision.Model) +} + +func complexityRoutingVariables() map[string]interface{} { + return map[string]interface{}{ + "model": "gpt-4o", + "provider": "openai", + "request_type": "chat_completion", + "headers": map[string]string{}, + "params": map[string]string{}, + "virtual_key_id": "", + "virtual_key_name": "", + "team_id": "", + "team_name": "", + "customer_id": "", + "customer_name": "", + "tokens_used": 0.0, + "request": 0.0, + "budget_used": 0.0, + "complexity_tier": "", + } +} + +func complexityRoutingRule(id string, expression string) *configstoreTables.TableRoutingRule { + provider := "anthropic" + model := "claude-3-5-sonnet" + return &configstoreTables.TableRoutingRule{ + ID: id, + Name: id, + Enabled: boolPtr(true), + CelExpression: expression, + Scope: "global", + Priority: 1, + Targets: []configstoreTables.TableRoutingTarget{ + {Provider: &provider, Model: &model, Weight: 1.0}, + }, + } +} diff --git a/plugins/governance/store.go b/plugins/governance/store.go index 612b7698b9..a522e56db9 100644 --- a/plugins/governance/store.go +++ b/plugins/governance/store.go @@ -1382,6 +1382,42 @@ func (gs *LocalGovernanceStore) GetTeamCustomerID(ctx context.Context, teamID st return *team.CustomerID } +// GetTeamName returns a team's display name from the in-memory store, or "" if +// the team is unknown. The enterprise layer uses it as the fallback for log +// stamping when its edge-driven name caches miss (e.g. a team with no user +// members and no business unit). +func (gs *LocalGovernanceStore) GetTeamName(ctx context.Context, teamID string) string { + if teamID == "" { + return "" + } + teamValue, exists := gs.teams.Load(teamID) + if !exists || teamValue == nil { + return "" + } + team, ok := teamValue.(*configstoreTables.TableTeam) + if !ok || team == nil { + return "" + } + return team.Name +} + +// GetCustomerName returns a customer's display name from the in-memory store, +// or "" if the customer is unknown. Same fallback role as GetTeamName. +func (gs *LocalGovernanceStore) GetCustomerName(ctx context.Context, customerID string) string { + if customerID == "" { + return "" + } + customerValue, exists := gs.customers.Load(customerID) + if !exists || customerValue == nil { + return "" + } + customer, ok := customerValue.(*configstoreTables.TableCustomer) + if !ok || customer == nil { + return "" + } + return customer.Name +} + // CheckCustomerBudget checks customer-level budget and returns evaluation result if violated func (gs *LocalGovernanceStore) CheckCustomerBudget(ctx context.Context, customerID string, request *EvaluationRequest, baselines map[string]float64) (Decision, error) { if customerID == "" { @@ -3730,8 +3766,16 @@ func (gs *LocalGovernanceStore) GetRoutingProgram(ctx context.Context, rule *con return nil, fmt.Errorf("CEL compile error: %s", issues.Err().Error()) } - // Create program - program, err := gs.routingCELEnv.Program(ast) + // Create program. Partial evaluation is only needed for complexity rules, + // where routing treats unavailable complexity_tier as unknown instead of + // leaking an empty-string sentinel. + var program cel.Program + var err error + if celASTReferencesIdentifier(ast, "complexity_tier") { + program, err = gs.routingCELEnv.Program(ast, cel.EvalOptions(cel.OptPartialEval)) + } else { + program, err = gs.routingCELEnv.Program(ast) + } if err != nil { return nil, fmt.Errorf("CEL program creation error: %w", err) } diff --git a/plugins/governance/store_test.go b/plugins/governance/store_test.go index 7d22ab1be6..e284e6e613 100644 --- a/plugins/governance/store_test.go +++ b/plugins/governance/store_test.go @@ -1243,6 +1243,26 @@ func TestCompileAndCacheProgram_EmptyExpression(t *testing.T) { assert.Equal(t, program, program2) } +// TestGetTeamNameAndGetCustomerName verifies the display-name accessors the +// enterprise layer uses as the log-stamping fallback when its edge-driven name +// caches miss: known entities return their name, unknown/empty ids return "". +func TestGetTeamNameAndGetCustomerName(t *testing.T) { + logger := NewMockLogger() + store, err := NewLocalGovernanceStore(context.Background(), logger, nil, &configstore.GovernanceConfig{}, nil) + require.NoError(t, err) + + store.CreateTeamInMemory(context.Background(), buildTeam("team-1", "Platform", nil)) + store.CreateCustomerInMemory(context.Background(), buildCustomer("cust-1", "ACME", nil)) + + assert.Equal(t, "Platform", store.GetTeamName(context.Background(), "team-1")) + assert.Equal(t, "ACME", store.GetCustomerName(context.Background(), "cust-1")) + + assert.Empty(t, store.GetTeamName(context.Background(), "unknown")) + assert.Empty(t, store.GetCustomerName(context.Background(), "unknown")) + assert.Empty(t, store.GetTeamName(context.Background(), "")) + assert.Empty(t, store.GetCustomerName(context.Background(), "")) +} + // TestGovernanceStore_Customer_CalendarAligned_CreateInMemory verifies that // CreateCustomerInMemory stamps IsCalendarAligned on the in-memory budget and // rate limit so ResetExpiredBudgetsInMemory uses the calendar-aligned reset path. diff --git a/plugins/governance/utils.go b/plugins/governance/utils.go index b24940d343..a6281d20d4 100644 --- a/plugins/governance/utils.go +++ b/plugins/governance/utils.go @@ -3,11 +3,12 @@ package governance import ( "context" - "slices" + "fmt" "strings" bifrost "github.com/maximhq/bifrost/core" "github.com/maximhq/bifrost/core/schemas" + configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" "github.com/valyala/fasthttp" ) @@ -47,49 +48,14 @@ func IsModelRequiredForRequest(requestType schemas.RequestType) bool { // Here we will have to check for some requests which do not need model // For example, batches, container, files, videos, passthrough requests // For these requests, we will only check for provider filtering - if requestType == schemas.ListModelsRequest || requestType == schemas.MCPToolExecutionRequest || requestType == schemas.BatchCreateRequest || requestType == schemas.BatchListRequest || requestType == schemas.BatchRetrieveRequest || requestType == schemas.BatchCancelRequest || requestType == schemas.BatchResultsRequest || requestType == schemas.FileUploadRequest || requestType == schemas.FileListRequest || requestType == schemas.FileRetrieveRequest || requestType == schemas.FileDeleteRequest || requestType == schemas.FileContentRequest || requestType == schemas.ContainerCreateRequest || requestType == schemas.ContainerListRequest || requestType == schemas.ContainerRetrieveRequest || requestType == schemas.ContainerDeleteRequest || requestType == schemas.ContainerFileCreateRequest || requestType == schemas.ContainerFileListRequest || requestType == schemas.ContainerFileRetrieveRequest || requestType == schemas.ContainerFileContentRequest || requestType == schemas.ContainerFileDeleteRequest || requestType == schemas.VideoRetrieveRequest || requestType == schemas.VideoDownloadRequest || requestType == schemas.VideoListRequest || requestType == schemas.VideoDeleteRequest || requestType == schemas.VideoRemixRequest || requestType == schemas.PassthroughRequest || requestType == schemas.PassthroughStreamRequest { + // Cached content list/retrieve/update/delete target a resource name (cachedContents/{id}), + // not a model, so they carry no model to filter on; only create binds a cache to a model. + if requestType == schemas.ListModelsRequest || requestType == schemas.MCPToolExecutionRequest || requestType == schemas.BatchCreateRequest || requestType == schemas.BatchListRequest || requestType == schemas.BatchRetrieveRequest || requestType == schemas.BatchCancelRequest || requestType == schemas.BatchResultsRequest || requestType == schemas.FileUploadRequest || requestType == schemas.FileListRequest || requestType == schemas.FileRetrieveRequest || requestType == schemas.FileDeleteRequest || requestType == schemas.FileContentRequest || requestType == schemas.ContainerCreateRequest || requestType == schemas.ContainerListRequest || requestType == schemas.ContainerRetrieveRequest || requestType == schemas.ContainerDeleteRequest || requestType == schemas.ContainerFileCreateRequest || requestType == schemas.ContainerFileListRequest || requestType == schemas.ContainerFileRetrieveRequest || requestType == schemas.ContainerFileContentRequest || requestType == schemas.ContainerFileDeleteRequest || requestType == schemas.CachedContentListRequest || requestType == schemas.CachedContentRetrieveRequest || requestType == schemas.CachedContentUpdateRequest || requestType == schemas.CachedContentDeleteRequest || requestType == schemas.VideoRetrieveRequest || requestType == schemas.VideoDownloadRequest || requestType == schemas.VideoListRequest || requestType == schemas.VideoDeleteRequest || requestType == schemas.VideoRemixRequest || requestType == schemas.PassthroughRequest || requestType == schemas.PassthroughStreamRequest { return false } return true } -// parseVirtualKeyFromHTTPRequest parses the virtual key from HTTP request headers. -// It checks multiple headers in order: x-bf-vk, Authorization (Bearer token), x-api-key, and x-goog-api-key. -// Parameters: -// - req: The HTTP request containing headers to parse -// -// Returns: -// - *string: The virtual key if found, nil otherwise -func parseVirtualKeyFromHTTPRequest(req *schemas.HTTPRequest) *string { - var virtualKeyValue string - vkHeader := req.CaseInsensitiveHeaderLookup("x-bf-vk") - if vkHeader != "" && strings.HasPrefix(strings.ToLower(vkHeader), VirtualKeyPrefix) { - return bifrost.Ptr(vkHeader) - } - authHeader := req.CaseInsensitiveHeaderLookup("Authorization") - if authHeader != "" { - if strings.HasPrefix(strings.ToLower(authHeader), "bearer ") { - authHeaderValue := strings.TrimSpace(authHeader[7:]) // Remove "Bearer " prefix - if authHeaderValue != "" && strings.HasPrefix(strings.ToLower(authHeaderValue), VirtualKeyPrefix) { - virtualKeyValue = authHeaderValue - } - } - } - if virtualKeyValue != "" { - return bifrost.Ptr(virtualKeyValue) - } - xAPIKey := req.CaseInsensitiveHeaderLookup("x-api-key") - if xAPIKey != "" && strings.HasPrefix(strings.ToLower(xAPIKey), VirtualKeyPrefix) { - return bifrost.Ptr(xAPIKey) - } - // Checking x-goog-api-key header - xGoogleAPIKey := req.CaseInsensitiveHeaderLookup("x-goog-api-key") - if xGoogleAPIKey != "" && strings.HasPrefix(strings.ToLower(xGoogleAPIKey), VirtualKeyPrefix) { - return bifrost.Ptr(xGoogleAPIKey) - } - return nil -} - // getWeight safely dereferences a *float64 weight pointer, returning 1.0 as default if nil. // This allows distinguishing between "not set" (nil -> 1.0) and "explicitly set to 0" (0.0). func getWeight(w *float64) float64 { @@ -99,34 +65,31 @@ func getWeight(w *float64) float64 { return *w } -func blockedModelCandidates(model string) []string { - _, normalized := schemas.ParseModelString(model, "") - - if strings.EqualFold(model, normalized) { - return []string{model} - } - - return []string{model, normalized} -} - -func isModelBlockedByList(blacklist schemas.BlackList, model string) bool { - if blacklist.IsBlockAll() { - return true - } - - modelForms := blockedModelCandidates(model) - for _, blocked := range blacklist { - blockedForms := blockedModelCandidates(blocked) - for _, form := range modelForms { - if slices.ContainsFunc(blockedForms, func(blockedForm string) bool { - return strings.EqualFold(blockedForm, form) - }) { - return true +// stampGovernanceCtxFromVK copies team/customer identifiers from the VK onto ctx so +// downstream plugins (logging, observability) see the governance scope. +func stampGovernanceCtxFromVK(ctx *schemas.BifrostContext, vk *configstoreTables.TableVirtualKey) { + if vk == nil { + return + } + if vk.TeamID != nil { + ctx.SetValue(schemas.BifrostContextKeyGovernanceTeamID, *vk.TeamID) + } + if vk.Team != nil { + ctx.SetValue(schemas.BifrostContextKeyGovernanceTeamName, vk.Team.Name) + if vk.Team.CustomerID != nil { + ctx.SetValue(schemas.BifrostContextKeyGovernanceCustomerID, *vk.Team.CustomerID) + if vk.Team.Customer != nil { + ctx.SetValue(schemas.BifrostContextKeyGovernanceCustomerName, vk.Team.Customer.Name) } } + } else { + if vk.CustomerID != nil { + ctx.SetValue(schemas.BifrostContextKeyGovernanceCustomerID, *vk.CustomerID) + } + if vk.Customer != nil { + ctx.SetValue(schemas.BifrostContextKeyGovernanceCustomerName, vk.Customer.Name) + } } - - return false } // filterModelsForVirtualKey filters models based on virtual key's provider configs @@ -156,7 +119,7 @@ func (p *GovernancePlugin) filterModelsForVirtualKey( // Pre-pass: if any matching config blacklists the model, block it entirely. isBlocked := false for _, pc := range vk.ProviderConfigs { - if pc.Provider == string(provider) && isModelBlockedByList(pc.BlacklistedModels, modelName) { + if pc.Provider == string(provider) && pc.BlacklistedModels.IsBlocked(modelName) { isBlocked = true break } @@ -195,3 +158,32 @@ func (p *GovernancePlugin) filterModelsForVirtualKey( return filteredModels } + +// validateRequiredHeaders checks that all configured required headers are present in the request. +// Headers are compared case-insensitively (both sides lowercased). +// Returns a BifrostError with status 400 if any required headers are missing, or nil if all present. +func (p *GovernancePlugin) validateRequiredHeaders(ctx *schemas.BifrostContext) *schemas.BifrostError { + if p.requiredHeaders == nil || len(*p.requiredHeaders) == 0 { + return nil + } + headers, _ := ctx.Value(schemas.BifrostContextKeyRequestHeaders).(map[string]string) + if headers == nil { + headers = map[string]string{} + } + var missing []string + for _, h := range *p.requiredHeaders { + if _, ok := headers[strings.ToLower(h)]; !ok { + missing = append(missing, h) + } + } + if len(missing) > 0 { + return &schemas.BifrostError{ + Type: bifrost.Ptr("missing_required_headers"), + StatusCode: bifrost.Ptr(400), + Error: &schemas.ErrorField{ + Message: fmt.Sprintf("missing required headers: %s", strings.Join(missing, ", ")), + }, + } + } + return nil +} diff --git a/plugins/jsonparser/go.mod b/plugins/jsonparser/go.mod index 8842e11460..03469d6567 100644 --- a/plugins/jsonparser/go.mod +++ b/plugins/jsonparser/go.mod @@ -12,13 +12,13 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect github.com/andybalholm/brotli v1.2.1 // indirect - github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect + github.com/aws/aws-sdk-go-v2 v1.41.12 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect github.com/aws/aws-sdk-go-v2/config v1.32.11 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect @@ -30,7 +30,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect - github.com/aws/smithy-go v1.25.1 // indirect + github.com/aws/smithy-go v1.27.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.2 // indirect github.com/bytedance/gopkg v0.1.3 // indirect diff --git a/plugins/jsonparser/go.sum b/plugins/jsonparser/go.sum index 3bbec4219d..50491fbed5 100644 --- a/plugins/jsonparser/go.sum +++ b/plugins/jsonparser/go.sum @@ -16,8 +16,8 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgv github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro= github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= -github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= -github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2 v1.41.12 h1:DIKX2c31ekm9RA2D9FBj1EWXx++9AdAqRw+e78Tq2Ck= +github.com/aws/aws-sdk-go-v2 v1.41.12/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= github.com/aws/aws-sdk-go-v2/config v1.32.11 h1:ftxI5sgz8jZkckuUHXfC/wMUc8u3fG1vQS0plr2F2Zs= @@ -26,10 +26,10 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8TH github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 h1:Xf2j7NdVcUKomlZ4iihOP4AZ3Fzlr8h4yKpXeP+OFPg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28/go.mod h1:O8cDo1dW63jU7ki//kRe1z+tLGcpnD1jrouitsQddDw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 h1:KqIfN9kpkKkcBqBbNpNGTIrXO6ExTUvFKvXkC+YAzVo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28/go.mod h1:uxtQiKvLtNS4iXVsH2McVD/ls8FKN/uUhe1hGxPjrw0= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 h1:clHU5fm//kWS1C2HgtgWxfQbFbx4b6rx+5jzhgX9HrI= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= @@ -52,8 +52,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6f github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= -github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= -github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8= +github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= diff --git a/plugins/jsonparser/main.go b/plugins/jsonparser/main.go index c0de696b0d..5cef86e4a8 100644 --- a/plugins/jsonparser/main.go +++ b/plugins/jsonparser/main.go @@ -98,6 +98,11 @@ func (p *JsonParserPlugin) HTTPTransportStreamChunkHook(ctx *schemas.BifrostCont return chunk, nil } +// PreRequestHook implements schemas.LLMPlugin (no-op — required for plugin indexing). +func (p *JsonParserPlugin) PreRequestHook(_ *schemas.BifrostContext, _ *schemas.BifrostRequest) error { + return nil +} + // PreLLMHook is not used for this plugin as we only process responses // Parameters: // - ctx: The Bifrost context diff --git a/plugins/logging/go.mod b/plugins/logging/go.mod index 9f165715cc..845f21c6ae 100644 --- a/plugins/logging/go.mod +++ b/plugins/logging/go.mod @@ -14,7 +14,7 @@ require ( cloud.google.com/go/auth v0.20.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect - cloud.google.com/go/iam v1.5.3 // indirect + cloud.google.com/go/iam v1.7.0 // indirect cloud.google.com/go/monitoring v1.24.3 // indirect cloud.google.com/go/storage v1.61.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect @@ -26,13 +26,13 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect github.com/andybalholm/brotli v1.2.1 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect - github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect + github.com/aws/aws-sdk-go-v2 v1.41.12 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect github.com/aws/aws-sdk-go-v2/config v1.32.11 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect @@ -44,7 +44,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect - github.com/aws/smithy-go v1.25.1 // indirect + github.com/aws/smithy-go v1.27.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.2 // indirect github.com/bytedance/gopkg v0.1.3 // indirect diff --git a/plugins/logging/go.sum b/plugins/logging/go.sum index 0e3b3e6274..2aa5cb03f9 100644 --- a/plugins/logging/go.sum +++ b/plugins/logging/go.sum @@ -8,8 +8,8 @@ cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIi cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= -cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= +cloud.google.com/go/iam v1.7.0 h1:JD3zh0C6LHl16aCn5Akff0+GELdp1+4hmh6ndoFLl8U= +cloud.google.com/go/iam v1.7.0/go.mod h1:tetWZW1PD/m6vcuY2Zj/aU0eCHNPuxedbnbRTyKXvdY= cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHAjA= cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZCn6hSBtKak= cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8= @@ -45,8 +45,8 @@ github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eT github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= -github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= -github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2 v1.41.12 h1:DIKX2c31ekm9RA2D9FBj1EWXx++9AdAqRw+e78Tq2Ck= +github.com/aws/aws-sdk-go-v2 v1.41.12/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= github.com/aws/aws-sdk-go-v2/config v1.32.11 h1:ftxI5sgz8jZkckuUHXfC/wMUc8u3fG1vQS0plr2F2Zs= @@ -55,10 +55,10 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8TH github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 h1:Xf2j7NdVcUKomlZ4iihOP4AZ3Fzlr8h4yKpXeP+OFPg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28/go.mod h1:O8cDo1dW63jU7ki//kRe1z+tLGcpnD1jrouitsQddDw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 h1:KqIfN9kpkKkcBqBbNpNGTIrXO6ExTUvFKvXkC+YAzVo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28/go.mod h1:uxtQiKvLtNS4iXVsH2McVD/ls8FKN/uUhe1hGxPjrw0= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 h1:clHU5fm//kWS1C2HgtgWxfQbFbx4b6rx+5jzhgX9HrI= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= @@ -81,8 +81,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6f github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= -github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= -github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8= +github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= diff --git a/plugins/logging/main.go b/plugins/logging/main.go index 3e21b23120..d9c8202ddf 100644 --- a/plugins/logging/main.go +++ b/plugins/logging/main.go @@ -503,6 +503,11 @@ func (p *LoggerPlugin) captureLoggingHeaders(ctx *schemas.BifrostContext) map[st return metadata } +// PreRequestHook implements schemas.LLMPlugin (no-op — required for plugin indexing). +func (p *LoggerPlugin) PreRequestHook(_ *schemas.BifrostContext, _ *schemas.BifrostRequest) error { + return nil +} + // PreLLMHook is called before a request is processed - FULLY ASYNC, NO DATABASE I/O // Parameters: // - ctx: The Bifrost context diff --git a/plugins/maxim/go.mod b/plugins/maxim/go.mod index 4d2a41649c..3be2780c56 100644 --- a/plugins/maxim/go.mod +++ b/plugins/maxim/go.mod @@ -19,7 +19,7 @@ require ( cloud.google.com/go/auth v0.20.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect - cloud.google.com/go/iam v1.5.3 // indirect + cloud.google.com/go/iam v1.7.0 // indirect cloud.google.com/go/monitoring v1.24.3 // indirect cloud.google.com/go/storage v1.61.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect @@ -31,13 +31,13 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect github.com/andybalholm/brotli v1.2.1 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect - github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect + github.com/aws/aws-sdk-go-v2 v1.41.12 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect github.com/aws/aws-sdk-go-v2/config v1.32.11 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect @@ -49,7 +49,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect - github.com/aws/smithy-go v1.25.1 // indirect + github.com/aws/smithy-go v1.27.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.2 // indirect github.com/bytedance/gopkg v0.1.3 // indirect diff --git a/plugins/maxim/go.sum b/plugins/maxim/go.sum index 6da2e34685..f191c35e6d 100644 --- a/plugins/maxim/go.sum +++ b/plugins/maxim/go.sum @@ -8,8 +8,8 @@ cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIi cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= -cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= +cloud.google.com/go/iam v1.7.0 h1:JD3zh0C6LHl16aCn5Akff0+GELdp1+4hmh6ndoFLl8U= +cloud.google.com/go/iam v1.7.0/go.mod h1:tetWZW1PD/m6vcuY2Zj/aU0eCHNPuxedbnbRTyKXvdY= cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHAjA= cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZCn6hSBtKak= cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8= @@ -45,8 +45,8 @@ github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eT github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= -github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= -github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2 v1.41.12 h1:DIKX2c31ekm9RA2D9FBj1EWXx++9AdAqRw+e78Tq2Ck= +github.com/aws/aws-sdk-go-v2 v1.41.12/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= github.com/aws/aws-sdk-go-v2/config v1.32.11 h1:ftxI5sgz8jZkckuUHXfC/wMUc8u3fG1vQS0plr2F2Zs= @@ -55,10 +55,10 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8TH github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 h1:Xf2j7NdVcUKomlZ4iihOP4AZ3Fzlr8h4yKpXeP+OFPg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28/go.mod h1:O8cDo1dW63jU7ki//kRe1z+tLGcpnD1jrouitsQddDw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 h1:KqIfN9kpkKkcBqBbNpNGTIrXO6ExTUvFKvXkC+YAzVo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28/go.mod h1:uxtQiKvLtNS4iXVsH2McVD/ls8FKN/uUhe1hGxPjrw0= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 h1:clHU5fm//kWS1C2HgtgWxfQbFbx4b6rx+5jzhgX9HrI= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= @@ -81,8 +81,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6f github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= -github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= -github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8= +github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= diff --git a/plugins/maxim/main.go b/plugins/maxim/main.go index ace1329e6a..14022998a6 100644 --- a/plugins/maxim/main.go +++ b/plugins/maxim/main.go @@ -229,6 +229,11 @@ func (plugin *Plugin) getOrCreateLogger(logRepoID string) (*logging.Logger, erro return logger, nil } +// PreRequestHook implements schemas.LLMPlugin (no-op — required for plugin indexing). +func (plugin *Plugin) PreRequestHook(_ *schemas.BifrostContext, _ *schemas.BifrostRequest) error { + return nil +} + // PreLLMHook is called before a request is processed by Bifrost. // It manages trace and generation tracking for incoming requests by either: // - Creating a new trace if none exists diff --git a/plugins/mocker/go.mod b/plugins/mocker/go.mod index 6e3fa596a5..d129dc486f 100644 --- a/plugins/mocker/go.mod +++ b/plugins/mocker/go.mod @@ -15,13 +15,13 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect github.com/andybalholm/brotli v1.2.1 // indirect - github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect + github.com/aws/aws-sdk-go-v2 v1.41.12 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect github.com/aws/aws-sdk-go-v2/config v1.32.11 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect @@ -33,7 +33,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect - github.com/aws/smithy-go v1.25.1 // indirect + github.com/aws/smithy-go v1.27.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.2 // indirect github.com/bytedance/gopkg v0.1.3 // indirect diff --git a/plugins/mocker/go.sum b/plugins/mocker/go.sum index e49912a4d8..c0c2bfa945 100644 --- a/plugins/mocker/go.sum +++ b/plugins/mocker/go.sum @@ -16,8 +16,8 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgv github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro= github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= -github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= -github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2 v1.41.12 h1:DIKX2c31ekm9RA2D9FBj1EWXx++9AdAqRw+e78Tq2Ck= +github.com/aws/aws-sdk-go-v2 v1.41.12/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= github.com/aws/aws-sdk-go-v2/config v1.32.11 h1:ftxI5sgz8jZkckuUHXfC/wMUc8u3fG1vQS0plr2F2Zs= @@ -26,10 +26,10 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8TH github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 h1:Xf2j7NdVcUKomlZ4iihOP4AZ3Fzlr8h4yKpXeP+OFPg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28/go.mod h1:O8cDo1dW63jU7ki//kRe1z+tLGcpnD1jrouitsQddDw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 h1:KqIfN9kpkKkcBqBbNpNGTIrXO6ExTUvFKvXkC+YAzVo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28/go.mod h1:uxtQiKvLtNS4iXVsH2McVD/ls8FKN/uUhe1hGxPjrw0= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 h1:clHU5fm//kWS1C2HgtgWxfQbFbx4b6rx+5jzhgX9HrI= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= @@ -52,8 +52,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6f github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= -github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= -github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8= +github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= diff --git a/plugins/mocker/main.go b/plugins/mocker/main.go index 53ce896615..06c79a6612 100644 --- a/plugins/mocker/main.go +++ b/plugins/mocker/main.go @@ -495,6 +495,11 @@ func (p *MockerPlugin) HTTPTransportStreamChunkHook(ctx *schemas.BifrostContext, return chunk, nil } +// PreRequestHook implements schemas.LLMPlugin (no-op — required for plugin indexing). +func (p *MockerPlugin) PreRequestHook(_ *schemas.BifrostContext, _ *schemas.BifrostRequest) error { + return nil +} + // PreLLMHook intercepts requests and applies mocking rules based on configuration // This is called before the actual provider request and can short-circuit the flow func (p *MockerPlugin) PreLLMHook(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) (*schemas.BifrostRequest, *schemas.LLMPluginShortCircuit, error) { @@ -853,10 +858,10 @@ func (p *MockerPlugin) generateSuccessShortCircuit(req *schemas.BifrostRequest, }, }, ExtraFields: schemas.BifrostResponseExtraFields{ - RequestType: req.RequestType, - Provider: provider, + RequestType: req.RequestType, + Provider: provider, OriginalModelRequested: model, - Latency: int64(time.Since(startTime).Milliseconds()), + Latency: int64(time.Since(startTime).Milliseconds()), }, } } else if req.RequestType == schemas.ResponsesRequest { @@ -877,10 +882,10 @@ func (p *MockerPlugin) generateSuccessShortCircuit(req *schemas.BifrostRequest, TotalTokens: usage.TotalTokens, }, ExtraFields: schemas.BifrostResponseExtraFields{ - RequestType: schemas.ResponsesRequest, - Provider: provider, + RequestType: schemas.ResponsesRequest, + Provider: provider, OriginalModelRequested: model, - Latency: int64(time.Since(startTime).Milliseconds()), + Latency: int64(time.Since(startTime).Milliseconds()), }, } } else if req.RequestType == schemas.ResponsesStreamRequest { @@ -905,10 +910,10 @@ func (p *MockerPlugin) generateSuccessShortCircuit(req *schemas.BifrostRequest, }, }, ExtraFields: schemas.BifrostResponseExtraFields{ - RequestType: schemas.ResponsesStreamRequest, - Provider: provider, + RequestType: schemas.ResponsesStreamRequest, + Provider: provider, OriginalModelRequested: model, - Latency: int64(time.Since(startTime).Milliseconds()), + Latency: int64(time.Since(startTime).Milliseconds()), }, } } @@ -959,8 +964,8 @@ func (p *MockerPlugin) generateErrorShortCircuit(req *schemas.BifrostRequest, re }, AllowFallbacks: allowFallbacks, ExtraFields: schemas.BifrostErrorExtraFields{ - RequestType: req.RequestType, - Provider: provider, + RequestType: req.RequestType, + Provider: provider, OriginalModelRequested: model, }, } @@ -1083,8 +1088,8 @@ func (p *MockerPlugin) handleDefaultBehavior(req *schemas.BifrostRequest) (*sche }, }, ExtraFields: schemas.BifrostResponseExtraFields{ - RequestType: schemas.ChatCompletionRequest, - Provider: provider, + RequestType: schemas.ChatCompletionRequest, + Provider: provider, OriginalModelRequested: model, }, }, diff --git a/plugins/modelcatalogresolver/go.mod b/plugins/modelcatalogresolver/go.mod new file mode 100644 index 0000000000..c311cf833d --- /dev/null +++ b/plugins/modelcatalogresolver/go.mod @@ -0,0 +1,159 @@ +module github.com/maximhq/bifrost/plugins/modelcatalogresolver + +go 1.26.3 + +require ( + github.com/maximhq/bifrost/core v1.5.15 + github.com/maximhq/bifrost/framework v1.3.15 +) + +require ( + cel.dev/expr v0.25.1 // indirect + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.20.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.7.0 // indirect + cloud.google.com/go/monitoring v1.24.3 // indirect + cloud.google.com/go/storage v1.61.3 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect + github.com/andybalholm/brotli v1.2.1 // indirect + github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.41.12 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect + github.com/aws/aws-sdk-go-v2/config v1.32.11 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect + github.com/aws/smithy-go v1.27.1 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/buger/jsonparser v1.1.2 // indirect + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic v1.15.1 // indirect + github.com/bytedance/sonic/loader v0.5.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-openapi/analysis v0.24.2 // indirect + github.com/go-openapi/errors v0.22.5 // indirect + github.com/go-openapi/jsonpointer v0.22.4 // indirect + github.com/go-openapi/jsonreference v0.21.4 // indirect + github.com/go-openapi/loads v0.23.2 // indirect + github.com/go-openapi/runtime v0.29.2 // indirect + github.com/go-openapi/spec v0.22.2 // indirect + github.com/go-openapi/strfmt v0.25.0 // indirect + github.com/go-openapi/swag v0.25.4 // indirect + github.com/go-openapi/swag/cmdutils v0.25.4 // indirect + github.com/go-openapi/swag/conv v0.25.4 // indirect + github.com/go-openapi/swag/fileutils v0.25.4 // indirect + github.com/go-openapi/swag/jsonname v0.25.4 // indirect + github.com/go-openapi/swag/jsonutils v0.25.4 // indirect + github.com/go-openapi/swag/loading v0.25.4 // indirect + github.com/go-openapi/swag/mangling v0.25.4 // indirect + github.com/go-openapi/swag/netutils v0.25.4 // indirect + github.com/go-openapi/swag/stringutils v0.25.4 // indirect + github.com/go-openapi/swag/typeutils v0.25.4 // indirect + github.com/go-openapi/swag/yamlutils v0.25.4 // indirect + github.com/go-openapi/validate v0.25.1 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.16 // indirect + github.com/googleapis/gax-go/v2 v2.22.0 // indirect + github.com/invopop/jsonschema v0.13.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.9.2 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + github.com/klauspost/compress v1.18.6 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/mailru/easyjson v0.9.1 // indirect + github.com/mark3labs/mcp-go v0.43.2 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-sqlite3 v1.14.32 // indirect + github.com/oapi-codegen/runtime v1.1.1 // indirect + github.com/oklog/ulid v1.3.1 // indirect + github.com/pinecone-io/go-pinecone/v5 v5.3.0 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/qdrant/go-client v1.16.2 // indirect + github.com/redis/go-redis/v9 v9.17.2 // indirect + github.com/rs/zerolog v1.34.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect + github.com/stretchr/testify v1.11.1 // indirect + github.com/tidwall/gjson v1.18.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect + github.com/tidwall/sjson v1.2.5 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.71.0 // indirect + github.com/weaviate/weaviate v1.36.5 // indirect + github.com/weaviate/weaviate-go-client/v5 v5.7.1 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect + go.mongodb.org/mongo-driver v1.17.6 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.42.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/sdk v1.43.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect + go.starlark.net v0.0.0-20260102030733-3fee463870c9 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/arch v0.23.0 // indirect + golang.org/x/crypto v0.52.0 // indirect + golang.org/x/net v0.55.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.45.0 // indirect + golang.org/x/text v0.37.0 // indirect + golang.org/x/time v0.15.0 // indirect + google.golang.org/api v0.282.0 // indirect + google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260523011958-0a33c5d7ca68 // indirect + google.golang.org/grpc v1.81.1 // indirect + google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + gorm.io/driver/postgres v1.6.0 // indirect + gorm.io/driver/sqlite v1.6.0 // indirect + gorm.io/gorm v1.31.1 // indirect +) diff --git a/plugins/modelcatalogresolver/go.sum b/plugins/modelcatalogresolver/go.sum new file mode 100644 index 0000000000..2c75ea025a --- /dev/null +++ b/plugins/modelcatalogresolver/go.sum @@ -0,0 +1,382 @@ +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= +cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/iam v1.7.0 h1:JD3zh0C6LHl16aCn5Akff0+GELdp1+4hmh6ndoFLl8U= +cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHAjA= +cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZCn6hSBtKak= +cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8= +cloud.google.com/go/longrunning v0.8.0/go.mod h1:UmErU2Onzi+fKDg2gR7dusz11Pe26aknR4kHmJJqIfk= +cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE= +cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI= +cloud.google.com/go/storage v1.61.3 h1:VS//ZfBuPGDvakfD9xyPW1RGF1Vy3BWUoVZXgW1KMOg= +cloud.google.com/go/storage v1.61.3/go.mod h1:JtqK8BBB7TWv0HVGHubtUdzYYrakOQIsMLffZ2Z/HWk= +cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U= +cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 h1:JXg2dwJUmPB9JmtVmdEB16APJ7jurfbY5jnfXpJoRMc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 h1:DHa2U07rk8syqvCge0QIGMCE1WxGj9njT44GH7zNJLQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0 h1:7t/qx5Ost0s0wbA/VDrByOooURhp+ikYwv20i9Y07TQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= +github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= +github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro= +github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= +github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= +github.com/aws/aws-sdk-go-v2 v1.41.12 h1:DIKX2c31ekm9RA2D9FBj1EWXx++9AdAqRw+e78Tq2Ck= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= +github.com/aws/aws-sdk-go-v2/config v1.32.11 h1:ftxI5sgz8jZkckuUHXfC/wMUc8u3fG1vQS0plr2F2Zs= +github.com/aws/aws-sdk-go-v2/config v1.32.11/go.mod h1:twF11+6ps9aNRKEDimksp923o44w/Thk9+8YIlzWMmo= +github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI= +github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 h1:Xf2j7NdVcUKomlZ4iihOP4AZ3Fzlr8h4yKpXeP+OFPg= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 h1:KqIfN9kpkKkcBqBbNpNGTIrXO6ExTUvFKvXkC+YAzVo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 h1:clHU5fm//kWS1C2HgtgWxfQbFbx4b6rx+5jzhgX9HrI= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 h1:HwxWTbTrIHm5qY+CAEur0s/figc3qwvLWsNkF4RPToo= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 h1:lFd1+ZSEYJZYvv9d6kXzhkZu07si3f+GQ1AaYwa2LUM= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.15/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6fuOwWlWpD2StNLTceKpys= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= +github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= +github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/sonic v1.15.1 h1:nJD5PmM0vY7J8CT6MxoqbVAAMhkSmV2HgRAUrrpLoOw= +github.com/bytedance/sonic v1.15.1/go.mod h1:mT2NbXunuaEbnZ+mRIX/vYqKISmgEuHFDI4UzmKx2SA= +github.com/bytedance/sonic/loader v0.5.1 h1:Ygpfa9zwRCCKSlrp5bBP/b/Xzc3VxsAW+5NIYXrOOpI= +github.com/bytedance/sonic/loader v0.5.1/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= +github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= +github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= +github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= +github.com/fasthttp/websocket v1.5.12 h1:e4RGPpWW2HTbL3zV0Y/t7g0ub294LkiuXXUuTOUInlE= +github.com/fasthttp/websocket v1.5.12/go.mod h1:I+liyL7/4moHojiOgUOIKEWm9EIxHqxZChS+aMFltyg= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-openapi/analysis v0.24.2 h1:6p7WXEuKy1llDgOH8FooVeO+Uq2za9qoAOq4ZN08B50= +github.com/go-openapi/analysis v0.24.2/go.mod h1:x27OOHKANE0lutg2ml4kzYLoHGMKgRm1Cj2ijVOjJuE= +github.com/go-openapi/errors v0.22.5 h1:Yfv4O/PRYpNF3BNmVkEizcHb3uLVVsrDt3LNdgAKRY4= +github.com/go-openapi/errors v0.22.5/go.mod h1:z9S8ASTUqx7+CP1Q8dD8ewGH/1JWFFLX/2PmAYNQLgk= +github.com/go-openapi/jsonpointer v0.22.4 h1:dZtK82WlNpVLDW2jlA1YCiVJFVqkED1MegOUy9kR5T4= +github.com/go-openapi/jsonpointer v0.22.4/go.mod h1:elX9+UgznpFhgBuaMQ7iu4lvvX1nvNsesQ3oxmYTw80= +github.com/go-openapi/jsonreference v0.21.4 h1:24qaE2y9bx/q3uRK/qN+TDwbok1NhbSmGjjySRCHtC8= +github.com/go-openapi/jsonreference v0.21.4/go.mod h1:rIENPTjDbLpzQmQWCj5kKj3ZlmEh+EFVbz3RTUh30/4= +github.com/go-openapi/loads v0.23.2 h1:rJXAcP7g1+lWyBHC7iTY+WAF0rprtM+pm8Jxv1uQJp4= +github.com/go-openapi/loads v0.23.2/go.mod h1:IEVw1GfRt/P2Pplkelxzj9BYFajiWOtY2nHZNj4UnWY= +github.com/go-openapi/runtime v0.29.2 h1:UmwSGWNmWQqKm1c2MGgXVpC2FTGwPDQeUsBMufc5Yj0= +github.com/go-openapi/runtime v0.29.2/go.mod h1:biq5kJXRJKBJxTDJXAa00DOTa/anflQPhT0/wmjuy+0= +github.com/go-openapi/spec v0.22.2 h1:KEU4Fb+Lp1qg0V4MxrSCPv403ZjBl8Lx1a83gIPU8Qc= +github.com/go-openapi/spec v0.22.2/go.mod h1:iIImLODL2loCh3Vnox8TY2YWYJZjMAKYyLH2Mu8lOZs= +github.com/go-openapi/strfmt v0.25.0 h1:7R0RX7mbKLa9EYCTHRcCuIPcaqlyQiWNPTXwClK0saQ= +github.com/go-openapi/strfmt v0.25.0/go.mod h1:nNXct7OzbwrMY9+5tLX4I21pzcmE6ccMGXl3jFdPfn8= +github.com/go-openapi/swag v0.25.4 h1:OyUPUFYDPDBMkqyxOTkqDYFnrhuhi9NR6QVUvIochMU= +github.com/go-openapi/swag v0.25.4/go.mod h1:zNfJ9WZABGHCFg2RnY0S4IOkAcVTzJ6z2Bi+Q4i6qFQ= +github.com/go-openapi/swag/cmdutils v0.25.4 h1:8rYhB5n6WawR192/BfUu2iVlxqVR9aRgGJP6WaBoW+4= +github.com/go-openapi/swag/cmdutils v0.25.4/go.mod h1:pdae/AFo6WxLl5L0rq87eRzVPm/XRHM3MoYgRMvG4A0= +github.com/go-openapi/swag/conv v0.25.4 h1:/Dd7p0LZXczgUcC/Ikm1+YqVzkEeCc9LnOWjfkpkfe4= +github.com/go-openapi/swag/conv v0.25.4/go.mod h1:3LXfie/lwoAv0NHoEuY1hjoFAYkvlqI/Bn5EQDD3PPU= +github.com/go-openapi/swag/fileutils v0.25.4 h1:2oI0XNW5y6UWZTC7vAxC8hmsK/tOkWXHJQH4lKjqw+Y= +github.com/go-openapi/swag/fileutils v0.25.4/go.mod h1:cdOT/PKbwcysVQ9Tpr0q20lQKH7MGhOEb6EwmHOirUk= +github.com/go-openapi/swag/jsonname v0.25.4 h1:bZH0+MsS03MbnwBXYhuTttMOqk+5KcQ9869Vye1bNHI= +github.com/go-openapi/swag/jsonname v0.25.4/go.mod h1:GPVEk9CWVhNvWhZgrnvRA6utbAltopbKwDu8mXNUMag= +github.com/go-openapi/swag/jsonutils v0.25.4 h1:VSchfbGhD4UTf4vCdR2F4TLBdLwHyUDTd1/q4i+jGZA= +github.com/go-openapi/swag/jsonutils v0.25.4/go.mod h1:7OYGXpvVFPn4PpaSdPHJBtF0iGnbEaTk8AvBkoWnaAY= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4 h1:IACsSvBhiNJwlDix7wq39SS2Fh7lUOCJRmx/4SN4sVo= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4/go.mod h1:Mt0Ost9l3cUzVv4OEZG+WSeoHwjWLnarzMePNDAOBiM= +github.com/go-openapi/swag/loading v0.25.4 h1:jN4MvLj0X6yhCDduRsxDDw1aHe+ZWoLjW+9ZQWIKn2s= +github.com/go-openapi/swag/loading v0.25.4/go.mod h1:rpUM1ZiyEP9+mNLIQUdMiD7dCETXvkkC30z53i+ftTE= +github.com/go-openapi/swag/mangling v0.25.4 h1:2b9kBJk9JvPgxr36V23FxJLdwBrpijI26Bx5JH4Hp48= +github.com/go-openapi/swag/mangling v0.25.4/go.mod h1:6dxwu6QyORHpIIApsdZgb6wBk/DPU15MdyYj/ikn0Hg= +github.com/go-openapi/swag/netutils v0.25.4 h1:Gqe6K71bGRb3ZQLusdI8p/y1KLgV4M/k+/HzVSqT8H0= +github.com/go-openapi/swag/netutils v0.25.4/go.mod h1:m2W8dtdaoX7oj9rEttLyTeEFFEBvnAx9qHd5nJEBzYg= +github.com/go-openapi/swag/stringutils v0.25.4 h1:O6dU1Rd8bej4HPA3/CLPciNBBDwZj9HiEpdVsb8B5A8= +github.com/go-openapi/swag/stringutils v0.25.4/go.mod h1:GTsRvhJW5xM5gkgiFe0fV3PUlFm0dr8vki6/VSRaZK0= +github.com/go-openapi/swag/typeutils v0.25.4 h1:1/fbZOUN472NTc39zpa+YGHn3jzHWhv42wAJSN91wRw= +github.com/go-openapi/swag/typeutils v0.25.4/go.mod h1:Ou7g//Wx8tTLS9vG0UmzfCsjZjKhpjxayRKTHXf2pTE= +github.com/go-openapi/swag/yamlutils v0.25.4 h1:6jdaeSItEUb7ioS9lFoCZ65Cne1/RZtPBZ9A56h92Sw= +github.com/go-openapi/swag/yamlutils v0.25.4/go.mod h1:MNzq1ulQu+yd8Kl7wPOut/YHAAU/H6hL91fF+E2RFwc= +github.com/go-openapi/testify/enable/yaml/v2 v2.0.2 h1:0+Y41Pz1NkbTHz8NngxTuAXxEodtNSI1WG1c/m5Akw4= +github.com/go-openapi/testify/enable/yaml/v2 v2.0.2/go.mod h1:kme83333GCtJQHXQ8UKX3IBZu6z8T5Dvy5+CW3NLUUg= +github.com/go-openapi/testify/v2 v2.0.2 h1:X999g3jeLcoY8qctY/c/Z8iBHTbwLz7R2WXd6Ub6wls= +github.com/go-openapi/testify/v2 v2.0.2/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= +github.com/go-openapi/validate v0.25.1 h1:sSACUI6Jcnbo5IWqbYHgjibrhhmt3vR6lCzKZnmAgBw= +github.com/go-openapi/validate v0.25.1/go.mod h1:RMVyVFYte0gbSTaZ0N4KmTn6u/kClvAFp+mAVfS/DQc= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.16 h1:F/VPrx0YPBdksZJQdCAp0WUsqnNmZpUZszzfYt0M5Dw= +github.com/googleapis/enterprise-certificate-proxy v0.3.16/go.mod h1:9Yb0eAkH/Xqhvv3zbeKf/+wMJqCeocWc6KIhDvEAuYE= +github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU5vlZD4= +github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= +github.com/hajimehoshi/go-mp3 v0.3.4 h1:NUP7pBYH8OguP4diaTZ9wJbUbk3tC0KlfzsEpWmYj68= +github.com/hajimehoshi/go-mp3 v0.3.4/go.mod h1:fRtZraRFcWb0pu7ok0LqyFhCUrPeMsGRSVop0eemFmo= +github.com/invopop/jsonschema v0.13.0 h1:KvpoAJWEjR3uD9Kbm2HWJmqsEaHt8lBUpd0qHcIi21E= +github.com/invopop/jsonschema v0.13.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw= +github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= +github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= +github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mailru/easyjson v0.9.1 h1:LbtsOm5WAswyWbvTEOqhypdPeZzHavpZx96/n553mR8= +github.com/mailru/easyjson v0.9.1/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= +github.com/mark3labs/mcp-go v0.43.2 h1:21PUSlWWiSbUPQwXIJ5WKlETixpFpq+WBpbMGDSVy/I= +github.com/mark3labs/mcp-go v0.43.2/go.mod h1:YnJfOL382MIWDx1kMY+2zsRHU/q78dBg9aFb8W6Thdw= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= +github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/maximhq/bifrost/core v1.5.15 h1:iXvDufyZd7willDmbVzFfzCdy/2NsKEI1S8Iv9LjCSM= +github.com/maximhq/bifrost/core v1.5.15/go.mod h1:f6QHCvvzCQziMZ4JCNZP/GdZSeD50hww0vt7Uwl7lYY= +github.com/maximhq/bifrost/framework v1.3.15 h1:Lf/0S5bmD6i4NU+GdhAdrDZltD8RSvXLuR1vv4eh30I= +github.com/maximhq/bifrost/framework v1.3.15/go.mod h1:FlqWzdsFwal2XIG1Ousk/P76zjjHN/MmV5XR3YqD2+8= +github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= +github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/pinecone-io/go-pinecone/v5 v5.3.0 h1:0YQlEtmXGWK/I8ztkOVM6PuBYgFJZhjSdb0ddU+bHPE= +github.com/pinecone-io/go-pinecone/v5 v5.3.0/go.mod h1:6Fg85fcyvMUQFf9KW7zniN81kelSYvsjF+KPLdc1MGA= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/qdrant/go-client v1.16.2 h1:UUMJJfvXTByhwhH1DwWdbkhZ2cTdvSqVkXSIfBrVWSg= +github.com/qdrant/go-client v1.16.2/go.mod h1:I+EL3h4HRoRTeHtbfOd/4kDXwCukZfkd41j/9wryGkw= +github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI= +github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= +github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= +github.com/savsgio/gotils v0.0.0-20250408102913-196191ec6287 h1:qIQ0tWF9vxGtkJa24bR+2i53WBCz1nW/Pc47oVYauC4= +github.com/savsgio/gotils v0.0.0-20250408102913-196191ec6287/go.mod h1:sM7Mt7uEoCeFSCBM+qBrqvEo+/9vdmj19wzp3yzUhmg= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= +github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= +github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.71.0 h1:tepR7H+Guh9VUqxxcPggYi8R3lGUu2Rsdh+z7/FCY3k= +github.com/valyala/fasthttp v1.71.0/go.mod h1:z1sDUvOShhXq/C9mwH/fSm1Vb71tUJwmQdgkBrBNwnA= +github.com/weaviate/weaviate v1.36.5 h1:lCiuEfQ08+5wK0DkTCUBb6ayNep9QpBH6JJhmZaRfzk= +github.com/weaviate/weaviate v1.36.5/go.mod h1:ljzrgEmGKn3CRzDdcxvhmBUUZIcghwIYd1Lmn54f3Z8= +github.com/weaviate/weaviate-go-client/v5 v5.7.1 h1:vEMxh486QqRqWaq58UEe/TiTbGbo9T5x7ZPFd5QENvQ= +github.com/weaviate/weaviate-go-client/v5 v5.7.1/go.mod h1:T/JDErjN074GrnYIa0AgK1TGUGP/6A/8vqXNPlv4c6E= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= +github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= +go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss= +go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/detectors/gcp v1.42.0 h1:kpt2PEJuOuqYkPcktfJqWWDjTEd/FNgrxcniL7kQrXQ= +go.opentelemetry.io/contrib/detectors/gcp v1.42.0/go.mod h1:W9zQ439utxymRrXsUOzZbFX4JhLxXU4+ZnCt8GG7yA8= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.40.0 h1:ZrPRak/kS4xI3AVXy8F7pipuDXmDsrO8Lg+yQjBLjw0= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.40.0/go.mod h1:3y6kQCWztq6hyW8Z9YxQDDm0Je9AJoFar2G0yDcmhRk= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.starlark.net v0.0.0-20260102030733-3fee463870c9 h1:nV1OyvU+0CYrp5eKfQ3rD03TpFYYhH08z31NK1HmtTk= +go.starlark.net v0.0.0-20260102030733-3fee463870c9/go.mod h1:YKMCv9b1WrfWmeqdV5MAuEHWsu5iC+fe6kYl2sQjdI8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/arch v0.23.0 h1:lKF64A2jF6Zd8L0knGltUnegD62JMFBiCPBmQpToHhg= +golang.org/x/arch v0.23.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= +golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= +golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/api v0.282.0 h1:WmJiSVqUnKqJCpJOx7YADbXaC+9DDsnGSfllFSj7R2I= +google.golang.org/api v0.282.0/go.mod h1:6Wssta4c5n9qHq5CBhmlai5h/PUa1djdDAIhYEHyvcM= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260523011958-0a33c5d7ca68 h1:PvEgGJf9C/1u5CHkInMg7UFYYUoiaQmW2LbtH0pjB78= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260523011958-0a33c5d7ca68/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= +google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4= +gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo= +gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= +gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8= +gorm.io/gorm v1.31.1 h1:7CA8FTFz/gRfgqgpeKIBcervUn3xSyPUmr6B2WXJ7kg= +gorm.io/gorm v1.31.1/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs= \ No newline at end of file diff --git a/plugins/modelcatalogresolver/main.go b/plugins/modelcatalogresolver/main.go new file mode 100644 index 0000000000..df251edb30 --- /dev/null +++ b/plugins/modelcatalogresolver/main.go @@ -0,0 +1,253 @@ +// Package modelcatalogresolver provides a built-in PreRequestHook plugin that resolves +// the default provider for an unprefixed model via the model catalog. It is the single +// owner of "if no provider specified, look up which providers serve this model" — the +// transport handlers, integrations router, and realtime handlers no longer do this +// inline. Governance/LB plugins run before this resolver; it only fires as a final +// fallback when no earlier routing plugin picked a provider. +package modelcatalogresolver + +import ( + "fmt" + "slices" + "strings" + + "github.com/maximhq/bifrost/core/schemas" + "github.com/maximhq/bifrost/framework/modelcatalog" +) + +const PluginName = "model-catalog-resolver" + +// integrationTypeToDefaultProvider maps the integration-type ctx value (set by +// transports/bifrost-http/integrations/router.go on integration routes) to the +// integration's canonical provider. When the catalog returns multiple providers +// for an unprefixed model, the resolver prefers the integration's canonical +// provider if it's in the candidate list. +var integrationTypeToDefaultProvider = map[string]schemas.ModelProvider{ + "openai": schemas.OpenAI, + "anthropic": schemas.Anthropic, + "genai": schemas.Gemini, + "bedrock": schemas.Bedrock, + "cohere": schemas.Cohere, +} + +// Plugin resolves the default provider for unprefixed model strings using the model catalog. +type Plugin struct { + catalog *modelcatalog.ModelCatalog + logger schemas.Logger +} + +// Init returns a new resolver plugin. The catalog is required; if nil, the plugin returns +// an error rather than silently no-op'ing — a nil catalog at boot is a misconfiguration. +func Init(catalog *modelcatalog.ModelCatalog, logger schemas.Logger) (*Plugin, error) { + if catalog == nil { + return nil, fmt.Errorf("model-catalog-resolver: catalog is required") + } + return &Plugin{catalog: catalog, logger: logger}, nil +} + +// GetName implements schemas.BasePlugin. +func (p *Plugin) GetName() string { return PluginName } + +// Cleanup implements schemas.BasePlugin. +func (p *Plugin) Cleanup() error { return nil } + +// PreRequestHook fills in req.Provider from the model catalog when no provider was specified. +// Skips passthrough requests and requests that already have a provider set (e.g., from a model +// string like "openai/gpt-5", or from an earlier routing plugin — governance, LB). +// +// When the catalog returns multiple providers for an unprefixed model, the resolver prefers the +// integration's canonical provider (looked up from BifrostContextKeyIntegrationType set by the +// integration router) if it's in the candidate list. Otherwise it picks the first candidate. +// +// If the catalog returns zero providers, the resolver leaves req.Provider empty — the +// empty-provider validation in handleRequest/handleStreamRequest then returns a clear error. +func (p *Plugin) PreRequestHook(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) error { + if req.RequestType == schemas.PassthroughRequest || req.RequestType == schemas.PassthroughStreamRequest { + return nil + } + provider, model, existingFallbacks := req.GetRequestFields() + if provider != "" || model == "" { + return nil + } + + selected, candidates := ResolveProviderFromCatalog(ctx, p.catalog, model) + if selected == "" { + return nil + } + req.SetProvider(selected) + + candidateStrs := make([]string, len(candidates)) + for i, prov := range candidates { + candidateStrs[i] = string(prov) + } + ctx.AppendRoutingEngineLog(schemas.RoutingEngineModelCatalog, schemas.LogLevelInfo, fmt.Sprintf( + "No provider specified for model %s, found %d options in model catalog: [%s], selected: %s", + model, len(candidates), strings.Join(candidateStrs, ", "), selected, + )) + + // Populate fallbacks from the remaining catalog candidates so the request gets + // cross-provider resilience automatically — matches the governance and load + // balancing plugins, which both promote unselected candidates to fallbacks + // when the caller didn't configure any. Only fires when the caller passed + // none; an explicit fallback list (even an empty one set deliberately) is + // always respected. Model refinement is not needed here: GetProvidersForModel + // only returns providers that already serve this exact model string. + if len(existingFallbacks) == 0 && len(candidates) > 1 { + fallbacks := make([]schemas.Fallback, 0, len(candidates)-1) + for _, prov := range candidates { + if prov == selected { + continue + } + fallbacks = append(fallbacks, schemas.Fallback{Provider: prov, Model: model}) + } + if len(fallbacks) > 0 { + req.SetFallbacks(fallbacks) + fallbackStrs := make([]string, len(fallbacks)) + for i, fb := range fallbacks { + fallbackStrs[i] = string(fb.Provider) + } + ctx.AppendRoutingEngineLog(schemas.RoutingEngineModelCatalog, schemas.LogLevelInfo, fmt.Sprintf( + "Added %d catalog fallback provider(s) for model %s: [%s]", + len(fallbacks), model, strings.Join(fallbackStrs, ", "), + )) + } + } + + schemas.AppendToContextList(ctx, schemas.BifrostContextKeyRoutingEnginesUsed, schemas.RoutingEngineModelCatalog) + return nil +} + +// ResolveProviderFromCatalog performs the deterministic, integration-aware provider pick +// that PreRequestHook does, exposed for transport paths that can't run through +// PreRequestHook (realtime client_secrets, WebRTC). Returns the selected provider plus the +// candidate list (post-allowlist when an allowlist is in effect). Returns ("", nil) when +// the catalog has no match for the model, or when an allowlist excludes every candidate. +// +// The integration hint (BifrostContextKeyIntegrationType, when present and mapped) biases +// the pick toward the integration's canonical provider if it is in the candidate set; +// otherwise selection falls back to the alphabetically-first candidate for determinism. +// +// For requests routed through the openai integration whose user-agent identifies an Azure +// OpenAI SDK (BifrostContextKeyIsAzureUserAgent), schemas.Azure is preferred over +// schemas.OpenAI when Azure is in the candidate list — the openai-format converters no +// longer apply this default inline. +// +// When BifrostContextKeyRoutingAllowedProviders is set on ctx by an earlier plugin (e.g., +// governance VK config), the candidate list is intersected with the allowlist before +// selection — emitting routing-engine logs visible to callers when the allowlist prunes +// candidates. Side effect: routing-engine logs are written to ctx when allowlist filtering +// is applied (nil ctx skips logging). +func ResolveProviderFromCatalog(ctx *schemas.BifrostContext, catalog *modelcatalog.ModelCatalog, model string) (schemas.ModelProvider, []schemas.ModelProvider) { + if catalog == nil || model == "" { + return "", nil + } + providers := catalog.GetProvidersForModel(model) + if len(providers) == 0 { + return "", nil + } + + // GetProvidersForModel iterates a Go map; the returned order is not stable. + // Sort alphabetically so the fallback pick (providers[0]) is deterministic across + // restarts and across processes — critical when no IntegrationType hint is set. + slices.SortFunc(providers, func(a, b schemas.ModelProvider) int { + return strings.Compare(string(a), string(b)) + }) + + var integrationType string + var isAzureUser bool + var allowed []schemas.ModelProvider + allowlistSet := false + if ctx != nil { + integrationType, _ = ctx.Value(schemas.BifrostContextKeyIntegrationType).(string) + isAzureUser, _ = ctx.Value(schemas.BifrostContextKeyIsAzureUserAgent).(bool) + allowed, allowlistSet = ctx.Value(schemas.BifrostContextKeyRoutingAllowedProviders).([]schemas.ModelProvider) + } + + // Respect the routing-allowlist set by an earlier plugin (e.g., governance VK config): + // intersect catalog candidates with the allowlist so the VK's provider restrictions hold + // even when no earlier routing plugin set req.Provider. Emit observability logs for both + // the partial-prune and all-pruned cases — the two-level enforcement (cooperative here + + // hard core enforcement) is only useful if the cooperative pruning is visible in routing + // engine logs when it fires. + if allowlistSet { + preFilterCount := len(providers) + preFilterStrs := make([]string, preFilterCount) + for i, prov := range providers { + preFilterStrs[i] = string(prov) + } + allowedStrs := make([]string, len(allowed)) + for i, prov := range allowed { + allowedStrs[i] = string(prov) + } + filtered := make([]schemas.ModelProvider, 0, preFilterCount) + excluded := make([]schemas.ModelProvider, 0) + for _, prov := range providers { + if slices.Contains(allowed, prov) { + filtered = append(filtered, prov) + } else { + excluded = append(excluded, prov) + } + } + if len(excluded) > 0 && ctx != nil { + filteredStrs := make([]string, len(filtered)) + for i, prov := range filtered { + filteredStrs[i] = string(prov) + } + ctx.AppendRoutingEngineLog(schemas.RoutingEngineModelCatalog, schemas.LogLevelInfo, fmt.Sprintf( + "Catalog returned %d candidate provider(s) for model %s: [%s]; provider allowlist is [%s], so excluded %d; remaining providers are [%s]", + preFilterCount, model, strings.Join(preFilterStrs, ", "), + strings.Join(allowedStrs, ", "), + len(excluded), + strings.Join(filteredStrs, ", "), + )) + } + providers = filtered + if len(providers) == 0 { + if ctx != nil { + ctx.AppendRoutingEngineLog(schemas.RoutingEngineModelCatalog, schemas.LogLevelInfo, fmt.Sprintf( + "Catalog returned %d candidate provider(s) for model %s: [%s]; provider allowlist [%s] excluded all of them; leaving req.Provider empty", + preFilterCount, model, strings.Join(preFilterStrs, ", "), + strings.Join(allowedStrs, ", "), + )) + } + return "", nil + } + } + + selected := providers[0] + if integrationType != "" { + if integrationDefault, mapped := integrationTypeToDefaultProvider[integrationType]; mapped && integrationDefault != "" { + preferred := integrationDefault + if integrationType == "openai" && isAzureUser { + preferred = schemas.Azure + } + if slices.Contains(providers, preferred) { + selected = preferred + } + + // For Anthropic-type routes, raw request body passthrough is only valid for + // providers that speak the Anthropic Messages API natively. When the model + // catalog falls back to a provider that doesn't (e.g. Bedrock), clear the + // flag so the provider performs its own format conversion. + if integrationType == "anthropic" && + selected != schemas.Anthropic && + selected != schemas.Vertex && + selected != schemas.Azure { + ctx.SetValue(schemas.BifrostContextKeyUseRawRequestBody, false) + ctx.SetValue(schemas.BifrostContextKeySendBackRawResponse, false) + ctx.SetValue(schemas.BifrostContextKeyPassthroughOverridesPresent, false) + } + } + } + return selected, providers +} + +// PreLLMHook implements schemas.LLMPlugin (no-op). +func (p *Plugin) PreLLMHook(_ *schemas.BifrostContext, req *schemas.BifrostRequest) (*schemas.BifrostRequest, *schemas.LLMPluginShortCircuit, error) { + return req, nil, nil +} + +// PostLLMHook implements schemas.LLMPlugin (no-op). +func (p *Plugin) PostLLMHook(_ *schemas.BifrostContext, resp *schemas.BifrostResponse, bifrostErr *schemas.BifrostError) (*schemas.BifrostResponse, *schemas.BifrostError, error) { + return resp, bifrostErr, nil +} diff --git a/plugins/otel/converter.go b/plugins/otel/converter.go index d5a6fe281b..924bbe679e 100644 --- a/plugins/otel/converter.go +++ b/plugins/otel/converter.go @@ -3,7 +3,6 @@ package otel import ( "encoding/hex" "fmt" - "slices" "strings" "github.com/maximhq/bifrost/core/schemas" @@ -70,72 +69,15 @@ func hexToBytes(hexStr string, length int) []byte { return bytes } -// shouldExportSpan reports whether a span should be included in the export. -// Non-plugin spans are always exported. Plugin spans are checked against pluginSpanFilter. -func (p *OtelPlugin) shouldExportSpan(span *schemas.Span) bool { - if span.Kind != schemas.SpanKindPlugin || p.pluginSpanFilter == nil { - return true - } - // Span names follow the pattern "plugin..prehook" / "plugin..posthook". - parts := strings.SplitN(span.Name, ".", 3) - if len(parts) < 2 { - return true - } - pluginName := parts[1] - - inList := slices.Contains(p.pluginSpanFilter.Plugins, pluginName) - - if p.pluginSpanFilter.Mode == PluginSpanFilterModeInclude { - return inList - } - return !inList // exclude mode -} - -// buildReparentMap returns a map of filteredSpanID → effective ancestor spanID for all -// spans that will be skipped. When plugin spans are chained (each span's parent is the -// previous plugin's span), removing a span from the middle would leave its children with -// a dangling parent ID. The map lets us rewrite those parent IDs to the nearest exported -// ancestor, handling consecutive filtered spans in a chain. -func (p *OtelPlugin) buildReparentMap(spans []*schemas.Span) map[string]string { - if p.pluginSpanFilter == nil { - return nil - } - // First pass: record direct parent ID for every filtered span. - filtered := make(map[string]string) // spanID -> parentID - for _, span := range spans { - if !p.shouldExportSpan(span) { - filtered[span.SpanID] = span.ParentID - } - } - if len(filtered) == 0 { - return nil - } - // Second pass: resolve chains so each filtered span maps to its first exported ancestor. - // Cap the walk at len(filtered) to break out of any cycle caused by malformed span data. - maxHops := len(filtered) - for spanID := range filtered { - parentID := filtered[spanID] - for range maxHops { - grandParentID, isFiltered := filtered[parentID] - if !isFiltered { - break - } - parentID = grandParentID - } - filtered[spanID] = parentID - } - return filtered -} - // convertTraceToResourceSpan converts a Bifrost trace to OTEL ResourceSpan for the given // profile service name. Span filtering and instance attributes are shared across profiles; // only the resource service name differs per profile. func (p *OtelPlugin) convertTraceToResourceSpan(serviceName string, trace *schemas.Trace, requestHeaders []string, disableContentLogging bool) *ResourceSpan { - reparent := p.buildReparentMap(trace.Spans) + reparent := p.pluginSpanFilter.BuildReparentMap(trace.Spans) filteredHeaders := schemas.FilterHeaders(trace.RequestHeaders, requestHeaders) otelSpans := make([]*Span, 0, len(trace.Spans)) for _, span := range trace.Spans { - if !p.shouldExportSpan(span) { + if !p.pluginSpanFilter.ShouldExportSpan(span) { continue } otelSpan := convertSpanToOTELSpan(trace.TraceID, span, disableContentLogging) diff --git a/plugins/otel/converter_test.go b/plugins/otel/converter_test.go index f9e8833b93..afcabf5b3d 100644 --- a/plugins/otel/converter_test.go +++ b/plugins/otel/converter_test.go @@ -1,6 +1,7 @@ package otel import ( + "bytes" "testing" "time" @@ -18,161 +19,50 @@ func makeSpan(id, parentID, name string, kind schemas.SpanKind) *schemas.Span { } } -func TestShouldExportSpan(t *testing.T) { - tests := []struct { - name string - filter *PluginSpanFilter - span *schemas.Span - want bool - }{ - { - name: "nil filter exports everything", - filter: nil, - span: makeSpan("1", "", "plugin.logging.prehook", schemas.SpanKindPlugin), - want: true, - }, - { - name: "non-plugin span always exported regardless of filter", - filter: &PluginSpanFilter{Mode: PluginSpanFilterModeExclude, Plugins: []string{"logging"}}, - span: makeSpan("1", "", "llm.call", schemas.SpanKindLLMCall), - want: true, - }, - { - name: "exclude mode: plugin in list is suppressed", - filter: &PluginSpanFilter{Mode: PluginSpanFilterModeExclude, Plugins: []string{"logging", "compat"}}, - span: makeSpan("1", "", "plugin.logging.prehook", schemas.SpanKindPlugin), - want: false, - }, - { - name: "exclude mode: plugin not in list is exported", - filter: &PluginSpanFilter{Mode: PluginSpanFilterModeExclude, Plugins: []string{"logging"}}, - span: makeSpan("1", "", "plugin.governance.posthook", schemas.SpanKindPlugin), - want: true, - }, - { - name: "exclude mode: posthook variant suppressed the same as prehook", - filter: &PluginSpanFilter{Mode: PluginSpanFilterModeExclude, Plugins: []string{"logging"}}, - span: makeSpan("1", "", "plugin.logging.posthook", schemas.SpanKindPlugin), - want: false, - }, - { - name: "include mode: plugin in list is exported", - filter: &PluginSpanFilter{Mode: PluginSpanFilterModeInclude, Plugins: []string{"guardrails"}}, - span: makeSpan("1", "", "plugin.guardrails.prehook", schemas.SpanKindPlugin), - want: true, - }, - { - name: "include mode: plugin not in list is suppressed", - filter: &PluginSpanFilter{Mode: PluginSpanFilterModeInclude, Plugins: []string{"guardrails"}}, - span: makeSpan("1", "", "plugin.logging.prehook", schemas.SpanKindPlugin), - want: false, - }, - { - name: "exclude mode: empty list suppresses nothing", - filter: &PluginSpanFilter{Mode: PluginSpanFilterModeExclude, Plugins: []string{}}, - span: makeSpan("1", "", "plugin.logging.prehook", schemas.SpanKindPlugin), - want: true, - }, - { - name: "include mode: empty list suppresses everything", - filter: &PluginSpanFilter{Mode: PluginSpanFilterModeInclude, Plugins: []string{}}, - span: makeSpan("1", "", "plugin.logging.prehook", schemas.SpanKindPlugin), - want: false, - }, - { - name: "span name without dots passes through", - filter: &PluginSpanFilter{Mode: PluginSpanFilterModeExclude, Plugins: []string{"logging"}}, - span: makeSpan("1", "", "nodots", schemas.SpanKindPlugin), - want: true, - }, - } +// TestConvertTraceToResourceSpan_PluginSpanFilter exercises the OTEL converter's end-to-end +// filtering behavior (the parts unique to this package; the filter/reparent logic itself is +// covered by core/schemas/span_filter_test.go). It asserts that filtered plugin spans are +// dropped from the exported ResourceSpan and that an exported child whose direct parent was +// filtered is re-parented to the nearest exported ancestor. +func TestConvertTraceToResourceSpan_PluginSpanFilter(t *testing.T) { + p := &OtelPlugin{pluginSpanFilter: &PluginSpanFilter{ + Mode: PluginSpanFilterModeExclude, + Plugins: []string{"logging"}, + }} - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - p := &OtelPlugin{pluginSpanFilter: tt.filter} - if got := p.shouldExportSpan(tt.span); got != tt.want { - t.Errorf("shouldExportSpan() = %v, want %v", got, tt.want) - } - }) + // Span tree: root (internal) -> logging.prehook (filtered) -> governance.prehook (kept). + root := makeSpan("aaaa", "", "request", schemas.SpanKindInternal) + trace := &schemas.Trace{ + TraceID: "00000000000000000000000000000001", + RootSpan: root, + Spans: []*schemas.Span{ + root, + makeSpan("bbbb", "aaaa", "plugin.logging.prehook", schemas.SpanKindPlugin), + makeSpan("cccc", "bbbb", "plugin.governance.prehook", schemas.SpanKindPlugin), + }, } -} - -func TestBuildReparentMap(t *testing.T) { - excludeLogging := &PluginSpanFilter{Mode: PluginSpanFilterModeExclude, Plugins: []string{"logging"}} - - t.Run("nil filter returns nil map", func(t *testing.T) { - p := &OtelPlugin{pluginSpanFilter: nil} - spans := []*schemas.Span{makeSpan("a", "root", "plugin.logging.prehook", schemas.SpanKindPlugin)} - if m := p.buildReparentMap(spans); m != nil { - t.Errorf("expected nil, got %v", m) - } - }) - - t.Run("no filtered spans returns nil map", func(t *testing.T) { - p := &OtelPlugin{pluginSpanFilter: excludeLogging} - spans := []*schemas.Span{ - makeSpan("a", "root", "plugin.governance.prehook", schemas.SpanKindPlugin), - } - if m := p.buildReparentMap(spans); m != nil { - t.Errorf("expected nil, got %v", m) - } - }) - - t.Run("single filtered span maps to its direct parent", func(t *testing.T) { - p := &OtelPlugin{pluginSpanFilter: excludeLogging} - // root -> logging (filtered) -> governance - spans := []*schemas.Span{ - makeSpan("root", "", "request", schemas.SpanKindInternal), - makeSpan("log-pre", "root", "plugin.logging.prehook", schemas.SpanKindPlugin), - makeSpan("gov-pre", "log-pre", "plugin.governance.prehook", schemas.SpanKindPlugin), - } - m := p.buildReparentMap(spans) - if m == nil { - t.Fatal("expected non-nil map") - } - if got := m["log-pre"]; got != "root" { - t.Errorf("filtered span should map to parent 'root', got %q", got) - } - }) - t.Run("chain of filtered spans resolves to nearest exported ancestor", func(t *testing.T) { - // root -> telemetry (filtered) -> logging (filtered) -> governance - p := &OtelPlugin{pluginSpanFilter: &PluginSpanFilter{ - Mode: PluginSpanFilterModeExclude, - Plugins: []string{"telemetry", "logging"}, - }} - spans := []*schemas.Span{ - makeSpan("root", "", "request", schemas.SpanKindInternal), - makeSpan("tel-pre", "root", "plugin.telemetry.prehook", schemas.SpanKindPlugin), - makeSpan("log-pre", "tel-pre", "plugin.logging.prehook", schemas.SpanKindPlugin), - makeSpan("gov-pre", "log-pre", "plugin.governance.prehook", schemas.SpanKindPlugin), - } - m := p.buildReparentMap(spans) - if m == nil { - t.Fatal("expected non-nil map") - } - // Both filtered spans must resolve to "root" so governance.prehook re-parents there. - if got := m["tel-pre"]; got != "root" { - t.Errorf("tel-pre should resolve to 'root', got %q", got) - } - if got := m["log-pre"]; got != "root" { - t.Errorf("log-pre should skip the chain and resolve to 'root', got %q", got) - } - }) + rs := p.convertTraceToResourceSpan("svc", trace, nil, false) + spans := rs.ScopeSpans[0].Spans - t.Run("filtered span with no parent resolves to empty string", func(t *testing.T) { - p := &OtelPlugin{pluginSpanFilter: excludeLogging} - spans := []*schemas.Span{ - // logging span has no parent (root of trace) - makeSpan("log-pre", "", "plugin.logging.prehook", schemas.SpanKindPlugin), - makeSpan("gov-pre", "log-pre", "plugin.governance.prehook", schemas.SpanKindPlugin), - } - m := p.buildReparentMap(spans) - if m == nil { - t.Fatal("expected non-nil map") - } - if got := m["log-pre"]; got != "" { - t.Errorf("root-level filtered span should resolve to empty string, got %q", got) - } - }) + // The filtered logging span is dropped; root + governance remain. + if len(spans) != 2 { + t.Fatalf("expected 2 exported spans (logging dropped), got %d", len(spans)) + } + byID := make(map[string]*Span, len(spans)) + for _, s := range spans { + byID[string(s.SpanId)] = s + } + if _, ok := byID[string(hexToBytes("bbbb", 8))]; ok { + t.Error("filtered logging span should not be exported") + } + gov, ok := byID[string(hexToBytes("cccc", 8))] + if !ok { + t.Fatal("governance span should be exported") + } + // governance's direct parent (logging) was filtered, so its parent must be rewritten to + // the nearest exported ancestor (root), not left dangling at the dropped logging span. + if !bytes.Equal(gov.ParentSpanId, hexToBytes("aaaa", 8)) { + t.Errorf("governance ParentSpanId = %x, want %x (reparented to root)", gov.ParentSpanId, hexToBytes("aaaa", 8)) + } } diff --git a/plugins/otel/go.mod b/plugins/otel/go.mod index 0ad39c256a..b020adab2d 100644 --- a/plugins/otel/go.mod +++ b/plugins/otel/go.mod @@ -22,7 +22,7 @@ require ( cloud.google.com/go/auth v0.20.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect - cloud.google.com/go/iam v1.5.3 // indirect + cloud.google.com/go/iam v1.7.0 // indirect cloud.google.com/go/monitoring v1.24.3 // indirect cloud.google.com/go/storage v1.61.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect @@ -34,13 +34,13 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect github.com/andybalholm/brotli v1.2.1 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect - github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect + github.com/aws/aws-sdk-go-v2 v1.41.12 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect github.com/aws/aws-sdk-go-v2/config v1.32.11 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect @@ -52,7 +52,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect - github.com/aws/smithy-go v1.25.1 // indirect + github.com/aws/smithy-go v1.27.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.2 // indirect github.com/bytedance/gopkg v0.1.3 // indirect diff --git a/plugins/otel/go.sum b/plugins/otel/go.sum index 89e62c5390..8eb748b4fe 100644 --- a/plugins/otel/go.sum +++ b/plugins/otel/go.sum @@ -8,8 +8,8 @@ cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIi cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= -cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= +cloud.google.com/go/iam v1.7.0 h1:JD3zh0C6LHl16aCn5Akff0+GELdp1+4hmh6ndoFLl8U= +cloud.google.com/go/iam v1.7.0/go.mod h1:tetWZW1PD/m6vcuY2Zj/aU0eCHNPuxedbnbRTyKXvdY= cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHAjA= cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZCn6hSBtKak= cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8= @@ -45,8 +45,8 @@ github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eT github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= -github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= -github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2 v1.41.12 h1:DIKX2c31ekm9RA2D9FBj1EWXx++9AdAqRw+e78Tq2Ck= +github.com/aws/aws-sdk-go-v2 v1.41.12/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= github.com/aws/aws-sdk-go-v2/config v1.32.11 h1:ftxI5sgz8jZkckuUHXfC/wMUc8u3fG1vQS0plr2F2Zs= @@ -55,10 +55,10 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8TH github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 h1:Xf2j7NdVcUKomlZ4iihOP4AZ3Fzlr8h4yKpXeP+OFPg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28/go.mod h1:O8cDo1dW63jU7ki//kRe1z+tLGcpnD1jrouitsQddDw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 h1:KqIfN9kpkKkcBqBbNpNGTIrXO6ExTUvFKvXkC+YAzVo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28/go.mod h1:uxtQiKvLtNS4iXVsH2McVD/ls8FKN/uUhe1hGxPjrw0= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 h1:clHU5fm//kWS1C2HgtgWxfQbFbx4b6rx+5jzhgX9HrI= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= @@ -81,8 +81,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6f github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= -github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= -github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8= +github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= diff --git a/plugins/otel/main.go b/plugins/otel/main.go index 13d4722e48..b56c42d357 100644 --- a/plugins/otel/main.go +++ b/plugins/otel/main.go @@ -41,21 +41,19 @@ const ( ProtocolGRPC Protocol = "grpc" ) -// PluginSpanFilterMode controls whether the plugins list is an allowlist or denylist. -type PluginSpanFilterMode string +// PluginSpanFilter, its mode type, and the include/exclude constants are shared across +// all observability connectors and live in core/schemas. They are re-exported here as +// aliases so existing OTEL config parsing, tests, and the UI keep their import paths. +type ( + PluginSpanFilterMode = schemas.PluginSpanFilterMode + PluginSpanFilter = schemas.PluginSpanFilter +) const ( - PluginSpanFilterModeInclude PluginSpanFilterMode = "include" - PluginSpanFilterModeExclude PluginSpanFilterMode = "exclude" + PluginSpanFilterModeInclude = schemas.PluginSpanFilterModeInclude + PluginSpanFilterModeExclude = schemas.PluginSpanFilterModeExclude ) -// PluginSpanFilter configures which plugin spans are exported to the OTEL collector. -// Mode "include" exports only the listed plugins; mode "exclude" exports everything except them. -type PluginSpanFilter struct { - Mode PluginSpanFilterMode `json:"mode"` - Plugins []string `json:"plugins"` -} - // Profile is a single OTEL export target: a collector endpoint and an optional // metrics-push destination. A Config holds one or more profiles; each profile gets // its own trace client and (when enabled) metrics exporter at runtime. @@ -349,13 +347,8 @@ func Init(ctx context.Context, config *Config, _logger schemas.Logger, pricingMa if len(config.Profiles) == 0 { return nil, fmt.Errorf("at least one otel profile is required") } - if config.PluginSpanFilter != nil { - switch config.PluginSpanFilter.Mode { - case PluginSpanFilterModeInclude, PluginSpanFilterModeExclude: - default: - return nil, fmt.Errorf("plugin_span_filter.mode %q is invalid: must be %q or %q", - config.PluginSpanFilter.Mode, PluginSpanFilterModeInclude, PluginSpanFilterModeExclude) - } + if err := config.PluginSpanFilter.Validate(); err != nil { + return nil, err } // Loading attributes from environment attributesFromEnvironment := make([]*commonpb.KeyValue, 0) @@ -606,6 +599,30 @@ func (p *OtelPlugin) anyMetricsEnabled() bool { return false } +// RecordHTTPMetrics records HTTP-layer metrics (request count, duration, request/response +// sizes) against every profile's metrics exporter. The HTTP transport's middleware calls +// this once per completed request; it is a no-op when no profile has metrics enabled. +// Non-positive sizes are skipped (fasthttp reports -1 when Content-Length is unknown). +func (p *OtelPlugin) RecordHTTPMetrics(ctx context.Context, path, method, status string, durationSeconds, requestSizeBytes, responseSizeBytes float64) { + if !p.anyMetricsEnabled() { + return + } + attrs := BuildHTTPAttributes(path, method, status) + for _, t := range p.targets { + if t.metricsExporter == nil { + continue + } + t.metricsExporter.RecordHTTPRequest(ctx, attrs...) + t.metricsExporter.RecordHTTPRequestDuration(ctx, durationSeconds, attrs...) + if requestSizeBytes > 0 { + t.metricsExporter.RecordHTTPRequestSize(ctx, requestSizeBytes, attrs...) + } + if responseSizeBytes > 0 { + t.metricsExporter.RecordHTTPResponseSize(ctx, responseSizeBytes, attrs...) + } + } +} + // Inject receives a completed trace and sends it to the OTEL collector. // Implements schemas.ObservabilityPlugin interface. // This method is called asynchronously by TracingMiddleware after the response diff --git a/plugins/otel/metrics.go b/plugins/otel/metrics.go index ccde3635c0..2a6123c7d8 100644 --- a/plugins/otel/metrics.go +++ b/plugins/otel/metrics.go @@ -143,6 +143,14 @@ var ( interTokenLatencyBuckets = []float64{ .001, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, } + + // httpBodySizeBuckets: HTTP request/response body sizes, 100B to 1GB + // (matches prometheus.ExponentialBuckets(100, 10, 8) on the Prometheus side). + // The SDK default boundaries top out at 10,000, which would collapse any + // payload over 10KB into +Inf. + httpBodySizeBuckets = []float64{ + 100, 1_000, 10_000, 100_000, 1_000_000, 10_000_000, 100_000_000, 1_000_000_000, + } ) // syncFloat64Histogram wraps metric.Float64Histogram with thread-safe lazy initialization @@ -416,17 +424,19 @@ func (m *MetricsExporter) initMetrics() { } m.httpRequestSizeBytes = &syncFloat64Histogram{ - name: "http_request_size_bytes", - desc: "Size of HTTP requests", - unit: "By", - meter: m.meter, + name: "http_request_size_bytes", + desc: "Size of HTTP requests", + unit: "By", + meter: m.meter, + boundaries: httpBodySizeBuckets, } m.httpResponseSizeBytes = &syncFloat64Histogram{ - name: "http_response_size_bytes", - desc: "Size of HTTP responses", - unit: "By", - meter: m.meter, + name: "http_response_size_bytes", + desc: "Size of HTTP responses", + unit: "By", + meter: m.meter, + boundaries: httpBodySizeBuckets, } } diff --git a/plugins/prompts/go.mod b/plugins/prompts/go.mod index c92c236285..0c8ea06291 100644 --- a/plugins/prompts/go.mod +++ b/plugins/prompts/go.mod @@ -16,13 +16,13 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect github.com/andybalholm/brotli v1.2.1 // indirect - github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect + github.com/aws/aws-sdk-go-v2 v1.41.12 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect github.com/aws/aws-sdk-go-v2/config v1.32.11 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect @@ -34,7 +34,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect - github.com/aws/smithy-go v1.25.1 // indirect + github.com/aws/smithy-go v1.27.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.2 // indirect github.com/bytedance/gopkg v0.1.3 // indirect diff --git a/plugins/prompts/go.sum b/plugins/prompts/go.sum index 82de6c638e..3236f7b781 100644 --- a/plugins/prompts/go.sum +++ b/plugins/prompts/go.sum @@ -16,8 +16,8 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgv github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro= github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= -github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= -github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2 v1.41.12 h1:DIKX2c31ekm9RA2D9FBj1EWXx++9AdAqRw+e78Tq2Ck= +github.com/aws/aws-sdk-go-v2 v1.41.12/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= github.com/aws/aws-sdk-go-v2/config v1.32.11 h1:ftxI5sgz8jZkckuUHXfC/wMUc8u3fG1vQS0plr2F2Zs= @@ -26,10 +26,10 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8TH github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 h1:Xf2j7NdVcUKomlZ4iihOP4AZ3Fzlr8h4yKpXeP+OFPg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28/go.mod h1:O8cDo1dW63jU7ki//kRe1z+tLGcpnD1jrouitsQddDw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 h1:KqIfN9kpkKkcBqBbNpNGTIrXO6ExTUvFKvXkC+YAzVo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28/go.mod h1:uxtQiKvLtNS4iXVsH2McVD/ls8FKN/uUhe1hGxPjrw0= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 h1:clHU5fm//kWS1C2HgtgWxfQbFbx4b6rx+5jzhgX9HrI= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= @@ -52,8 +52,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6f github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= -github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= -github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8= +github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= diff --git a/plugins/prompts/main.go b/plugins/prompts/main.go index e710980dcb..d74a5b9fea 100644 --- a/plugins/prompts/main.go +++ b/plugins/prompts/main.go @@ -203,6 +203,11 @@ func (p *Plugin) HTTPTransportStreamChunkHook(ctx *schemas.BifrostContext, req * return chunk, nil } +// PreRequestHook implements schemas.LLMPlugin (no-op — required for plugin indexing). +func (p *Plugin) PreRequestHook(_ *schemas.BifrostContext, _ *schemas.BifrostRequest) error { + return nil +} + // PreLLMHook resolves the prompt via PromptResolver, loads the version from the in-memory // cache, sets governance/observability context (selected prompt name and version), merges // version ModelParams with the request (request overrides), converts stored messages to diff --git a/plugins/semanticcache/go.mod b/plugins/semanticcache/go.mod index 85b2de88a4..b2a3862b38 100644 --- a/plugins/semanticcache/go.mod +++ b/plugins/semanticcache/go.mod @@ -19,13 +19,13 @@ require ( github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect github.com/andybalholm/brotli v1.2.1 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect - github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect + github.com/aws/aws-sdk-go-v2 v1.41.12 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect github.com/aws/aws-sdk-go-v2/config v1.32.11 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect @@ -37,7 +37,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect - github.com/aws/smithy-go v1.25.1 // indirect + github.com/aws/smithy-go v1.27.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.2 // indirect github.com/bytedance/gopkg v0.1.3 // indirect diff --git a/plugins/semanticcache/go.sum b/plugins/semanticcache/go.sum index 8e17189415..99b6fafd53 100644 --- a/plugins/semanticcache/go.sum +++ b/plugins/semanticcache/go.sum @@ -19,8 +19,8 @@ github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eT github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= -github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= -github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2 v1.41.12 h1:DIKX2c31ekm9RA2D9FBj1EWXx++9AdAqRw+e78Tq2Ck= +github.com/aws/aws-sdk-go-v2 v1.41.12/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= github.com/aws/aws-sdk-go-v2/config v1.32.11 h1:ftxI5sgz8jZkckuUHXfC/wMUc8u3fG1vQS0plr2F2Zs= @@ -29,10 +29,10 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8TH github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 h1:Xf2j7NdVcUKomlZ4iihOP4AZ3Fzlr8h4yKpXeP+OFPg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28/go.mod h1:O8cDo1dW63jU7ki//kRe1z+tLGcpnD1jrouitsQddDw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 h1:KqIfN9kpkKkcBqBbNpNGTIrXO6ExTUvFKvXkC+YAzVo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28/go.mod h1:uxtQiKvLtNS4iXVsH2McVD/ls8FKN/uUhe1hGxPjrw0= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 h1:clHU5fm//kWS1C2HgtgWxfQbFbx4b6rx+5jzhgX9HrI= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= @@ -55,8 +55,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6f github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= -github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= -github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8= +github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= diff --git a/plugins/semanticcache/main.go b/plugins/semanticcache/main.go index 7dbad109dc..737ef7c96e 100644 --- a/plugins/semanticcache/main.go +++ b/plugins/semanticcache/main.go @@ -330,6 +330,11 @@ func (plugin *Plugin) HTTPTransportStreamChunkHook(ctx *schemas.BifrostContext, return chunk, nil } +// PreRequestHook implements schemas.LLMPlugin (no-op — required for plugin indexing). +func (plugin *Plugin) PreRequestHook(_ *schemas.BifrostContext, _ *schemas.BifrostRequest) error { + return nil +} + // PreLLMHook performs the cache lookup before the request reaches the // provider. It runs the direct hash path first (cheapest), falls back to // semantic similarity search when configured, and short-circuits the diff --git a/plugins/semanticcache/plugin_no_mutation_test.go b/plugins/semanticcache/plugin_no_mutation_test.go index d0a65b681f..a00b2a7bb9 100644 --- a/plugins/semanticcache/plugin_no_mutation_test.go +++ b/plugins/semanticcache/plugin_no_mutation_test.go @@ -30,6 +30,10 @@ type requestCapturer struct { func (p *requestCapturer) GetName() string { return "test-request-capturer" } func (p *requestCapturer) Cleanup() error { return nil } +func (p *requestCapturer) PreRequestHook(_ *schemas.BifrostContext, _ *schemas.BifrostRequest) error { + return nil +} + func (p *requestCapturer) PreLLMHook(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) (*schemas.BifrostRequest, *schemas.LLMPluginShortCircuit, error) { p.mu.Lock() // Snapshot the request via JSON round-trip so any later mutation by the diff --git a/plugins/telemetry/go.mod b/plugins/telemetry/go.mod index 0d62e9e588..e15097c23d 100644 --- a/plugins/telemetry/go.mod +++ b/plugins/telemetry/go.mod @@ -16,7 +16,7 @@ require ( cloud.google.com/go/auth v0.20.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect - cloud.google.com/go/iam v1.5.3 // indirect + cloud.google.com/go/iam v1.7.0 // indirect cloud.google.com/go/monitoring v1.24.3 // indirect cloud.google.com/go/storage v1.61.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect @@ -28,13 +28,13 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect github.com/andybalholm/brotli v1.2.1 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect - github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect + github.com/aws/aws-sdk-go-v2 v1.41.12 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect github.com/aws/aws-sdk-go-v2/config v1.32.11 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect @@ -46,7 +46,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect - github.com/aws/smithy-go v1.25.1 // indirect + github.com/aws/smithy-go v1.27.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/buger/jsonparser v1.1.2 // indirect diff --git a/plugins/telemetry/go.sum b/plugins/telemetry/go.sum index c3c4dafa24..be9332bf46 100644 --- a/plugins/telemetry/go.sum +++ b/plugins/telemetry/go.sum @@ -8,8 +8,8 @@ cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIi cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= -cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= +cloud.google.com/go/iam v1.7.0 h1:JD3zh0C6LHl16aCn5Akff0+GELdp1+4hmh6ndoFLl8U= +cloud.google.com/go/iam v1.7.0/go.mod h1:tetWZW1PD/m6vcuY2Zj/aU0eCHNPuxedbnbRTyKXvdY= cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHAjA= cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZCn6hSBtKak= cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8= @@ -45,8 +45,8 @@ github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eT github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= -github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= -github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2 v1.41.12 h1:DIKX2c31ekm9RA2D9FBj1EWXx++9AdAqRw+e78Tq2Ck= +github.com/aws/aws-sdk-go-v2 v1.41.12/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= github.com/aws/aws-sdk-go-v2/config v1.32.11 h1:ftxI5sgz8jZkckuUHXfC/wMUc8u3fG1vQS0plr2F2Zs= @@ -55,10 +55,10 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8TH github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 h1:Xf2j7NdVcUKomlZ4iihOP4AZ3Fzlr8h4yKpXeP+OFPg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28/go.mod h1:O8cDo1dW63jU7ki//kRe1z+tLGcpnD1jrouitsQddDw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 h1:KqIfN9kpkKkcBqBbNpNGTIrXO6ExTUvFKvXkC+YAzVo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28/go.mod h1:uxtQiKvLtNS4iXVsH2McVD/ls8FKN/uUhe1hGxPjrw0= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 h1:clHU5fm//kWS1C2HgtgWxfQbFbx4b6rx+5jzhgX9HrI= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= @@ -81,8 +81,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6f github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= -github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= -github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8= +github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= diff --git a/plugins/telemetry/main.go b/plugins/telemetry/main.go index f28a6b0f2f..f823f1bbc8 100644 --- a/plugins/telemetry/main.go +++ b/plugins/telemetry/main.go @@ -613,6 +613,11 @@ func (p *PrometheusPlugin) HTTPTransportStreamChunkHook(ctx *schemas.BifrostCont return chunk, nil } +// PreRequestHook implements schemas.LLMPlugin (no-op — required for plugin indexing). +func (p *PrometheusPlugin) PreRequestHook(_ *schemas.BifrostContext, _ *schemas.BifrostRequest) error { + return nil +} + // PreLLMHook records the start time of the request in the context. // This time is used later in PostLLMHook to calculate request duration. func (p *PrometheusPlugin) PreLLMHook(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) (*schemas.BifrostRequest, *schemas.LLMPluginShortCircuit, error) { diff --git a/tests/cmd/e2eseed/go.mod b/tests/cmd/e2eseed/go.mod index aec9024217..4a4f0a7339 100644 --- a/tests/cmd/e2eseed/go.mod +++ b/tests/cmd/e2eseed/go.mod @@ -16,7 +16,7 @@ require ( cloud.google.com/go/auth v0.20.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect - cloud.google.com/go/iam v1.5.3 // indirect + cloud.google.com/go/iam v1.7.0 // indirect cloud.google.com/go/monitoring v1.24.3 // indirect cloud.google.com/go/storage v1.61.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect @@ -27,13 +27,13 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect github.com/andybalholm/brotli v1.2.1 // indirect - github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect + github.com/aws/aws-sdk-go-v2 v1.41.12 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect github.com/aws/aws-sdk-go-v2/config v1.32.11 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect @@ -45,7 +45,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect - github.com/aws/smithy-go v1.25.1 // indirect + github.com/aws/smithy-go v1.27.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.2 // indirect github.com/bytedance/gopkg v0.1.3 // indirect diff --git a/tests/cmd/e2eseed/go.sum b/tests/cmd/e2eseed/go.sum index a691b58ab0..c9daa84c19 100644 --- a/tests/cmd/e2eseed/go.sum +++ b/tests/cmd/e2eseed/go.sum @@ -8,8 +8,8 @@ cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIi cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= -cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= +cloud.google.com/go/iam v1.7.0 h1:JD3zh0C6LHl16aCn5Akff0+GELdp1+4hmh6ndoFLl8U= +cloud.google.com/go/iam v1.7.0/go.mod h1:tetWZW1PD/m6vcuY2Zj/aU0eCHNPuxedbnbRTyKXvdY= cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHAjA= cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZCn6hSBtKak= cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8= @@ -42,8 +42,8 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapp github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro= github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= -github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= -github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2 v1.41.12 h1:DIKX2c31ekm9RA2D9FBj1EWXx++9AdAqRw+e78Tq2Ck= +github.com/aws/aws-sdk-go-v2 v1.41.12/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= github.com/aws/aws-sdk-go-v2/config v1.32.11 h1:ftxI5sgz8jZkckuUHXfC/wMUc8u3fG1vQS0plr2F2Zs= @@ -52,10 +52,10 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8TH github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 h1:Xf2j7NdVcUKomlZ4iihOP4AZ3Fzlr8h4yKpXeP+OFPg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28/go.mod h1:O8cDo1dW63jU7ki//kRe1z+tLGcpnD1jrouitsQddDw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 h1:KqIfN9kpkKkcBqBbNpNGTIrXO6ExTUvFKvXkC+YAzVo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28/go.mod h1:uxtQiKvLtNS4iXVsH2McVD/ls8FKN/uUhe1hGxPjrw0= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 h1:clHU5fm//kWS1C2HgtgWxfQbFbx4b6rx+5jzhgX9HrI= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= @@ -78,8 +78,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6f github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= -github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= -github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8= +github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= diff --git a/tests/cmd/seed/go.mod b/tests/cmd/seed/go.mod index c35b19326b..c8da28d6f3 100644 --- a/tests/cmd/seed/go.mod +++ b/tests/cmd/seed/go.mod @@ -21,7 +21,7 @@ require ( cloud.google.com/go/auth v0.20.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect - cloud.google.com/go/iam v1.5.3 // indirect + cloud.google.com/go/iam v1.7.0 // indirect cloud.google.com/go/monitoring v1.24.3 // indirect cloud.google.com/go/storage v1.61.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect @@ -32,13 +32,13 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect github.com/andybalholm/brotli v1.2.1 // indirect - github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect + github.com/aws/aws-sdk-go-v2 v1.41.12 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect github.com/aws/aws-sdk-go-v2/config v1.32.11 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect @@ -50,7 +50,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect - github.com/aws/smithy-go v1.25.1 // indirect + github.com/aws/smithy-go v1.27.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.2 // indirect github.com/bytedance/gopkg v0.1.3 // indirect diff --git a/tests/cmd/seed/go.sum b/tests/cmd/seed/go.sum index a691b58ab0..c9daa84c19 100644 --- a/tests/cmd/seed/go.sum +++ b/tests/cmd/seed/go.sum @@ -8,8 +8,8 @@ cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIi cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= -cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= +cloud.google.com/go/iam v1.7.0 h1:JD3zh0C6LHl16aCn5Akff0+GELdp1+4hmh6ndoFLl8U= +cloud.google.com/go/iam v1.7.0/go.mod h1:tetWZW1PD/m6vcuY2Zj/aU0eCHNPuxedbnbRTyKXvdY= cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHAjA= cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZCn6hSBtKak= cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8= @@ -42,8 +42,8 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapp github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro= github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= -github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= -github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2 v1.41.12 h1:DIKX2c31ekm9RA2D9FBj1EWXx++9AdAqRw+e78Tq2Ck= +github.com/aws/aws-sdk-go-v2 v1.41.12/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= github.com/aws/aws-sdk-go-v2/config v1.32.11 h1:ftxI5sgz8jZkckuUHXfC/wMUc8u3fG1vQS0plr2F2Zs= @@ -52,10 +52,10 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8TH github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 h1:Xf2j7NdVcUKomlZ4iihOP4AZ3Fzlr8h4yKpXeP+OFPg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28/go.mod h1:O8cDo1dW63jU7ki//kRe1z+tLGcpnD1jrouitsQddDw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 h1:KqIfN9kpkKkcBqBbNpNGTIrXO6ExTUvFKvXkC+YAzVo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28/go.mod h1:uxtQiKvLtNS4iXVsH2McVD/ls8FKN/uUhe1hGxPjrw0= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 h1:clHU5fm//kWS1C2HgtgWxfQbFbx4b6rx+5jzhgX9HrI= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= @@ -78,8 +78,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6f github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= -github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= -github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8= +github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= diff --git a/tests/cmd/seedvks/go.mod b/tests/cmd/seedvks/go.mod index 730c25cdd2..c1f33dd95a 100644 --- a/tests/cmd/seedvks/go.mod +++ b/tests/cmd/seedvks/go.mod @@ -23,13 +23,13 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect github.com/andybalholm/brotli v1.2.1 // indirect - github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect + github.com/aws/aws-sdk-go-v2 v1.41.12 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect github.com/aws/aws-sdk-go-v2/config v1.32.11 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect @@ -41,7 +41,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect - github.com/aws/smithy-go v1.25.1 // indirect + github.com/aws/smithy-go v1.27.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.2 // indirect github.com/bytedance/gopkg v0.1.3 // indirect diff --git a/tests/cmd/seedvks/go.sum b/tests/cmd/seedvks/go.sum index 2f6d11e525..ded12856b1 100644 --- a/tests/cmd/seedvks/go.sum +++ b/tests/cmd/seedvks/go.sum @@ -16,8 +16,8 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgv github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro= github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= -github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= -github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2 v1.41.12 h1:DIKX2c31ekm9RA2D9FBj1EWXx++9AdAqRw+e78Tq2Ck= +github.com/aws/aws-sdk-go-v2 v1.41.12/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= github.com/aws/aws-sdk-go-v2/config v1.32.11 h1:ftxI5sgz8jZkckuUHXfC/wMUc8u3fG1vQS0plr2F2Zs= @@ -26,10 +26,10 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8TH github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 h1:Xf2j7NdVcUKomlZ4iihOP4AZ3Fzlr8h4yKpXeP+OFPg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28/go.mod h1:O8cDo1dW63jU7ki//kRe1z+tLGcpnD1jrouitsQddDw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 h1:KqIfN9kpkKkcBqBbNpNGTIrXO6ExTUvFKvXkC+YAzVo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28/go.mod h1:uxtQiKvLtNS4iXVsH2McVD/ls8FKN/uUhe1hGxPjrw0= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 h1:clHU5fm//kWS1C2HgtgWxfQbFbx4b6rx+5jzhgX9HrI= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= @@ -52,8 +52,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6f github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= -github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= -github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8= +github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= diff --git a/tests/e2e/api/README.md b/tests/e2e/api/README.md index 420558cfbb..350b312a62 100644 --- a/tests/e2e/api/README.md +++ b/tests/e2e/api/README.md @@ -26,6 +26,14 @@ End-to-end API tests for the Bifrost API using Postman collections and [Newman]( | `run-newman-composite-integration.sh` | Script to run composite integration tests | | `run-all-integration-tests.sh` | Master script to run all integration test suites | +### Model Catalog Wiring Tests + +| Path | Description | +|------|-------------| +| `collections/bifrost-model-catalog-wiring.postman_collection.json` | Generated collection asserting that management-API mutations (add/update/delete provider and key, toggle key, alias) propagate into the model catalog read endpoints. **Generated — do not hand-edit.** | +| `runners/build-model-catalog-wiring-collection.py` | Generator for the collection above. Holds the scenario spec (the source of truth) and emits the JSON. | +| `runners/individual/run-newman-model-catalog-wiring-tests.sh` | Script to run the model-catalog wiring collection. | + ### Shared Resources | Path | Description | @@ -80,25 +88,34 @@ From this directory (`tests/e2e/api`): ./runners/run-newman-inference-tests.sh --html --verbose ``` +### Routing Harness Ledger + +Harness days are journaled in `routing/ledger-YYYY-MM-DD.md` (gitignored, one +file per day the routing/catalog suites run). Each ledger holds: an +open-divergences snapshot, per-suite scenario tables (setup / expected / +actual, with ✅ / ⚠️ recalibrated / 🐞 bug-found markers), the day's run +results, and day notes. Append to the current day's file during a session; +never rewrite past days. + ### API Management Extensions -The API management runner can merge enterprise-only Postman folders without checking -them into OSS: +The API management runner can merge additional Postman folders maintained +outside this repo: ```bash -./runners/run-newman-api-tests.sh --extra-collection /path/to/enterprise.postman_collection.json +./runners/run-newman-api-tests.sh --extra-collection /path/to/extra.postman_collection.json ``` You can also pass extensions via environment variable: ```bash -BIFROST_API_EXTRA_COLLECTION=/path/to/enterprise.postman_collection.json \ +BIFROST_API_EXTRA_COLLECTION=/path/to/extra.postman_collection.json \ ./runners/run-newman-api-tests.sh ``` -The default OSS run does not load extra collections. Enterprise should pass its -collection from the enterprise repo, so shared management requests stay in OSS and -DAC-specific assertions stay out of OSS. +The default run loads no extra collections. Downstream repos pass their own +collections at run time, so the shared management requests live here while +assertions specific to those repos stay with them. **Retry logic (CI)** When `CI=1` or `CI=true` is set (case-insensitive), each failing request in the V1 collection is retried up to 3 times before moving to the next request. This helps with flaky tests in CI. The runner passes the value through to Newman when the environment variable is set (e.g. `CI=1 ./runners/run-newman-inference-tests.sh --env openai` or `CI=true ./runners/run-newman-inference-tests.sh --env openai`). Retry attempts are logged to the console as `[RETRY] Request "..." failed (attempt n/3). Retrying...`. @@ -123,6 +140,61 @@ When `CI=1` or `CI=true` is set (case-insensitive), each failing request in the ./run-newman-openai-integration.sh --env azure # Test Azure-specific paths ``` +### Model Catalog Wiring Tests + +These tests cover the path **HTTP mutation → config write → server-side catalog +hook → read endpoint**: the wiring that keeps the model catalog (`/api/models`, +`/api/models/details`) in sync with provider and key changes made through the +management API. Each scenario stands up an isolated custom provider backed by a +real upstream (OpenAI), drives a sequence of mutations, and asserts the catalog +reflects each one. + +What it covers (one scenario per contract): + +- **Add provider + key** — a gated key surfaces its allowed model. +- **Update key model set** — changing a key's allow-list re-gates the catalog. +- **Disable / re-enable key** — a disabled key drops its models; re-enabling restores them. +- **Delete one of two keys** — only the deleted key's models drop; the sibling's survive. +- **Delete provider** — the provider and its models disappear from the catalog. +- **Alias resolution** — an inference call via a key alias routes to the underlying model. + +Run locally (from this directory): + +```bash +./runners/individual/run-newman-model-catalog-wiring-tests.sh +``` + +Requirements: + +- Bifrost running at `{{base_url}}` (default `http://localhost:8080`), ideally + against a clean config store so no pre-existing `catwiring-*` providers linger. +- `openai_api_key` available — either in the seed env file (`generated/seed.env` + or `$BIFROST_E2E_SEED_ENV`) or exported in the shell. Scenarios whose required + credentials are missing skip themselves rather than fail. + +Notes: + +- Every resource is named `catwiring-openai--`, where the + run-id is built once per run from `e2e_seed_prefix` plus a timestamp nonce, so + parallel runs never collide and a failed run leaves no blocking state. +- The catalog's live-model cache is populated asynchronously by the key hooks, so + every post-mutation read polls with exponential backoff (up to 8 attempts) + instead of asserting immediately. +- Each scenario has a cleanup folder that deletes its provider (cascading to its + keys); it runs even when a mid-scenario step fails, and accepts 200/204/404. +- To change or extend the scenarios, edit + `runners/build-model-catalog-wiring-collection.py` and re-run it, then commit + both the script and the regenerated collection: + + ```bash + python3 runners/build-model-catalog-wiring-collection.py + ``` + +Required seed-env vars: `openai_api_key`, plus `e2e_seed_prefix` for +run-id namespacing. The runner also forwards the full per-provider credential set +(`anthropic_api_key`, `azure_*`, `bedrock_*`, `vertex_*`, etc.) so per-provider +expansion needs no runner change. + ### Test Success Criteria A request **passes** if either: diff --git a/tests/e2e/api/collections/bifrost-model-catalog-wiring.postman_collection.json b/tests/e2e/api/collections/bifrost-model-catalog-wiring.postman_collection.json new file mode 100644 index 0000000000..c02f9422cd --- /dev/null +++ b/tests/e2e/api/collections/bifrost-model-catalog-wiring.postman_collection.json @@ -0,0 +1,11699 @@ +{ + "info": { + "_postman_id": "bifrost-model-catalog-wiring", + "name": "Bifrost Model Catalog Wiring", + "description": "End-to-end wiring tests between the management API and the model catalog. Each scenario stands up an isolated, run-namespaced custom provider backed by a real upstream (OpenAI, Anthropic, or Gemini), mutates its providers/keys, and asserts the catalog read endpoints reflect each mutation. Reads that depend on the asynchronously populated live-model cache poll with exponential backoff. Machine-generated by runners/build-model-catalog-wiring.mjs — do not hand-edit.", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" + }, + "variable": [ + { + "key": "base_url", + "value": "http://localhost:8080", + "type": "string" + }, + { + "key": "run_id", + "value": "", + "type": "string" + }, + { + "key": "__purge_target", + "value": "", + "type": "string" + }, + { + "key": "__purge_queue", + "value": "", + "type": "string" + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "if (!pm.collectionVariables.get('run_id')) {", + " var seed = pm.variables.get('e2e_seed_prefix') || pm.environment.get('e2e_seed_prefix') || 'local';", + " var nonce = Date.now().toString(36) + '-' + Math.floor(Math.random() * 1000000).toString(36);", + " pm.collectionVariables.set('run_id', seed + '-' + nonce);", + " console.log('run_id = ' + pm.collectionVariables.get('run_id'));", + "}" + ] + } + } + ], + "item": [ + { + "id": "setup-clear-providers", + "name": "Setup: clear providers", + "item": [ + { + "id": "setup-list-providers", + "name": "setup: list providers to clear", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('setup: list providers', function () { pm.expect(pm.response.code, pm.response.text()).to.equal(200); });", + "if (pm.response.code !== 200) { return; }", + "var provs = ((pm.response.json() || {}).providers || []).map(function (p) { return p.name; }).filter(Boolean);", + "if (provs.length === 0) { pm.collectionVariables.set('__purge_target', ''); pm.collectionVariables.set('__purge_queue', ''); return; }", + "pm.collectionVariables.set('__purge_target', provs[0]);", + "pm.collectionVariables.set('__purge_queue', provs.slice(1).join(','));", + "pm.execution.setNextRequest(\"setup: delete provider\");" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + } + } + }, + { + "id": "setup-delete-provider", + "name": "setup: delete provider", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "if (!pm.collectionVariables.get('__purge_target')) { pm.execution.skipRequest(); }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('setup: delete provider', function () { pm.expect([200, 204, 404], 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code); });", + "var queue = (pm.collectionVariables.get('__purge_queue') || '').split(',').filter(Boolean);", + "if (queue.length) {", + " pm.collectionVariables.set('__purge_target', queue[0]);", + " pm.collectionVariables.set('__purge_queue', queue.slice(1).join(','));", + " pm.execution.setNextRequest(\"setup: delete provider\");", + "} else {", + " pm.collectionVariables.set('__purge_target', '');", + "}" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/{{__purge_target}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "{{__purge_target}}" + ] + } + } + } + ] + }, + { + "name": "Add provider and key surfaces models (openai)", + "description": "A fresh provider plus one gated key surfaces that key's allowed model in the catalog.", + "item": [ + { + "id": "catwiring-openai-add-provider-and-key-01-add-provider", + "name": "01. add provider [openai-add-provider-and-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [openai-add-provider-and-key]\";", + "pm.test(\"01. add provider [openai-add-provider-and-key]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-openai-add-provider-and-key-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-openai-add-provider-and-key-02-add-key", + "name": "02. add key k1 [openai-add-provider-and-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [openai-add-provider-and-key]\";", + "pm.test(\"02. add key k1 [openai-add-provider-and-key]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-add-provider-and-key-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-add-provider-and-key-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"openai-add-provider-and-key-k1-{{run_id}}\",\"name\":\"catwiring-mc-openai-add-provider-and-key-k1-{{run_id}}\",\"value\":\"env.OPENAI_API_KEY\",\"models\":[\"gpt-4o\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-openai-add-provider-and-key-03-assert-models", + "name": "03. key model appears in catalog [openai-add-provider-and-key]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [openai-add-provider-and-key]\";", + "function assertNow() {", + " var providerName = 'catwiring-openai-add-provider-and-key-' + pm.variables.get('run_id');", + " var expectSubset = [\"gpt-4o\"];", + " var expectSuperset = [];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. key model appears in catalog [openai-add-provider-and-key]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. key model appears in catalog [openai-add-provider-and-key]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-openai-add-provider-and-key-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-openai-add-provider-and-key-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-openai-add-provider-and-key-cleanup", + "name": "cleanup: delete provider [openai-add-provider-and-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [openai-add-provider-and-key]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-add-provider-and-key-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-add-provider-and-key-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Updating key model set re-gates the catalog (openai)", + "description": "Changing a key's allow-list invalidates the stale live entry and re-gates the surfaced models.", + "item": [ + { + "id": "catwiring-openai-update-key-models-01-add-provider", + "name": "01. add provider [openai-update-key-models]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [openai-update-key-models]\";", + "pm.test(\"01. add provider [openai-update-key-models]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-openai-update-key-models-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-openai-update-key-models-02-add-key", + "name": "02. add key k1 [openai-update-key-models]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [openai-update-key-models]\";", + "pm.test(\"02. add key k1 [openai-update-key-models]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-update-key-models-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-update-key-models-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"openai-update-key-models-k1-{{run_id}}\",\"name\":\"catwiring-mc-openai-update-key-models-k1-{{run_id}}\",\"value\":\"env.OPENAI_API_KEY\",\"models\":[\"gpt-4o\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-openai-update-key-models-03-assert-models", + "name": "03. initial model gated in [openai-update-key-models]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [openai-update-key-models]\";", + "function assertNow() {", + " var providerName = 'catwiring-openai-update-key-models-' + pm.variables.get('run_id');", + " var expectSubset = [\"gpt-4o\"];", + " var expectSuperset = [];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. initial model gated in [openai-update-key-models]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. initial model gated in [openai-update-key-models]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-openai-update-key-models-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-openai-update-key-models-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "id": "catwiring-openai-update-key-models-04-update-key", + "name": "04. update key k1 [openai-update-key-models]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200];", + "var cleanupReq = \"cleanup: delete provider [openai-update-key-models]\";", + "pm.test(\"04. update key k1 [openai-update-key-models]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "PUT", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-update-key-models-{{run_id}}/keys/openai-update-key-models-k1-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-update-key-models-{{run_id}}", + "keys", + "openai-update-key-models-k1-{{run_id}}" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"openai-update-key-models-k1-{{run_id}}\",\"name\":\"catwiring-mc-openai-update-key-models-k1-{{run_id}}\",\"value\":\"env.OPENAI_API_KEY\",\"models\":[\"gpt-4o-mini\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-openai-update-key-models-05-assert-models", + "name": "05. new model gated in, old gated out [openai-update-key-models]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [openai-update-key-models]\";", + "function assertNow() {", + " var providerName = 'catwiring-openai-update-key-models-' + pm.variables.get('run_id');", + " var expectSubset = [\"gpt-4o-mini\"];", + " var expectSuperset = [];", + " var expectAbsent = [\"gpt-4o\"];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. new model gated in, old gated out [openai-update-key-models]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. new model gated in, old gated out [openai-update-key-models]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-openai-update-key-models-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-openai-update-key-models-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-openai-update-key-models-cleanup", + "name": "cleanup: delete provider [openai-update-key-models]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [openai-update-key-models]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-update-key-models-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-update-key-models-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Disabled key drops models, re-enable restores (openai)", + "description": "A disabled key is skipped during aggregation; re-enabling it brings its models back.", + "item": [ + { + "id": "catwiring-openai-disable-reenable-key-01-add-provider", + "name": "01. add provider [openai-disable-reenable-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [openai-disable-reenable-key]\";", + "pm.test(\"01. add provider [openai-disable-reenable-key]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-openai-disable-reenable-key-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-openai-disable-reenable-key-02-add-key", + "name": "02. add key k1 [openai-disable-reenable-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [openai-disable-reenable-key]\";", + "pm.test(\"02. add key k1 [openai-disable-reenable-key]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-disable-reenable-key-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-disable-reenable-key-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"openai-disable-reenable-key-k1-{{run_id}}\",\"name\":\"catwiring-mc-openai-disable-reenable-key-k1-{{run_id}}\",\"value\":\"env.OPENAI_API_KEY\",\"models\":[\"gpt-4o\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-openai-disable-reenable-key-03-assert-models", + "name": "03. model present while enabled [openai-disable-reenable-key]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [openai-disable-reenable-key]\";", + "function assertNow() {", + " var providerName = 'catwiring-openai-disable-reenable-key-' + pm.variables.get('run_id');", + " var expectSubset = [\"gpt-4o\"];", + " var expectSuperset = [];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. model present while enabled [openai-disable-reenable-key]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. model present while enabled [openai-disable-reenable-key]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-openai-disable-reenable-key-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-openai-disable-reenable-key-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "id": "catwiring-openai-disable-reenable-key-04-update-key", + "name": "04. update key k1 [openai-disable-reenable-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200];", + "var cleanupReq = \"cleanup: delete provider [openai-disable-reenable-key]\";", + "pm.test(\"04. update key k1 [openai-disable-reenable-key]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "PUT", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-disable-reenable-key-{{run_id}}/keys/openai-disable-reenable-key-k1-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-disable-reenable-key-{{run_id}}", + "keys", + "openai-disable-reenable-key-k1-{{run_id}}" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"openai-disable-reenable-key-k1-{{run_id}}\",\"name\":\"catwiring-mc-openai-disable-reenable-key-k1-{{run_id}}\",\"value\":\"env.OPENAI_API_KEY\",\"models\":[\"gpt-4o\"],\"enabled\":false}" + } + } + }, + { + "id": "catwiring-openai-disable-reenable-key-05-assert-models", + "name": "05. model gone while disabled [openai-disable-reenable-key]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [openai-disable-reenable-key]\";", + "function assertNow() {", + " var providerName = 'catwiring-openai-disable-reenable-key-' + pm.variables.get('run_id');", + " var expectSubset = [];", + " var expectSuperset = [];", + " var expectAbsent = [\"gpt-4o\"];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. model gone while disabled [openai-disable-reenable-key]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. model gone while disabled [openai-disable-reenable-key]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-openai-disable-reenable-key-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-openai-disable-reenable-key-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "id": "catwiring-openai-disable-reenable-key-06-update-key", + "name": "06. update key k1 [openai-disable-reenable-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200];", + "var cleanupReq = \"cleanup: delete provider [openai-disable-reenable-key]\";", + "pm.test(\"06. update key k1 [openai-disable-reenable-key]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "PUT", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-disable-reenable-key-{{run_id}}/keys/openai-disable-reenable-key-k1-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-disable-reenable-key-{{run_id}}", + "keys", + "openai-disable-reenable-key-k1-{{run_id}}" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"openai-disable-reenable-key-k1-{{run_id}}\",\"name\":\"catwiring-mc-openai-disable-reenable-key-k1-{{run_id}}\",\"value\":\"env.OPENAI_API_KEY\",\"models\":[\"gpt-4o\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-openai-disable-reenable-key-07-assert-models", + "name": "07. model returns when re-enabled [openai-disable-reenable-key]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [openai-disable-reenable-key]\";", + "function assertNow() {", + " var providerName = 'catwiring-openai-disable-reenable-key-' + pm.variables.get('run_id');", + " var expectSubset = [\"gpt-4o\"];", + " var expectSuperset = [];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"07. model returns when re-enabled [openai-disable-reenable-key]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"07. model returns when re-enabled [openai-disable-reenable-key]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-openai-disable-reenable-key-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-openai-disable-reenable-key-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-openai-disable-reenable-key-cleanup", + "name": "cleanup: delete provider [openai-disable-reenable-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [openai-disable-reenable-key]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-disable-reenable-key-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-disable-reenable-key-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Deleting one key leaves sibling models intact (openai)", + "description": "With two keys gating distinct models, deleting one removes only its models; the sibling's survive.", + "item": [ + { + "id": "catwiring-openai-delete-key-sibling-survives-01-add-provider", + "name": "01. add provider [openai-delete-key-sibling-survives]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [openai-delete-key-sibling-survives]\";", + "pm.test(\"01. add provider [openai-delete-key-sibling-survives]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-openai-delete-key-sibling-survives-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-openai-delete-key-sibling-survives-02-add-key", + "name": "02. add key k1 [openai-delete-key-sibling-survives]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [openai-delete-key-sibling-survives]\";", + "pm.test(\"02. add key k1 [openai-delete-key-sibling-survives]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-delete-key-sibling-survives-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-delete-key-sibling-survives-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"openai-delete-key-sibling-survives-k1-{{run_id}}\",\"name\":\"catwiring-mc-openai-delete-key-sibling-survives-k1-{{run_id}}\",\"value\":\"env.OPENAI_API_KEY\",\"models\":[\"gpt-4o\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-openai-delete-key-sibling-survives-03-add-key", + "name": "03. add key k2 [openai-delete-key-sibling-survives]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [openai-delete-key-sibling-survives]\";", + "pm.test(\"03. add key k2 [openai-delete-key-sibling-survives]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-delete-key-sibling-survives-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-delete-key-sibling-survives-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"openai-delete-key-sibling-survives-k2-{{run_id}}\",\"name\":\"catwiring-mc-openai-delete-key-sibling-survives-k2-{{run_id}}\",\"value\":\"env.OPENAI_API_KEY\",\"models\":[\"gpt-4o-mini\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-openai-delete-key-sibling-survives-04-assert-models", + "name": "04. both keys' models present [openai-delete-key-sibling-survives]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [openai-delete-key-sibling-survives]\";", + "function assertNow() {", + " var providerName = 'catwiring-openai-delete-key-sibling-survives-' + pm.variables.get('run_id');", + " var expectSubset = [];", + " var expectSuperset = [\"gpt-4o\",\"gpt-4o-mini\"];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. both keys' models present [openai-delete-key-sibling-survives]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. both keys' models present [openai-delete-key-sibling-survives]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-openai-delete-key-sibling-survives-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-openai-delete-key-sibling-survives-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "id": "catwiring-openai-delete-key-sibling-survives-05-delete-key", + "name": "05. delete key k1 [openai-delete-key-sibling-survives]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,204];", + "var cleanupReq = \"cleanup: delete provider [openai-delete-key-sibling-survives]\";", + "pm.test(\"05. delete key k1 [openai-delete-key-sibling-survives]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-delete-key-sibling-survives-{{run_id}}/keys/openai-delete-key-sibling-survives-k1-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-delete-key-sibling-survives-{{run_id}}", + "keys", + "openai-delete-key-sibling-survives-k1-{{run_id}}" + ] + } + } + }, + { + "id": "catwiring-openai-delete-key-sibling-survives-06-assert-models", + "name": "06. sibling model survives delete [openai-delete-key-sibling-survives]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [openai-delete-key-sibling-survives]\";", + "function assertNow() {", + " var providerName = 'catwiring-openai-delete-key-sibling-survives-' + pm.variables.get('run_id');", + " var expectSubset = [\"gpt-4o-mini\"];", + " var expectSuperset = [];", + " var expectAbsent = [\"gpt-4o\"];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"06. sibling model survives delete [openai-delete-key-sibling-survives]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"06. sibling model survives delete [openai-delete-key-sibling-survives]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-openai-delete-key-sibling-survives-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-openai-delete-key-sibling-survives-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-openai-delete-key-sibling-survives-cleanup", + "name": "cleanup: delete provider [openai-delete-key-sibling-survives]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [openai-delete-key-sibling-survives]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-delete-key-sibling-survives-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-delete-key-sibling-survives-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Deleting provider removes it from the catalog (openai)", + "description": "Removing the provider drops all of its models from the catalog read endpoints.", + "item": [ + { + "id": "catwiring-openai-delete-provider-01-add-provider", + "name": "01. add provider [openai-delete-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [openai-delete-provider]\";", + "pm.test(\"01. add provider [openai-delete-provider]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-openai-delete-provider-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-openai-delete-provider-02-add-key", + "name": "02. add key k1 [openai-delete-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [openai-delete-provider]\";", + "pm.test(\"02. add key k1 [openai-delete-provider]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-delete-provider-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-delete-provider-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"openai-delete-provider-k1-{{run_id}}\",\"name\":\"catwiring-mc-openai-delete-provider-k1-{{run_id}}\",\"value\":\"env.OPENAI_API_KEY\",\"models\":[\"gpt-4o\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-openai-delete-provider-03-assert-models", + "name": "03. model present before delete [openai-delete-provider]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [openai-delete-provider]\";", + "function assertNow() {", + " var providerName = 'catwiring-openai-delete-provider-' + pm.variables.get('run_id');", + " var expectSubset = [\"gpt-4o\"];", + " var expectSuperset = [];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. model present before delete [openai-delete-provider]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. model present before delete [openai-delete-provider]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-openai-delete-provider-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-openai-delete-provider-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "id": "catwiring-openai-delete-provider-04-delete-provider", + "name": "04. delete provider [openai-delete-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,204];", + "var cleanupReq = \"cleanup: delete provider [openai-delete-provider]\";", + "pm.test(\"04. delete provider [openai-delete-provider]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-delete-provider-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-delete-provider-{{run_id}}" + ] + } + } + }, + { + "id": "catwiring-openai-delete-provider-05-assert-models", + "name": "05. no models after provider delete [openai-delete-provider]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 1000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [openai-delete-provider]\";", + "function assertNow() {", + " var providerName = 'catwiring-openai-delete-provider-' + pm.variables.get('run_id');", + " var expectSubset = [];", + " var expectSuperset = [];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = true;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. no models after provider delete [openai-delete-provider]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. no models after provider delete [openai-delete-provider]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-openai-delete-provider-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-openai-delete-provider-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-openai-delete-provider-cleanup", + "name": "cleanup: delete provider [openai-delete-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [openai-delete-provider]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-delete-provider-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-delete-provider-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Alias resolves to underlying model at inference (openai)", + "description": "A key alias routes an inference request to the underlying wire model rather than being rejected.", + "item": [ + { + "id": "catwiring-openai-alias-resolution-01-add-provider", + "name": "01. add provider [openai-alias-resolution]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [openai-alias-resolution]\";", + "pm.test(\"01. add provider [openai-alias-resolution]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-openai-alias-resolution-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-openai-alias-resolution-02-add-key", + "name": "02. add key k1 [openai-alias-resolution]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [openai-alias-resolution]\";", + "pm.test(\"02. add key k1 [openai-alias-resolution]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-alias-resolution-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-alias-resolution-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"openai-alias-resolution-k1-{{run_id}}\",\"name\":\"catwiring-mc-openai-alias-resolution-k1-{{run_id}}\",\"value\":\"env.OPENAI_API_KEY\",\"models\":[\"catwiring-alias-openai-{{run_id}}\",\"gpt-4o-mini\"],\"enabled\":true,\"aliases\":{\"catwiring-alias-openai-{{run_id}}\":\"gpt-4o-mini\"}}" + } + } + }, + { + "id": "catwiring-openai-alias-resolution-03-assert-models", + "name": "03. aliased key model present [openai-alias-resolution]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [openai-alias-resolution]\";", + "function assertNow() {", + " var providerName = 'catwiring-openai-alias-resolution-' + pm.variables.get('run_id');", + " var expectSubset = [\"gpt-4o-mini\"];", + " var expectSuperset = [];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. aliased key model present [openai-alias-resolution]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. aliased key model present [openai-alias-resolution]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-openai-alias-resolution-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-openai-alias-resolution-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "id": "catwiring-openai-alias-resolution-04-assert-inference", + "name": "04. alias resolves to underlying model [openai-alias-resolution]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [openai-alias-resolution]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('inference status ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " if (!body.choices || body.choices.length === 0) { throw new Error('inference returned no choices'); }", + " var used = body.model || '';", + " if (used.indexOf(\"gpt-4o-mini\") < 0) { throw new Error('expected resolved model gpt-4o-mini in response.model=' + used); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. alias resolves to underlying model [openai-alias-resolution]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. alias resolves to underlying model [openai-alias-resolution]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-openai-alias-resolution-{{run_id}}/catwiring-alias-openai-{{run_id}}\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-openai-alias-resolution-cleanup", + "name": "cleanup: delete provider [openai-alias-resolution]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [openai-alias-resolution]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-alias-resolution-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-alias-resolution-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Blacklisting a model re-gates a wildcard catalog (openai)", + "description": "A wildcard key surfaces the upstream's live model list; adding one of those models to the key's blacklist drops it from the catalog while the rest of the list stays. The target model is captured from the live list at run time because some upstreams only report dated ids.", + "item": [ + { + "id": "catwiring-openai-blacklist-regate-01-add-provider", + "name": "01. add provider [openai-blacklist-regate]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [openai-blacklist-regate]\";", + "pm.test(\"01. add provider [openai-blacklist-regate]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-openai-blacklist-regate-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-openai-blacklist-regate-02-add-key", + "name": "02. add key k1 [openai-blacklist-regate]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [openai-blacklist-regate]\";", + "pm.test(\"02. add key k1 [openai-blacklist-regate]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-blacklist-regate-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-blacklist-regate-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"openai-blacklist-regate-k1-{{run_id}}\",\"name\":\"catwiring-mc-openai-blacklist-regate-k1-{{run_id}}\",\"value\":\"env.OPENAI_API_KEY\",\"models\":[\"*\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-openai-blacklist-regate-03-capture-model", + "name": "03. wildcard catalog serves live models [openai-blacklist-regate]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [openai-blacklist-regate]\";", + "function assertNow() {", + " var providerName = 'catwiring-openai-blacklist-regate-' + pm.variables.get('run_id');", + " var prefix = \"gpt-4o\";", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " var hit = null;", + " for (var i = 0; i < names.length; i++) {", + " if (names[i].indexOf(prefix) === 0) { hit = names[i]; break; }", + " }", + " if (!hit) { throw new Error('no live model with prefix ' + prefix + ' in ' + JSON.stringify(names.slice(0, 20))); }", + " pm.collectionVariables.set(\"bl_target_openai\", hit);", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. wildcard catalog serves live models [openai-blacklist-regate]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. wildcard catalog serves live models [openai-blacklist-regate]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-openai-blacklist-regate-{{run_id}}&limit=2000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-openai-blacklist-regate-{{run_id}}" + }, + { + "key": "limit", + "value": "2000" + } + ] + } + } + }, + { + "id": "catwiring-openai-blacklist-regate-04-update-key", + "name": "04. update key k1 [openai-blacklist-regate]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200];", + "var cleanupReq = \"cleanup: delete provider [openai-blacklist-regate]\";", + "pm.test(\"04. update key k1 [openai-blacklist-regate]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "PUT", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-blacklist-regate-{{run_id}}/keys/openai-blacklist-regate-k1-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-blacklist-regate-{{run_id}}", + "keys", + "openai-blacklist-regate-k1-{{run_id}}" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"openai-blacklist-regate-k1-{{run_id}}\",\"name\":\"catwiring-mc-openai-blacklist-regate-k1-{{run_id}}\",\"value\":\"env.OPENAI_API_KEY\",\"models\":[\"*\"],\"enabled\":true,\"blacklisted_models\":[\"{{bl_target_openai}}\"]}" + } + } + }, + { + "id": "catwiring-openai-blacklist-regate-05-assert-models", + "name": "05. blacklisted model gated out, catalog still populated [openai-blacklist-regate]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [openai-blacklist-regate]\";", + "function assertNow() {", + " var providerName = 'catwiring-openai-blacklist-regate-' + pm.variables.get('run_id');", + " var expectSubset = [];", + " var expectSuperset = [];", + " var expectAbsent = [];", + " var expectAbsentVars = [\"bl_target_openai\"];", + " var expectEmpty = false;", + " var expectNonEmpty = true;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. blacklisted model gated out, catalog still populated [openai-blacklist-regate]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. blacklisted model gated out, catalog still populated [openai-blacklist-regate]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-openai-blacklist-regate-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-openai-blacklist-regate-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-openai-blacklist-regate-cleanup", + "name": "cleanup: delete provider [openai-blacklist-regate]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [openai-blacklist-regate]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-blacklist-regate-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-blacklist-regate-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Catalog aggregates the union across enabled keys (openai)", + "description": "Two keys gating distinct models both contribute: the catalog lists the union of their allow-lists.", + "item": [ + { + "id": "catwiring-openai-multi-key-union-01-add-provider", + "name": "01. add provider [openai-multi-key-union]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [openai-multi-key-union]\";", + "pm.test(\"01. add provider [openai-multi-key-union]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-openai-multi-key-union-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-openai-multi-key-union-02-add-key", + "name": "02. add key k1 [openai-multi-key-union]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [openai-multi-key-union]\";", + "pm.test(\"02. add key k1 [openai-multi-key-union]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-multi-key-union-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-multi-key-union-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"openai-multi-key-union-k1-{{run_id}}\",\"name\":\"catwiring-mc-openai-multi-key-union-k1-{{run_id}}\",\"value\":\"env.OPENAI_API_KEY\",\"models\":[\"gpt-4o\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-openai-multi-key-union-03-add-key", + "name": "03. add key k2 [openai-multi-key-union]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [openai-multi-key-union]\";", + "pm.test(\"03. add key k2 [openai-multi-key-union]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-multi-key-union-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-multi-key-union-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"openai-multi-key-union-k2-{{run_id}}\",\"name\":\"catwiring-mc-openai-multi-key-union-k2-{{run_id}}\",\"value\":\"env.OPENAI_API_KEY\",\"models\":[\"gpt-4o-mini\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-openai-multi-key-union-04-assert-models", + "name": "04. catalog lists union of both keys' models [openai-multi-key-union]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [openai-multi-key-union]\";", + "function assertNow() {", + " var providerName = 'catwiring-openai-multi-key-union-' + pm.variables.get('run_id');", + " var expectSubset = [];", + " var expectSuperset = [\"gpt-4o\",\"gpt-4o-mini\"];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. catalog lists union of both keys' models [openai-multi-key-union]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. catalog lists union of both keys' models [openai-multi-key-union]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-openai-multi-key-union-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-openai-multi-key-union-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-openai-multi-key-union-cleanup", + "name": "cleanup: delete provider [openai-multi-key-union]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [openai-multi-key-union]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-multi-key-union-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-multi-key-union-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Model details endpoint respects the key gate (openai)", + "description": "/api/models/details lists only the models the provider's keys allow, like the plain list endpoint.", + "item": [ + { + "id": "catwiring-openai-model-details-gating-01-add-provider", + "name": "01. add provider [openai-model-details-gating]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [openai-model-details-gating]\";", + "pm.test(\"01. add provider [openai-model-details-gating]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-openai-model-details-gating-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-openai-model-details-gating-02-add-key", + "name": "02. add key k1 [openai-model-details-gating]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [openai-model-details-gating]\";", + "pm.test(\"02. add key k1 [openai-model-details-gating]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-model-details-gating-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-model-details-gating-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"openai-model-details-gating-k1-{{run_id}}\",\"name\":\"catwiring-mc-openai-model-details-gating-k1-{{run_id}}\",\"value\":\"env.OPENAI_API_KEY\",\"models\":[\"gpt-4o\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-openai-model-details-gating-03-assert-model-details", + "name": "03. details list gated to key models [openai-model-details-gating]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [openai-model-details-gating]\";", + "function assertNow() {", + " var providerName = 'catwiring-openai-model-details-gating-' + pm.variables.get('run_id');", + " var expectSubset = [\"gpt-4o\"];", + " var expectSuperset = [];", + " var expectAbsent = [\"gpt-4o-mini\"];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. details list gated to key models [openai-model-details-gating]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. details list gated to key models [openai-model-details-gating]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models/details?provider=catwiring-openai-model-details-gating-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models", + "details" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-openai-model-details-gating-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-openai-model-details-gating-cleanup", + "name": "cleanup: delete provider [openai-model-details-gating]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [openai-model-details-gating]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-model-details-gating-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-model-details-gating-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Base model list is unaffected by key changes (openai)", + "description": "/api/models/base reflects the datasheet's distinct base names; removing a model from a key's allow-list must not remove its base name from that list.", + "item": [ + { + "id": "catwiring-openai-base-models-stable-01-add-provider", + "name": "01. add provider [openai-base-models-stable]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [openai-base-models-stable]\";", + "pm.test(\"01. add provider [openai-base-models-stable]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-openai-base-models-stable-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-openai-base-models-stable-02-add-key", + "name": "02. add key k1 [openai-base-models-stable]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [openai-base-models-stable]\";", + "pm.test(\"02. add key k1 [openai-base-models-stable]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-base-models-stable-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-base-models-stable-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"openai-base-models-stable-k1-{{run_id}}\",\"name\":\"catwiring-mc-openai-base-models-stable-k1-{{run_id}}\",\"value\":\"env.OPENAI_API_KEY\",\"models\":[\"gpt-4o\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-openai-base-models-stable-03-assert-base-models", + "name": "03. base name listed while key allows it [openai-base-models-stable]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 1000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [openai-base-models-stable]\";", + "function assertNow() {", + " var target = \"gpt-4o\";", + " if (pm.response.code !== 200) { throw new Error('list base models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = body.models || [];", + " if (names.indexOf(target) < 0) { throw new Error('expected base model ' + target + ' in ' + JSON.stringify(names)); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. base name listed while key allows it [openai-base-models-stable]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. base name listed while key allows it [openai-base-models-stable]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models/base?query=gpt-4o&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models", + "base" + ], + "query": [ + { + "key": "query", + "value": "gpt-4o" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "id": "catwiring-openai-base-models-stable-04-update-key", + "name": "04. update key k1 [openai-base-models-stable]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200];", + "var cleanupReq = \"cleanup: delete provider [openai-base-models-stable]\";", + "pm.test(\"04. update key k1 [openai-base-models-stable]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "PUT", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-base-models-stable-{{run_id}}/keys/openai-base-models-stable-k1-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-base-models-stable-{{run_id}}", + "keys", + "openai-base-models-stable-k1-{{run_id}}" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"openai-base-models-stable-k1-{{run_id}}\",\"name\":\"catwiring-mc-openai-base-models-stable-k1-{{run_id}}\",\"value\":\"env.OPENAI_API_KEY\",\"models\":[\"gpt-4o-mini\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-openai-base-models-stable-05-assert-base-models", + "name": "05. base name still listed after key change [openai-base-models-stable]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 1000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [openai-base-models-stable]\";", + "function assertNow() {", + " var target = \"gpt-4o\";", + " if (pm.response.code !== 200) { throw new Error('list base models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = body.models || [];", + " if (names.indexOf(target) < 0) { throw new Error('expected base model ' + target + ' in ' + JSON.stringify(names)); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. base name still listed after key change [openai-base-models-stable]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. base name still listed after key change [openai-base-models-stable]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models/base?query=gpt-4o&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models", + "base" + ], + "query": [ + { + "key": "query", + "value": "gpt-4o" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-openai-base-models-stable-cleanup", + "name": "cleanup: delete provider [openai-base-models-stable]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [openai-base-models-stable]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-openai-base-models-stable-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-openai-base-models-stable-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Add provider and key surfaces models (anthropic)", + "description": "A fresh provider plus one gated key surfaces that key's allowed model in the catalog.", + "item": [ + { + "id": "catwiring-anthropic-add-provider-and-key-01-add-provider", + "name": "01. add provider [anthropic-add-provider-and-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [anthropic-add-provider-and-key]\";", + "pm.test(\"01. add provider [anthropic-add-provider-and-key]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-anthropic-add-provider-and-key-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"anthropic\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-anthropic-add-provider-and-key-02-add-key", + "name": "02. add key k1 [anthropic-add-provider-and-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [anthropic-add-provider-and-key]\";", + "pm.test(\"02. add key k1 [anthropic-add-provider-and-key]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-add-provider-and-key-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-add-provider-and-key-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"anthropic-add-provider-and-key-k1-{{run_id}}\",\"name\":\"catwiring-mc-anthropic-add-provider-and-key-k1-{{run_id}}\",\"value\":\"env.ANTHROPIC_API_KEY\",\"models\":[\"claude-sonnet-4-5\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-anthropic-add-provider-and-key-03-assert-models", + "name": "03. key model appears in catalog [anthropic-add-provider-and-key]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [anthropic-add-provider-and-key]\";", + "function assertNow() {", + " var providerName = 'catwiring-anthropic-add-provider-and-key-' + pm.variables.get('run_id');", + " var expectSubset = [\"claude-sonnet-4-5\"];", + " var expectSuperset = [];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. key model appears in catalog [anthropic-add-provider-and-key]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. key model appears in catalog [anthropic-add-provider-and-key]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-anthropic-add-provider-and-key-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-anthropic-add-provider-and-key-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-anthropic-add-provider-and-key-cleanup", + "name": "cleanup: delete provider [anthropic-add-provider-and-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [anthropic-add-provider-and-key]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-add-provider-and-key-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-add-provider-and-key-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Updating key model set re-gates the catalog (anthropic)", + "description": "Changing a key's allow-list invalidates the stale live entry and re-gates the surfaced models.", + "item": [ + { + "id": "catwiring-anthropic-update-key-models-01-add-provider", + "name": "01. add provider [anthropic-update-key-models]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [anthropic-update-key-models]\";", + "pm.test(\"01. add provider [anthropic-update-key-models]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-anthropic-update-key-models-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"anthropic\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-anthropic-update-key-models-02-add-key", + "name": "02. add key k1 [anthropic-update-key-models]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [anthropic-update-key-models]\";", + "pm.test(\"02. add key k1 [anthropic-update-key-models]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-update-key-models-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-update-key-models-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"anthropic-update-key-models-k1-{{run_id}}\",\"name\":\"catwiring-mc-anthropic-update-key-models-k1-{{run_id}}\",\"value\":\"env.ANTHROPIC_API_KEY\",\"models\":[\"claude-sonnet-4-5\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-anthropic-update-key-models-03-assert-models", + "name": "03. initial model gated in [anthropic-update-key-models]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [anthropic-update-key-models]\";", + "function assertNow() {", + " var providerName = 'catwiring-anthropic-update-key-models-' + pm.variables.get('run_id');", + " var expectSubset = [\"claude-sonnet-4-5\"];", + " var expectSuperset = [];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. initial model gated in [anthropic-update-key-models]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. initial model gated in [anthropic-update-key-models]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-anthropic-update-key-models-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-anthropic-update-key-models-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "id": "catwiring-anthropic-update-key-models-04-update-key", + "name": "04. update key k1 [anthropic-update-key-models]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200];", + "var cleanupReq = \"cleanup: delete provider [anthropic-update-key-models]\";", + "pm.test(\"04. update key k1 [anthropic-update-key-models]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "PUT", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-update-key-models-{{run_id}}/keys/anthropic-update-key-models-k1-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-update-key-models-{{run_id}}", + "keys", + "anthropic-update-key-models-k1-{{run_id}}" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"anthropic-update-key-models-k1-{{run_id}}\",\"name\":\"catwiring-mc-anthropic-update-key-models-k1-{{run_id}}\",\"value\":\"env.ANTHROPIC_API_KEY\",\"models\":[\"claude-haiku-4-5\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-anthropic-update-key-models-05-assert-models", + "name": "05. new model gated in, old gated out [anthropic-update-key-models]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [anthropic-update-key-models]\";", + "function assertNow() {", + " var providerName = 'catwiring-anthropic-update-key-models-' + pm.variables.get('run_id');", + " var expectSubset = [\"claude-haiku-4-5\"];", + " var expectSuperset = [];", + " var expectAbsent = [\"claude-sonnet-4-5\"];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. new model gated in, old gated out [anthropic-update-key-models]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. new model gated in, old gated out [anthropic-update-key-models]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-anthropic-update-key-models-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-anthropic-update-key-models-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-anthropic-update-key-models-cleanup", + "name": "cleanup: delete provider [anthropic-update-key-models]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [anthropic-update-key-models]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-update-key-models-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-update-key-models-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Disabled key drops models, re-enable restores (anthropic)", + "description": "A disabled key is skipped during aggregation; re-enabling it brings its models back.", + "item": [ + { + "id": "catwiring-anthropic-disable-reenable-key-01-add-provider", + "name": "01. add provider [anthropic-disable-reenable-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [anthropic-disable-reenable-key]\";", + "pm.test(\"01. add provider [anthropic-disable-reenable-key]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-anthropic-disable-reenable-key-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"anthropic\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-anthropic-disable-reenable-key-02-add-key", + "name": "02. add key k1 [anthropic-disable-reenable-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [anthropic-disable-reenable-key]\";", + "pm.test(\"02. add key k1 [anthropic-disable-reenable-key]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-disable-reenable-key-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-disable-reenable-key-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"anthropic-disable-reenable-key-k1-{{run_id}}\",\"name\":\"catwiring-mc-anthropic-disable-reenable-key-k1-{{run_id}}\",\"value\":\"env.ANTHROPIC_API_KEY\",\"models\":[\"claude-sonnet-4-5\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-anthropic-disable-reenable-key-03-assert-models", + "name": "03. model present while enabled [anthropic-disable-reenable-key]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [anthropic-disable-reenable-key]\";", + "function assertNow() {", + " var providerName = 'catwiring-anthropic-disable-reenable-key-' + pm.variables.get('run_id');", + " var expectSubset = [\"claude-sonnet-4-5\"];", + " var expectSuperset = [];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. model present while enabled [anthropic-disable-reenable-key]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. model present while enabled [anthropic-disable-reenable-key]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-anthropic-disable-reenable-key-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-anthropic-disable-reenable-key-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "id": "catwiring-anthropic-disable-reenable-key-04-update-key", + "name": "04. update key k1 [anthropic-disable-reenable-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200];", + "var cleanupReq = \"cleanup: delete provider [anthropic-disable-reenable-key]\";", + "pm.test(\"04. update key k1 [anthropic-disable-reenable-key]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "PUT", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-disable-reenable-key-{{run_id}}/keys/anthropic-disable-reenable-key-k1-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-disable-reenable-key-{{run_id}}", + "keys", + "anthropic-disable-reenable-key-k1-{{run_id}}" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"anthropic-disable-reenable-key-k1-{{run_id}}\",\"name\":\"catwiring-mc-anthropic-disable-reenable-key-k1-{{run_id}}\",\"value\":\"env.ANTHROPIC_API_KEY\",\"models\":[\"claude-sonnet-4-5\"],\"enabled\":false}" + } + } + }, + { + "id": "catwiring-anthropic-disable-reenable-key-05-assert-models", + "name": "05. model gone while disabled [anthropic-disable-reenable-key]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [anthropic-disable-reenable-key]\";", + "function assertNow() {", + " var providerName = 'catwiring-anthropic-disable-reenable-key-' + pm.variables.get('run_id');", + " var expectSubset = [];", + " var expectSuperset = [];", + " var expectAbsent = [\"claude-sonnet-4-5\"];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. model gone while disabled [anthropic-disable-reenable-key]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. model gone while disabled [anthropic-disable-reenable-key]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-anthropic-disable-reenable-key-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-anthropic-disable-reenable-key-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "id": "catwiring-anthropic-disable-reenable-key-06-update-key", + "name": "06. update key k1 [anthropic-disable-reenable-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200];", + "var cleanupReq = \"cleanup: delete provider [anthropic-disable-reenable-key]\";", + "pm.test(\"06. update key k1 [anthropic-disable-reenable-key]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "PUT", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-disable-reenable-key-{{run_id}}/keys/anthropic-disable-reenable-key-k1-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-disable-reenable-key-{{run_id}}", + "keys", + "anthropic-disable-reenable-key-k1-{{run_id}}" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"anthropic-disable-reenable-key-k1-{{run_id}}\",\"name\":\"catwiring-mc-anthropic-disable-reenable-key-k1-{{run_id}}\",\"value\":\"env.ANTHROPIC_API_KEY\",\"models\":[\"claude-sonnet-4-5\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-anthropic-disable-reenable-key-07-assert-models", + "name": "07. model returns when re-enabled [anthropic-disable-reenable-key]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [anthropic-disable-reenable-key]\";", + "function assertNow() {", + " var providerName = 'catwiring-anthropic-disable-reenable-key-' + pm.variables.get('run_id');", + " var expectSubset = [\"claude-sonnet-4-5\"];", + " var expectSuperset = [];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"07. model returns when re-enabled [anthropic-disable-reenable-key]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"07. model returns when re-enabled [anthropic-disable-reenable-key]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-anthropic-disable-reenable-key-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-anthropic-disable-reenable-key-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-anthropic-disable-reenable-key-cleanup", + "name": "cleanup: delete provider [anthropic-disable-reenable-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [anthropic-disable-reenable-key]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-disable-reenable-key-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-disable-reenable-key-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Deleting one key leaves sibling models intact (anthropic)", + "description": "With two keys gating distinct models, deleting one removes only its models; the sibling's survive.", + "item": [ + { + "id": "catwiring-anthropic-delete-key-sibling-survives-01-add-provider", + "name": "01. add provider [anthropic-delete-key-sibling-survives]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [anthropic-delete-key-sibling-survives]\";", + "pm.test(\"01. add provider [anthropic-delete-key-sibling-survives]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-anthropic-delete-key-sibling-survives-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"anthropic\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-anthropic-delete-key-sibling-survives-02-add-key", + "name": "02. add key k1 [anthropic-delete-key-sibling-survives]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [anthropic-delete-key-sibling-survives]\";", + "pm.test(\"02. add key k1 [anthropic-delete-key-sibling-survives]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-delete-key-sibling-survives-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-delete-key-sibling-survives-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"anthropic-delete-key-sibling-survives-k1-{{run_id}}\",\"name\":\"catwiring-mc-anthropic-delete-key-sibling-survives-k1-{{run_id}}\",\"value\":\"env.ANTHROPIC_API_KEY\",\"models\":[\"claude-sonnet-4-5\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-anthropic-delete-key-sibling-survives-03-add-key", + "name": "03. add key k2 [anthropic-delete-key-sibling-survives]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [anthropic-delete-key-sibling-survives]\";", + "pm.test(\"03. add key k2 [anthropic-delete-key-sibling-survives]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-delete-key-sibling-survives-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-delete-key-sibling-survives-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"anthropic-delete-key-sibling-survives-k2-{{run_id}}\",\"name\":\"catwiring-mc-anthropic-delete-key-sibling-survives-k2-{{run_id}}\",\"value\":\"env.ANTHROPIC_API_KEY\",\"models\":[\"claude-haiku-4-5\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-anthropic-delete-key-sibling-survives-04-assert-models", + "name": "04. both keys' models present [anthropic-delete-key-sibling-survives]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [anthropic-delete-key-sibling-survives]\";", + "function assertNow() {", + " var providerName = 'catwiring-anthropic-delete-key-sibling-survives-' + pm.variables.get('run_id');", + " var expectSubset = [];", + " var expectSuperset = [\"claude-sonnet-4-5\",\"claude-haiku-4-5\"];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. both keys' models present [anthropic-delete-key-sibling-survives]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. both keys' models present [anthropic-delete-key-sibling-survives]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-anthropic-delete-key-sibling-survives-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-anthropic-delete-key-sibling-survives-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "id": "catwiring-anthropic-delete-key-sibling-survives-05-delete-key", + "name": "05. delete key k1 [anthropic-delete-key-sibling-survives]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,204];", + "var cleanupReq = \"cleanup: delete provider [anthropic-delete-key-sibling-survives]\";", + "pm.test(\"05. delete key k1 [anthropic-delete-key-sibling-survives]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-delete-key-sibling-survives-{{run_id}}/keys/anthropic-delete-key-sibling-survives-k1-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-delete-key-sibling-survives-{{run_id}}", + "keys", + "anthropic-delete-key-sibling-survives-k1-{{run_id}}" + ] + } + } + }, + { + "id": "catwiring-anthropic-delete-key-sibling-survives-06-assert-models", + "name": "06. sibling model survives delete [anthropic-delete-key-sibling-survives]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [anthropic-delete-key-sibling-survives]\";", + "function assertNow() {", + " var providerName = 'catwiring-anthropic-delete-key-sibling-survives-' + pm.variables.get('run_id');", + " var expectSubset = [\"claude-haiku-4-5\"];", + " var expectSuperset = [];", + " var expectAbsent = [\"claude-sonnet-4-5\"];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"06. sibling model survives delete [anthropic-delete-key-sibling-survives]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"06. sibling model survives delete [anthropic-delete-key-sibling-survives]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-anthropic-delete-key-sibling-survives-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-anthropic-delete-key-sibling-survives-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-anthropic-delete-key-sibling-survives-cleanup", + "name": "cleanup: delete provider [anthropic-delete-key-sibling-survives]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [anthropic-delete-key-sibling-survives]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-delete-key-sibling-survives-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-delete-key-sibling-survives-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Deleting provider removes it from the catalog (anthropic)", + "description": "Removing the provider drops all of its models from the catalog read endpoints.", + "item": [ + { + "id": "catwiring-anthropic-delete-provider-01-add-provider", + "name": "01. add provider [anthropic-delete-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [anthropic-delete-provider]\";", + "pm.test(\"01. add provider [anthropic-delete-provider]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-anthropic-delete-provider-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"anthropic\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-anthropic-delete-provider-02-add-key", + "name": "02. add key k1 [anthropic-delete-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [anthropic-delete-provider]\";", + "pm.test(\"02. add key k1 [anthropic-delete-provider]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-delete-provider-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-delete-provider-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"anthropic-delete-provider-k1-{{run_id}}\",\"name\":\"catwiring-mc-anthropic-delete-provider-k1-{{run_id}}\",\"value\":\"env.ANTHROPIC_API_KEY\",\"models\":[\"claude-sonnet-4-5\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-anthropic-delete-provider-03-assert-models", + "name": "03. model present before delete [anthropic-delete-provider]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [anthropic-delete-provider]\";", + "function assertNow() {", + " var providerName = 'catwiring-anthropic-delete-provider-' + pm.variables.get('run_id');", + " var expectSubset = [\"claude-sonnet-4-5\"];", + " var expectSuperset = [];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. model present before delete [anthropic-delete-provider]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. model present before delete [anthropic-delete-provider]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-anthropic-delete-provider-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-anthropic-delete-provider-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "id": "catwiring-anthropic-delete-provider-04-delete-provider", + "name": "04. delete provider [anthropic-delete-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,204];", + "var cleanupReq = \"cleanup: delete provider [anthropic-delete-provider]\";", + "pm.test(\"04. delete provider [anthropic-delete-provider]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-delete-provider-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-delete-provider-{{run_id}}" + ] + } + } + }, + { + "id": "catwiring-anthropic-delete-provider-05-assert-models", + "name": "05. no models after provider delete [anthropic-delete-provider]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 1000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [anthropic-delete-provider]\";", + "function assertNow() {", + " var providerName = 'catwiring-anthropic-delete-provider-' + pm.variables.get('run_id');", + " var expectSubset = [];", + " var expectSuperset = [];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = true;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. no models after provider delete [anthropic-delete-provider]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. no models after provider delete [anthropic-delete-provider]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-anthropic-delete-provider-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-anthropic-delete-provider-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-anthropic-delete-provider-cleanup", + "name": "cleanup: delete provider [anthropic-delete-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [anthropic-delete-provider]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-delete-provider-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-delete-provider-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Alias resolves to underlying model at inference (anthropic)", + "description": "A key alias routes an inference request to the underlying wire model rather than being rejected.", + "item": [ + { + "id": "catwiring-anthropic-alias-resolution-01-add-provider", + "name": "01. add provider [anthropic-alias-resolution]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [anthropic-alias-resolution]\";", + "pm.test(\"01. add provider [anthropic-alias-resolution]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-anthropic-alias-resolution-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"anthropic\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-anthropic-alias-resolution-02-add-key", + "name": "02. add key k1 [anthropic-alias-resolution]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [anthropic-alias-resolution]\";", + "pm.test(\"02. add key k1 [anthropic-alias-resolution]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-alias-resolution-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-alias-resolution-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"anthropic-alias-resolution-k1-{{run_id}}\",\"name\":\"catwiring-mc-anthropic-alias-resolution-k1-{{run_id}}\",\"value\":\"env.ANTHROPIC_API_KEY\",\"models\":[\"catwiring-alias-anthropic-{{run_id}}\",\"claude-haiku-4-5\"],\"enabled\":true,\"aliases\":{\"catwiring-alias-anthropic-{{run_id}}\":\"claude-haiku-4-5\"}}" + } + } + }, + { + "id": "catwiring-anthropic-alias-resolution-03-assert-models", + "name": "03. aliased key model present [anthropic-alias-resolution]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [anthropic-alias-resolution]\";", + "function assertNow() {", + " var providerName = 'catwiring-anthropic-alias-resolution-' + pm.variables.get('run_id');", + " var expectSubset = [\"claude-haiku-4-5\"];", + " var expectSuperset = [];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. aliased key model present [anthropic-alias-resolution]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. aliased key model present [anthropic-alias-resolution]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-anthropic-alias-resolution-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-anthropic-alias-resolution-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "id": "catwiring-anthropic-alias-resolution-04-assert-inference", + "name": "04. alias resolves to underlying model [anthropic-alias-resolution]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [anthropic-alias-resolution]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('inference status ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " if (!body.choices || body.choices.length === 0) { throw new Error('inference returned no choices'); }", + " var used = body.model || '';", + " if (used.indexOf(\"claude-haiku-4-5\") < 0) { throw new Error('expected resolved model claude-haiku-4-5 in response.model=' + used); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. alias resolves to underlying model [anthropic-alias-resolution]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. alias resolves to underlying model [anthropic-alias-resolution]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-anthropic-alias-resolution-{{run_id}}/catwiring-alias-anthropic-{{run_id}}\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-anthropic-alias-resolution-cleanup", + "name": "cleanup: delete provider [anthropic-alias-resolution]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [anthropic-alias-resolution]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-alias-resolution-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-alias-resolution-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Blacklisting a model re-gates a wildcard catalog (anthropic)", + "description": "A wildcard key surfaces the upstream's live model list; adding one of those models to the key's blacklist drops it from the catalog while the rest of the list stays. The target model is captured from the live list at run time because some upstreams only report dated ids.", + "item": [ + { + "id": "catwiring-anthropic-blacklist-regate-01-add-provider", + "name": "01. add provider [anthropic-blacklist-regate]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [anthropic-blacklist-regate]\";", + "pm.test(\"01. add provider [anthropic-blacklist-regate]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-anthropic-blacklist-regate-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"anthropic\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-anthropic-blacklist-regate-02-add-key", + "name": "02. add key k1 [anthropic-blacklist-regate]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [anthropic-blacklist-regate]\";", + "pm.test(\"02. add key k1 [anthropic-blacklist-regate]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-blacklist-regate-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-blacklist-regate-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"anthropic-blacklist-regate-k1-{{run_id}}\",\"name\":\"catwiring-mc-anthropic-blacklist-regate-k1-{{run_id}}\",\"value\":\"env.ANTHROPIC_API_KEY\",\"models\":[\"*\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-anthropic-blacklist-regate-03-capture-model", + "name": "03. wildcard catalog serves live models [anthropic-blacklist-regate]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [anthropic-blacklist-regate]\";", + "function assertNow() {", + " var providerName = 'catwiring-anthropic-blacklist-regate-' + pm.variables.get('run_id');", + " var prefix = \"claude-sonnet-4-5\";", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " var hit = null;", + " for (var i = 0; i < names.length; i++) {", + " if (names[i].indexOf(prefix) === 0) { hit = names[i]; break; }", + " }", + " if (!hit) { throw new Error('no live model with prefix ' + prefix + ' in ' + JSON.stringify(names.slice(0, 20))); }", + " pm.collectionVariables.set(\"bl_target_anthropic\", hit);", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. wildcard catalog serves live models [anthropic-blacklist-regate]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. wildcard catalog serves live models [anthropic-blacklist-regate]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-anthropic-blacklist-regate-{{run_id}}&limit=2000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-anthropic-blacklist-regate-{{run_id}}" + }, + { + "key": "limit", + "value": "2000" + } + ] + } + } + }, + { + "id": "catwiring-anthropic-blacklist-regate-04-update-key", + "name": "04. update key k1 [anthropic-blacklist-regate]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200];", + "var cleanupReq = \"cleanup: delete provider [anthropic-blacklist-regate]\";", + "pm.test(\"04. update key k1 [anthropic-blacklist-regate]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "PUT", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-blacklist-regate-{{run_id}}/keys/anthropic-blacklist-regate-k1-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-blacklist-regate-{{run_id}}", + "keys", + "anthropic-blacklist-regate-k1-{{run_id}}" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"anthropic-blacklist-regate-k1-{{run_id}}\",\"name\":\"catwiring-mc-anthropic-blacklist-regate-k1-{{run_id}}\",\"value\":\"env.ANTHROPIC_API_KEY\",\"models\":[\"*\"],\"enabled\":true,\"blacklisted_models\":[\"{{bl_target_anthropic}}\"]}" + } + } + }, + { + "id": "catwiring-anthropic-blacklist-regate-05-assert-models", + "name": "05. blacklisted model gated out, catalog still populated [anthropic-blacklist-regate]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [anthropic-blacklist-regate]\";", + "function assertNow() {", + " var providerName = 'catwiring-anthropic-blacklist-regate-' + pm.variables.get('run_id');", + " var expectSubset = [];", + " var expectSuperset = [];", + " var expectAbsent = [];", + " var expectAbsentVars = [\"bl_target_anthropic\"];", + " var expectEmpty = false;", + " var expectNonEmpty = true;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. blacklisted model gated out, catalog still populated [anthropic-blacklist-regate]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. blacklisted model gated out, catalog still populated [anthropic-blacklist-regate]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-anthropic-blacklist-regate-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-anthropic-blacklist-regate-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-anthropic-blacklist-regate-cleanup", + "name": "cleanup: delete provider [anthropic-blacklist-regate]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [anthropic-blacklist-regate]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-blacklist-regate-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-blacklist-regate-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Catalog aggregates the union across enabled keys (anthropic)", + "description": "Two keys gating distinct models both contribute: the catalog lists the union of their allow-lists.", + "item": [ + { + "id": "catwiring-anthropic-multi-key-union-01-add-provider", + "name": "01. add provider [anthropic-multi-key-union]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [anthropic-multi-key-union]\";", + "pm.test(\"01. add provider [anthropic-multi-key-union]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-anthropic-multi-key-union-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"anthropic\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-anthropic-multi-key-union-02-add-key", + "name": "02. add key k1 [anthropic-multi-key-union]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [anthropic-multi-key-union]\";", + "pm.test(\"02. add key k1 [anthropic-multi-key-union]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-multi-key-union-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-multi-key-union-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"anthropic-multi-key-union-k1-{{run_id}}\",\"name\":\"catwiring-mc-anthropic-multi-key-union-k1-{{run_id}}\",\"value\":\"env.ANTHROPIC_API_KEY\",\"models\":[\"claude-sonnet-4-5\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-anthropic-multi-key-union-03-add-key", + "name": "03. add key k2 [anthropic-multi-key-union]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [anthropic-multi-key-union]\";", + "pm.test(\"03. add key k2 [anthropic-multi-key-union]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-multi-key-union-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-multi-key-union-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"anthropic-multi-key-union-k2-{{run_id}}\",\"name\":\"catwiring-mc-anthropic-multi-key-union-k2-{{run_id}}\",\"value\":\"env.ANTHROPIC_API_KEY\",\"models\":[\"claude-haiku-4-5\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-anthropic-multi-key-union-04-assert-models", + "name": "04. catalog lists union of both keys' models [anthropic-multi-key-union]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [anthropic-multi-key-union]\";", + "function assertNow() {", + " var providerName = 'catwiring-anthropic-multi-key-union-' + pm.variables.get('run_id');", + " var expectSubset = [];", + " var expectSuperset = [\"claude-sonnet-4-5\",\"claude-haiku-4-5\"];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. catalog lists union of both keys' models [anthropic-multi-key-union]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. catalog lists union of both keys' models [anthropic-multi-key-union]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-anthropic-multi-key-union-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-anthropic-multi-key-union-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-anthropic-multi-key-union-cleanup", + "name": "cleanup: delete provider [anthropic-multi-key-union]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [anthropic-multi-key-union]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-multi-key-union-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-multi-key-union-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Model details endpoint respects the key gate (anthropic)", + "description": "/api/models/details lists only the models the provider's keys allow, like the plain list endpoint.", + "item": [ + { + "id": "catwiring-anthropic-model-details-gating-01-add-provider", + "name": "01. add provider [anthropic-model-details-gating]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [anthropic-model-details-gating]\";", + "pm.test(\"01. add provider [anthropic-model-details-gating]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-anthropic-model-details-gating-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"anthropic\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-anthropic-model-details-gating-02-add-key", + "name": "02. add key k1 [anthropic-model-details-gating]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [anthropic-model-details-gating]\";", + "pm.test(\"02. add key k1 [anthropic-model-details-gating]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-model-details-gating-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-model-details-gating-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"anthropic-model-details-gating-k1-{{run_id}}\",\"name\":\"catwiring-mc-anthropic-model-details-gating-k1-{{run_id}}\",\"value\":\"env.ANTHROPIC_API_KEY\",\"models\":[\"claude-sonnet-4-5\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-anthropic-model-details-gating-03-assert-model-details", + "name": "03. details list gated to key models [anthropic-model-details-gating]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [anthropic-model-details-gating]\";", + "function assertNow() {", + " var providerName = 'catwiring-anthropic-model-details-gating-' + pm.variables.get('run_id');", + " var expectSubset = [\"claude-sonnet-4-5\"];", + " var expectSuperset = [];", + " var expectAbsent = [\"claude-haiku-4-5\"];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. details list gated to key models [anthropic-model-details-gating]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. details list gated to key models [anthropic-model-details-gating]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models/details?provider=catwiring-anthropic-model-details-gating-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models", + "details" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-anthropic-model-details-gating-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-anthropic-model-details-gating-cleanup", + "name": "cleanup: delete provider [anthropic-model-details-gating]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [anthropic-model-details-gating]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-model-details-gating-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-model-details-gating-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Base model list is unaffected by key changes (anthropic)", + "description": "/api/models/base reflects the datasheet's distinct base names; removing a model from a key's allow-list must not remove its base name from that list.", + "item": [ + { + "id": "catwiring-anthropic-base-models-stable-01-add-provider", + "name": "01. add provider [anthropic-base-models-stable]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [anthropic-base-models-stable]\";", + "pm.test(\"01. add provider [anthropic-base-models-stable]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-anthropic-base-models-stable-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"anthropic\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-anthropic-base-models-stable-02-add-key", + "name": "02. add key k1 [anthropic-base-models-stable]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [anthropic-base-models-stable]\";", + "pm.test(\"02. add key k1 [anthropic-base-models-stable]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-base-models-stable-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-base-models-stable-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"anthropic-base-models-stable-k1-{{run_id}}\",\"name\":\"catwiring-mc-anthropic-base-models-stable-k1-{{run_id}}\",\"value\":\"env.ANTHROPIC_API_KEY\",\"models\":[\"claude-sonnet-4-5\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-anthropic-base-models-stable-03-assert-base-models", + "name": "03. base name listed while key allows it [anthropic-base-models-stable]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 1000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [anthropic-base-models-stable]\";", + "function assertNow() {", + " var target = \"claude-sonnet-4-5\";", + " if (pm.response.code !== 200) { throw new Error('list base models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = body.models || [];", + " if (names.indexOf(target) < 0) { throw new Error('expected base model ' + target + ' in ' + JSON.stringify(names)); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. base name listed while key allows it [anthropic-base-models-stable]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. base name listed while key allows it [anthropic-base-models-stable]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models/base?query=claude-sonnet-4-5&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models", + "base" + ], + "query": [ + { + "key": "query", + "value": "claude-sonnet-4-5" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "id": "catwiring-anthropic-base-models-stable-04-update-key", + "name": "04. update key k1 [anthropic-base-models-stable]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200];", + "var cleanupReq = \"cleanup: delete provider [anthropic-base-models-stable]\";", + "pm.test(\"04. update key k1 [anthropic-base-models-stable]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "PUT", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-base-models-stable-{{run_id}}/keys/anthropic-base-models-stable-k1-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-base-models-stable-{{run_id}}", + "keys", + "anthropic-base-models-stable-k1-{{run_id}}" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"anthropic-base-models-stable-k1-{{run_id}}\",\"name\":\"catwiring-mc-anthropic-base-models-stable-k1-{{run_id}}\",\"value\":\"env.ANTHROPIC_API_KEY\",\"models\":[\"claude-haiku-4-5\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-anthropic-base-models-stable-05-assert-base-models", + "name": "05. base name still listed after key change [anthropic-base-models-stable]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 1000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [anthropic-base-models-stable]\";", + "function assertNow() {", + " var target = \"claude-sonnet-4-5\";", + " if (pm.response.code !== 200) { throw new Error('list base models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = body.models || [];", + " if (names.indexOf(target) < 0) { throw new Error('expected base model ' + target + ' in ' + JSON.stringify(names)); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. base name still listed after key change [anthropic-base-models-stable]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. base name still listed after key change [anthropic-base-models-stable]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models/base?query=claude-sonnet-4-5&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models", + "base" + ], + "query": [ + { + "key": "query", + "value": "claude-sonnet-4-5" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-anthropic-base-models-stable-cleanup", + "name": "cleanup: delete provider [anthropic-base-models-stable]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [anthropic-base-models-stable]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-anthropic-base-models-stable-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-anthropic-base-models-stable-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Add provider and key surfaces models (gemini)", + "description": "A fresh provider plus one gated key surfaces that key's allowed model in the catalog.", + "item": [ + { + "id": "catwiring-gemini-add-provider-and-key-01-add-provider", + "name": "01. add provider [gemini-add-provider-and-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [gemini-add-provider-and-key]\";", + "pm.test(\"01. add provider [gemini-add-provider-and-key]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-gemini-add-provider-and-key-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"gemini\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-gemini-add-provider-and-key-02-add-key", + "name": "02. add key k1 [gemini-add-provider-and-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [gemini-add-provider-and-key]\";", + "pm.test(\"02. add key k1 [gemini-add-provider-and-key]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-add-provider-and-key-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-add-provider-and-key-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"gemini-add-provider-and-key-k1-{{run_id}}\",\"name\":\"catwiring-mc-gemini-add-provider-and-key-k1-{{run_id}}\",\"value\":\"env.GEMINI_API_KEY\",\"models\":[\"gemini-2.5-flash\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-gemini-add-provider-and-key-03-assert-models", + "name": "03. key model appears in catalog [gemini-add-provider-and-key]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [gemini-add-provider-and-key]\";", + "function assertNow() {", + " var providerName = 'catwiring-gemini-add-provider-and-key-' + pm.variables.get('run_id');", + " var expectSubset = [\"gemini-2.5-flash\"];", + " var expectSuperset = [];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. key model appears in catalog [gemini-add-provider-and-key]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. key model appears in catalog [gemini-add-provider-and-key]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-gemini-add-provider-and-key-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-gemini-add-provider-and-key-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-gemini-add-provider-and-key-cleanup", + "name": "cleanup: delete provider [gemini-add-provider-and-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [gemini-add-provider-and-key]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-add-provider-and-key-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-add-provider-and-key-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Updating key model set re-gates the catalog (gemini)", + "description": "Changing a key's allow-list invalidates the stale live entry and re-gates the surfaced models.", + "item": [ + { + "id": "catwiring-gemini-update-key-models-01-add-provider", + "name": "01. add provider [gemini-update-key-models]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [gemini-update-key-models]\";", + "pm.test(\"01. add provider [gemini-update-key-models]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-gemini-update-key-models-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"gemini\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-gemini-update-key-models-02-add-key", + "name": "02. add key k1 [gemini-update-key-models]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [gemini-update-key-models]\";", + "pm.test(\"02. add key k1 [gemini-update-key-models]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-update-key-models-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-update-key-models-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"gemini-update-key-models-k1-{{run_id}}\",\"name\":\"catwiring-mc-gemini-update-key-models-k1-{{run_id}}\",\"value\":\"env.GEMINI_API_KEY\",\"models\":[\"gemini-2.5-flash\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-gemini-update-key-models-03-assert-models", + "name": "03. initial model gated in [gemini-update-key-models]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [gemini-update-key-models]\";", + "function assertNow() {", + " var providerName = 'catwiring-gemini-update-key-models-' + pm.variables.get('run_id');", + " var expectSubset = [\"gemini-2.5-flash\"];", + " var expectSuperset = [];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. initial model gated in [gemini-update-key-models]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. initial model gated in [gemini-update-key-models]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-gemini-update-key-models-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-gemini-update-key-models-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "id": "catwiring-gemini-update-key-models-04-update-key", + "name": "04. update key k1 [gemini-update-key-models]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200];", + "var cleanupReq = \"cleanup: delete provider [gemini-update-key-models]\";", + "pm.test(\"04. update key k1 [gemini-update-key-models]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "PUT", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-update-key-models-{{run_id}}/keys/gemini-update-key-models-k1-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-update-key-models-{{run_id}}", + "keys", + "gemini-update-key-models-k1-{{run_id}}" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"gemini-update-key-models-k1-{{run_id}}\",\"name\":\"catwiring-mc-gemini-update-key-models-k1-{{run_id}}\",\"value\":\"env.GEMINI_API_KEY\",\"models\":[\"gemini-2.5-flash-lite\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-gemini-update-key-models-05-assert-models", + "name": "05. new model gated in, old gated out [gemini-update-key-models]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [gemini-update-key-models]\";", + "function assertNow() {", + " var providerName = 'catwiring-gemini-update-key-models-' + pm.variables.get('run_id');", + " var expectSubset = [\"gemini-2.5-flash-lite\"];", + " var expectSuperset = [];", + " var expectAbsent = [\"gemini-2.5-flash\"];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. new model gated in, old gated out [gemini-update-key-models]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. new model gated in, old gated out [gemini-update-key-models]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-gemini-update-key-models-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-gemini-update-key-models-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-gemini-update-key-models-cleanup", + "name": "cleanup: delete provider [gemini-update-key-models]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [gemini-update-key-models]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-update-key-models-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-update-key-models-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Disabled key drops models, re-enable restores (gemini)", + "description": "A disabled key is skipped during aggregation; re-enabling it brings its models back.", + "item": [ + { + "id": "catwiring-gemini-disable-reenable-key-01-add-provider", + "name": "01. add provider [gemini-disable-reenable-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [gemini-disable-reenable-key]\";", + "pm.test(\"01. add provider [gemini-disable-reenable-key]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-gemini-disable-reenable-key-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"gemini\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-gemini-disable-reenable-key-02-add-key", + "name": "02. add key k1 [gemini-disable-reenable-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [gemini-disable-reenable-key]\";", + "pm.test(\"02. add key k1 [gemini-disable-reenable-key]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-disable-reenable-key-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-disable-reenable-key-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"gemini-disable-reenable-key-k1-{{run_id}}\",\"name\":\"catwiring-mc-gemini-disable-reenable-key-k1-{{run_id}}\",\"value\":\"env.GEMINI_API_KEY\",\"models\":[\"gemini-2.5-flash\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-gemini-disable-reenable-key-03-assert-models", + "name": "03. model present while enabled [gemini-disable-reenable-key]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [gemini-disable-reenable-key]\";", + "function assertNow() {", + " var providerName = 'catwiring-gemini-disable-reenable-key-' + pm.variables.get('run_id');", + " var expectSubset = [\"gemini-2.5-flash\"];", + " var expectSuperset = [];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. model present while enabled [gemini-disable-reenable-key]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. model present while enabled [gemini-disable-reenable-key]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-gemini-disable-reenable-key-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-gemini-disable-reenable-key-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "id": "catwiring-gemini-disable-reenable-key-04-update-key", + "name": "04. update key k1 [gemini-disable-reenable-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200];", + "var cleanupReq = \"cleanup: delete provider [gemini-disable-reenable-key]\";", + "pm.test(\"04. update key k1 [gemini-disable-reenable-key]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "PUT", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-disable-reenable-key-{{run_id}}/keys/gemini-disable-reenable-key-k1-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-disable-reenable-key-{{run_id}}", + "keys", + "gemini-disable-reenable-key-k1-{{run_id}}" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"gemini-disable-reenable-key-k1-{{run_id}}\",\"name\":\"catwiring-mc-gemini-disable-reenable-key-k1-{{run_id}}\",\"value\":\"env.GEMINI_API_KEY\",\"models\":[\"gemini-2.5-flash\"],\"enabled\":false}" + } + } + }, + { + "id": "catwiring-gemini-disable-reenable-key-05-assert-models", + "name": "05. model gone while disabled [gemini-disable-reenable-key]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [gemini-disable-reenable-key]\";", + "function assertNow() {", + " var providerName = 'catwiring-gemini-disable-reenable-key-' + pm.variables.get('run_id');", + " var expectSubset = [];", + " var expectSuperset = [];", + " var expectAbsent = [\"gemini-2.5-flash\"];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. model gone while disabled [gemini-disable-reenable-key]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. model gone while disabled [gemini-disable-reenable-key]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-gemini-disable-reenable-key-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-gemini-disable-reenable-key-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "id": "catwiring-gemini-disable-reenable-key-06-update-key", + "name": "06. update key k1 [gemini-disable-reenable-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200];", + "var cleanupReq = \"cleanup: delete provider [gemini-disable-reenable-key]\";", + "pm.test(\"06. update key k1 [gemini-disable-reenable-key]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "PUT", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-disable-reenable-key-{{run_id}}/keys/gemini-disable-reenable-key-k1-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-disable-reenable-key-{{run_id}}", + "keys", + "gemini-disable-reenable-key-k1-{{run_id}}" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"gemini-disable-reenable-key-k1-{{run_id}}\",\"name\":\"catwiring-mc-gemini-disable-reenable-key-k1-{{run_id}}\",\"value\":\"env.GEMINI_API_KEY\",\"models\":[\"gemini-2.5-flash\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-gemini-disable-reenable-key-07-assert-models", + "name": "07. model returns when re-enabled [gemini-disable-reenable-key]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [gemini-disable-reenable-key]\";", + "function assertNow() {", + " var providerName = 'catwiring-gemini-disable-reenable-key-' + pm.variables.get('run_id');", + " var expectSubset = [\"gemini-2.5-flash\"];", + " var expectSuperset = [];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"07. model returns when re-enabled [gemini-disable-reenable-key]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"07. model returns when re-enabled [gemini-disable-reenable-key]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-gemini-disable-reenable-key-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-gemini-disable-reenable-key-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-gemini-disable-reenable-key-cleanup", + "name": "cleanup: delete provider [gemini-disable-reenable-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [gemini-disable-reenable-key]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-disable-reenable-key-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-disable-reenable-key-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Deleting one key leaves sibling models intact (gemini)", + "description": "With two keys gating distinct models, deleting one removes only its models; the sibling's survive.", + "item": [ + { + "id": "catwiring-gemini-delete-key-sibling-survives-01-add-provider", + "name": "01. add provider [gemini-delete-key-sibling-survives]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [gemini-delete-key-sibling-survives]\";", + "pm.test(\"01. add provider [gemini-delete-key-sibling-survives]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-gemini-delete-key-sibling-survives-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"gemini\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-gemini-delete-key-sibling-survives-02-add-key", + "name": "02. add key k1 [gemini-delete-key-sibling-survives]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [gemini-delete-key-sibling-survives]\";", + "pm.test(\"02. add key k1 [gemini-delete-key-sibling-survives]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-delete-key-sibling-survives-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-delete-key-sibling-survives-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"gemini-delete-key-sibling-survives-k1-{{run_id}}\",\"name\":\"catwiring-mc-gemini-delete-key-sibling-survives-k1-{{run_id}}\",\"value\":\"env.GEMINI_API_KEY\",\"models\":[\"gemini-2.5-flash\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-gemini-delete-key-sibling-survives-03-add-key", + "name": "03. add key k2 [gemini-delete-key-sibling-survives]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [gemini-delete-key-sibling-survives]\";", + "pm.test(\"03. add key k2 [gemini-delete-key-sibling-survives]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-delete-key-sibling-survives-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-delete-key-sibling-survives-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"gemini-delete-key-sibling-survives-k2-{{run_id}}\",\"name\":\"catwiring-mc-gemini-delete-key-sibling-survives-k2-{{run_id}}\",\"value\":\"env.GEMINI_API_KEY\",\"models\":[\"gemini-2.5-flash-lite\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-gemini-delete-key-sibling-survives-04-assert-models", + "name": "04. both keys' models present [gemini-delete-key-sibling-survives]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [gemini-delete-key-sibling-survives]\";", + "function assertNow() {", + " var providerName = 'catwiring-gemini-delete-key-sibling-survives-' + pm.variables.get('run_id');", + " var expectSubset = [];", + " var expectSuperset = [\"gemini-2.5-flash\",\"gemini-2.5-flash-lite\"];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. both keys' models present [gemini-delete-key-sibling-survives]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. both keys' models present [gemini-delete-key-sibling-survives]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-gemini-delete-key-sibling-survives-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-gemini-delete-key-sibling-survives-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "id": "catwiring-gemini-delete-key-sibling-survives-05-delete-key", + "name": "05. delete key k1 [gemini-delete-key-sibling-survives]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,204];", + "var cleanupReq = \"cleanup: delete provider [gemini-delete-key-sibling-survives]\";", + "pm.test(\"05. delete key k1 [gemini-delete-key-sibling-survives]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-delete-key-sibling-survives-{{run_id}}/keys/gemini-delete-key-sibling-survives-k1-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-delete-key-sibling-survives-{{run_id}}", + "keys", + "gemini-delete-key-sibling-survives-k1-{{run_id}}" + ] + } + } + }, + { + "id": "catwiring-gemini-delete-key-sibling-survives-06-assert-models", + "name": "06. sibling model survives delete [gemini-delete-key-sibling-survives]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [gemini-delete-key-sibling-survives]\";", + "function assertNow() {", + " var providerName = 'catwiring-gemini-delete-key-sibling-survives-' + pm.variables.get('run_id');", + " var expectSubset = [\"gemini-2.5-flash-lite\"];", + " var expectSuperset = [];", + " var expectAbsent = [\"gemini-2.5-flash\"];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"06. sibling model survives delete [gemini-delete-key-sibling-survives]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"06. sibling model survives delete [gemini-delete-key-sibling-survives]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-gemini-delete-key-sibling-survives-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-gemini-delete-key-sibling-survives-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-gemini-delete-key-sibling-survives-cleanup", + "name": "cleanup: delete provider [gemini-delete-key-sibling-survives]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [gemini-delete-key-sibling-survives]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-delete-key-sibling-survives-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-delete-key-sibling-survives-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Deleting provider removes it from the catalog (gemini)", + "description": "Removing the provider drops all of its models from the catalog read endpoints.", + "item": [ + { + "id": "catwiring-gemini-delete-provider-01-add-provider", + "name": "01. add provider [gemini-delete-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [gemini-delete-provider]\";", + "pm.test(\"01. add provider [gemini-delete-provider]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-gemini-delete-provider-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"gemini\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-gemini-delete-provider-02-add-key", + "name": "02. add key k1 [gemini-delete-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [gemini-delete-provider]\";", + "pm.test(\"02. add key k1 [gemini-delete-provider]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-delete-provider-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-delete-provider-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"gemini-delete-provider-k1-{{run_id}}\",\"name\":\"catwiring-mc-gemini-delete-provider-k1-{{run_id}}\",\"value\":\"env.GEMINI_API_KEY\",\"models\":[\"gemini-2.5-flash\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-gemini-delete-provider-03-assert-models", + "name": "03. model present before delete [gemini-delete-provider]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [gemini-delete-provider]\";", + "function assertNow() {", + " var providerName = 'catwiring-gemini-delete-provider-' + pm.variables.get('run_id');", + " var expectSubset = [\"gemini-2.5-flash\"];", + " var expectSuperset = [];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. model present before delete [gemini-delete-provider]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. model present before delete [gemini-delete-provider]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-gemini-delete-provider-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-gemini-delete-provider-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "id": "catwiring-gemini-delete-provider-04-delete-provider", + "name": "04. delete provider [gemini-delete-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,204];", + "var cleanupReq = \"cleanup: delete provider [gemini-delete-provider]\";", + "pm.test(\"04. delete provider [gemini-delete-provider]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-delete-provider-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-delete-provider-{{run_id}}" + ] + } + } + }, + { + "id": "catwiring-gemini-delete-provider-05-assert-models", + "name": "05. no models after provider delete [gemini-delete-provider]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 1000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [gemini-delete-provider]\";", + "function assertNow() {", + " var providerName = 'catwiring-gemini-delete-provider-' + pm.variables.get('run_id');", + " var expectSubset = [];", + " var expectSuperset = [];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = true;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. no models after provider delete [gemini-delete-provider]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. no models after provider delete [gemini-delete-provider]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-gemini-delete-provider-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-gemini-delete-provider-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-gemini-delete-provider-cleanup", + "name": "cleanup: delete provider [gemini-delete-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [gemini-delete-provider]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-delete-provider-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-delete-provider-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Alias resolves to underlying model at inference (gemini)", + "description": "A key alias routes an inference request to the underlying wire model rather than being rejected.", + "item": [ + { + "id": "catwiring-gemini-alias-resolution-01-add-provider", + "name": "01. add provider [gemini-alias-resolution]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [gemini-alias-resolution]\";", + "pm.test(\"01. add provider [gemini-alias-resolution]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-gemini-alias-resolution-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"gemini\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-gemini-alias-resolution-02-add-key", + "name": "02. add key k1 [gemini-alias-resolution]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [gemini-alias-resolution]\";", + "pm.test(\"02. add key k1 [gemini-alias-resolution]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-alias-resolution-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-alias-resolution-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"gemini-alias-resolution-k1-{{run_id}}\",\"name\":\"catwiring-mc-gemini-alias-resolution-k1-{{run_id}}\",\"value\":\"env.GEMINI_API_KEY\",\"models\":[\"catwiring-alias-gemini-{{run_id}}\",\"gemini-2.5-flash-lite\"],\"enabled\":true,\"aliases\":{\"catwiring-alias-gemini-{{run_id}}\":\"gemini-2.5-flash-lite\"}}" + } + } + }, + { + "id": "catwiring-gemini-alias-resolution-03-assert-models", + "name": "03. aliased key model present [gemini-alias-resolution]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [gemini-alias-resolution]\";", + "function assertNow() {", + " var providerName = 'catwiring-gemini-alias-resolution-' + pm.variables.get('run_id');", + " var expectSubset = [\"gemini-2.5-flash-lite\"];", + " var expectSuperset = [];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. aliased key model present [gemini-alias-resolution]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. aliased key model present [gemini-alias-resolution]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-gemini-alias-resolution-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-gemini-alias-resolution-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "id": "catwiring-gemini-alias-resolution-04-assert-inference", + "name": "04. alias resolves to underlying model [gemini-alias-resolution]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [gemini-alias-resolution]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('inference status ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " if (!body.choices || body.choices.length === 0) { throw new Error('inference returned no choices'); }", + " var used = body.model || '';", + " if (used.indexOf(\"gemini-2.5-flash-lite\") < 0) { throw new Error('expected resolved model gemini-2.5-flash-lite in response.model=' + used); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. alias resolves to underlying model [gemini-alias-resolution]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. alias resolves to underlying model [gemini-alias-resolution]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-gemini-alias-resolution-{{run_id}}/catwiring-alias-gemini-{{run_id}}\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-gemini-alias-resolution-cleanup", + "name": "cleanup: delete provider [gemini-alias-resolution]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [gemini-alias-resolution]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-alias-resolution-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-alias-resolution-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Blacklisting a model re-gates a wildcard catalog (gemini)", + "description": "A wildcard key surfaces the upstream's live model list; adding one of those models to the key's blacklist drops it from the catalog while the rest of the list stays. The target model is captured from the live list at run time because some upstreams only report dated ids.", + "item": [ + { + "id": "catwiring-gemini-blacklist-regate-01-add-provider", + "name": "01. add provider [gemini-blacklist-regate]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [gemini-blacklist-regate]\";", + "pm.test(\"01. add provider [gemini-blacklist-regate]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-gemini-blacklist-regate-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"gemini\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-gemini-blacklist-regate-02-add-key", + "name": "02. add key k1 [gemini-blacklist-regate]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [gemini-blacklist-regate]\";", + "pm.test(\"02. add key k1 [gemini-blacklist-regate]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-blacklist-regate-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-blacklist-regate-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"gemini-blacklist-regate-k1-{{run_id}}\",\"name\":\"catwiring-mc-gemini-blacklist-regate-k1-{{run_id}}\",\"value\":\"env.GEMINI_API_KEY\",\"models\":[\"*\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-gemini-blacklist-regate-03-capture-model", + "name": "03. wildcard catalog serves live models [gemini-blacklist-regate]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [gemini-blacklist-regate]\";", + "function assertNow() {", + " var providerName = 'catwiring-gemini-blacklist-regate-' + pm.variables.get('run_id');", + " var prefix = \"gemini-2.5-flash\";", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " var hit = null;", + " for (var i = 0; i < names.length; i++) {", + " if (names[i].indexOf(prefix) === 0) { hit = names[i]; break; }", + " }", + " if (!hit) { throw new Error('no live model with prefix ' + prefix + ' in ' + JSON.stringify(names.slice(0, 20))); }", + " pm.collectionVariables.set(\"bl_target_gemini\", hit);", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. wildcard catalog serves live models [gemini-blacklist-regate]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. wildcard catalog serves live models [gemini-blacklist-regate]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-gemini-blacklist-regate-{{run_id}}&limit=2000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-gemini-blacklist-regate-{{run_id}}" + }, + { + "key": "limit", + "value": "2000" + } + ] + } + } + }, + { + "id": "catwiring-gemini-blacklist-regate-04-update-key", + "name": "04. update key k1 [gemini-blacklist-regate]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200];", + "var cleanupReq = \"cleanup: delete provider [gemini-blacklist-regate]\";", + "pm.test(\"04. update key k1 [gemini-blacklist-regate]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "PUT", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-blacklist-regate-{{run_id}}/keys/gemini-blacklist-regate-k1-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-blacklist-regate-{{run_id}}", + "keys", + "gemini-blacklist-regate-k1-{{run_id}}" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"gemini-blacklist-regate-k1-{{run_id}}\",\"name\":\"catwiring-mc-gemini-blacklist-regate-k1-{{run_id}}\",\"value\":\"env.GEMINI_API_KEY\",\"models\":[\"*\"],\"enabled\":true,\"blacklisted_models\":[\"{{bl_target_gemini}}\"]}" + } + } + }, + { + "id": "catwiring-gemini-blacklist-regate-05-assert-models", + "name": "05. blacklisted model gated out, catalog still populated [gemini-blacklist-regate]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [gemini-blacklist-regate]\";", + "function assertNow() {", + " var providerName = 'catwiring-gemini-blacklist-regate-' + pm.variables.get('run_id');", + " var expectSubset = [];", + " var expectSuperset = [];", + " var expectAbsent = [];", + " var expectAbsentVars = [\"bl_target_gemini\"];", + " var expectEmpty = false;", + " var expectNonEmpty = true;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. blacklisted model gated out, catalog still populated [gemini-blacklist-regate]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. blacklisted model gated out, catalog still populated [gemini-blacklist-regate]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-gemini-blacklist-regate-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-gemini-blacklist-regate-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-gemini-blacklist-regate-cleanup", + "name": "cleanup: delete provider [gemini-blacklist-regate]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [gemini-blacklist-regate]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-blacklist-regate-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-blacklist-regate-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Catalog aggregates the union across enabled keys (gemini)", + "description": "Two keys gating distinct models both contribute: the catalog lists the union of their allow-lists.", + "item": [ + { + "id": "catwiring-gemini-multi-key-union-01-add-provider", + "name": "01. add provider [gemini-multi-key-union]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [gemini-multi-key-union]\";", + "pm.test(\"01. add provider [gemini-multi-key-union]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-gemini-multi-key-union-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"gemini\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-gemini-multi-key-union-02-add-key", + "name": "02. add key k1 [gemini-multi-key-union]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [gemini-multi-key-union]\";", + "pm.test(\"02. add key k1 [gemini-multi-key-union]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-multi-key-union-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-multi-key-union-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"gemini-multi-key-union-k1-{{run_id}}\",\"name\":\"catwiring-mc-gemini-multi-key-union-k1-{{run_id}}\",\"value\":\"env.GEMINI_API_KEY\",\"models\":[\"gemini-2.5-flash\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-gemini-multi-key-union-03-add-key", + "name": "03. add key k2 [gemini-multi-key-union]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [gemini-multi-key-union]\";", + "pm.test(\"03. add key k2 [gemini-multi-key-union]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-multi-key-union-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-multi-key-union-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"gemini-multi-key-union-k2-{{run_id}}\",\"name\":\"catwiring-mc-gemini-multi-key-union-k2-{{run_id}}\",\"value\":\"env.GEMINI_API_KEY\",\"models\":[\"gemini-2.5-flash-lite\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-gemini-multi-key-union-04-assert-models", + "name": "04. catalog lists union of both keys' models [gemini-multi-key-union]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [gemini-multi-key-union]\";", + "function assertNow() {", + " var providerName = 'catwiring-gemini-multi-key-union-' + pm.variables.get('run_id');", + " var expectSubset = [];", + " var expectSuperset = [\"gemini-2.5-flash\",\"gemini-2.5-flash-lite\"];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. catalog lists union of both keys' models [gemini-multi-key-union]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. catalog lists union of both keys' models [gemini-multi-key-union]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-gemini-multi-key-union-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-gemini-multi-key-union-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-gemini-multi-key-union-cleanup", + "name": "cleanup: delete provider [gemini-multi-key-union]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [gemini-multi-key-union]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-multi-key-union-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-multi-key-union-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Model details endpoint respects the key gate (gemini)", + "description": "/api/models/details lists only the models the provider's keys allow, like the plain list endpoint.", + "item": [ + { + "id": "catwiring-gemini-model-details-gating-01-add-provider", + "name": "01. add provider [gemini-model-details-gating]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [gemini-model-details-gating]\";", + "pm.test(\"01. add provider [gemini-model-details-gating]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-gemini-model-details-gating-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"gemini\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-gemini-model-details-gating-02-add-key", + "name": "02. add key k1 [gemini-model-details-gating]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [gemini-model-details-gating]\";", + "pm.test(\"02. add key k1 [gemini-model-details-gating]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-model-details-gating-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-model-details-gating-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"gemini-model-details-gating-k1-{{run_id}}\",\"name\":\"catwiring-mc-gemini-model-details-gating-k1-{{run_id}}\",\"value\":\"env.GEMINI_API_KEY\",\"models\":[\"gemini-2.5-flash\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-gemini-model-details-gating-03-assert-model-details", + "name": "03. details list gated to key models [gemini-model-details-gating]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [gemini-model-details-gating]\";", + "function assertNow() {", + " var providerName = 'catwiring-gemini-model-details-gating-' + pm.variables.get('run_id');", + " var expectSubset = [\"gemini-2.5-flash\"];", + " var expectSuperset = [];", + " var expectAbsent = [\"gemini-2.5-flash-lite\"];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. details list gated to key models [gemini-model-details-gating]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. details list gated to key models [gemini-model-details-gating]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models/details?provider=catwiring-gemini-model-details-gating-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models", + "details" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-gemini-model-details-gating-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-gemini-model-details-gating-cleanup", + "name": "cleanup: delete provider [gemini-model-details-gating]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [gemini-model-details-gating]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-model-details-gating-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-model-details-gating-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Base model list is unaffected by key changes (gemini)", + "description": "/api/models/base reflects the datasheet's distinct base names; removing a model from a key's allow-list must not remove its base name from that list.", + "item": [ + { + "id": "catwiring-gemini-base-models-stable-01-add-provider", + "name": "01. add provider [gemini-base-models-stable]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [gemini-base-models-stable]\";", + "pm.test(\"01. add provider [gemini-base-models-stable]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-gemini-base-models-stable-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"gemini\",\"is_key_less\":false}}" + } + } + }, + { + "id": "catwiring-gemini-base-models-stable-02-add-key", + "name": "02. add key k1 [gemini-base-models-stable]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [gemini-base-models-stable]\";", + "pm.test(\"02. add key k1 [gemini-base-models-stable]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-base-models-stable-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-base-models-stable-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"gemini-base-models-stable-k1-{{run_id}}\",\"name\":\"catwiring-mc-gemini-base-models-stable-k1-{{run_id}}\",\"value\":\"env.GEMINI_API_KEY\",\"models\":[\"gemini-2.5-flash\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-gemini-base-models-stable-03-assert-base-models", + "name": "03. base name listed while key allows it [gemini-base-models-stable]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 1000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [gemini-base-models-stable]\";", + "function assertNow() {", + " var target = \"gemini-2.5-flash\";", + " if (pm.response.code !== 200) { throw new Error('list base models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = body.models || [];", + " if (names.indexOf(target) < 0) { throw new Error('expected base model ' + target + ' in ' + JSON.stringify(names)); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. base name listed while key allows it [gemini-base-models-stable]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. base name listed while key allows it [gemini-base-models-stable]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models/base?query=gemini-2.5-flash&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models", + "base" + ], + "query": [ + { + "key": "query", + "value": "gemini-2.5-flash" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "id": "catwiring-gemini-base-models-stable-04-update-key", + "name": "04. update key k1 [gemini-base-models-stable]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200];", + "var cleanupReq = \"cleanup: delete provider [gemini-base-models-stable]\";", + "pm.test(\"04. update key k1 [gemini-base-models-stable]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "PUT", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-base-models-stable-{{run_id}}/keys/gemini-base-models-stable-k1-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-base-models-stable-{{run_id}}", + "keys", + "gemini-base-models-stable-k1-{{run_id}}" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"gemini-base-models-stable-k1-{{run_id}}\",\"name\":\"catwiring-mc-gemini-base-models-stable-k1-{{run_id}}\",\"value\":\"env.GEMINI_API_KEY\",\"models\":[\"gemini-2.5-flash-lite\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-gemini-base-models-stable-05-assert-base-models", + "name": "05. base name still listed after key change [gemini-base-models-stable]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 1000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [gemini-base-models-stable]\";", + "function assertNow() {", + " var target = \"gemini-2.5-flash\";", + " if (pm.response.code !== 200) { throw new Error('list base models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = body.models || [];", + " if (names.indexOf(target) < 0) { throw new Error('expected base model ' + target + ' in ' + JSON.stringify(names)); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. base name still listed after key change [gemini-base-models-stable]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. base name still listed after key change [gemini-base-models-stable]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models/base?query=gemini-2.5-flash&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models", + "base" + ], + "query": [ + { + "key": "query", + "value": "gemini-2.5-flash" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-gemini-base-models-stable-cleanup", + "name": "cleanup: delete provider [gemini-base-models-stable]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [gemini-base-models-stable]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-gemini-base-models-stable-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-gemini-base-models-stable-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Keyless provider is listed but contributes no catalog rows", + "description": "A custom provider created with is_key_less=true appears in the providers list; its catalog read succeeds and is empty (no live discovery without keys, no datasheet rows under the custom name).", + "item": [ + { + "id": "catwiring-keyless-provider-01-add-provider", + "name": "01. add provider [keyless-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [keyless-provider]\";", + "pm.test(\"01. add provider [keyless-provider]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-keyless-provider-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":true}}" + } + } + }, + { + "id": "catwiring-keyless-provider-02-assert-providers", + "name": "02. keyless provider appears in providers list [keyless-provider]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 1000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [keyless-provider]\";", + "function assertNow() {", + " var providerName = 'catwiring-keyless-provider-' + pm.variables.get('run_id');", + " if (pm.response.code !== 200) { throw new Error('list providers status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.providers || []).map(function (p) { return p.name; });", + " if (names.indexOf(providerName) < 0) { throw new Error('expected provider ' + providerName + ' in providers list'); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"02. keyless provider appears in providers list [keyless-provider]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"02. keyless provider appears in providers list [keyless-provider]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + } + } + }, + { + "id": "catwiring-keyless-provider-03-assert-models", + "name": "03. keyless provider has no catalog rows [keyless-provider]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 1000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [keyless-provider]\";", + "function assertNow() {", + " var providerName = 'catwiring-keyless-provider-' + pm.variables.get('run_id');", + " var expectSubset = [];", + " var expectSuperset = [];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = true;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. keyless provider has no catalog rows [keyless-provider]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. keyless provider has no catalog rows [keyless-provider]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-keyless-provider-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-keyless-provider-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-keyless-provider-cleanup", + "name": "cleanup: delete provider [keyless-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [keyless-provider]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-keyless-provider-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-keyless-provider-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Unreachable list-models upstream keeps key allow-list models", + "description": "A provider whose base_url points at a dead port cannot complete live model discovery; the catalog still surfaces the models explicitly allowed by its keys.", + "item": [ + { + "id": "catwiring-list-models-failing-01-add-provider", + "name": "01. add provider [list-models-failing]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [list-models-failing]\";", + "pm.test(\"01. add provider [list-models-failing]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-list-models-failing-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false},\"network_config\":{\"base_url\":\"http://127.0.0.1:9\"}}" + } + } + }, + { + "id": "catwiring-list-models-failing-02-add-key", + "name": "02. add key k1 [list-models-failing]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [list-models-failing]\";", + "pm.test(\"02. add key k1 [list-models-failing]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-list-models-failing-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-list-models-failing-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"list-models-failing-k1-{{run_id}}\",\"name\":\"catwiring-mc-list-models-failing-k1-{{run_id}}\",\"value\":\"env.OPENAI_API_KEY\",\"models\":[\"gpt-4o\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-list-models-failing-03-assert-models", + "name": "03. key allow-list survives discovery failure [list-models-failing]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [list-models-failing]\";", + "function assertNow() {", + " var providerName = 'catwiring-list-models-failing-' + pm.variables.get('run_id');", + " var expectSubset = [\"gpt-4o\"];", + " var expectSuperset = [];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. key allow-list survives discovery failure [list-models-failing]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. key allow-list survives discovery failure [list-models-failing]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-list-models-failing-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-list-models-failing-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-list-models-failing-cleanup", + "name": "cleanup: delete provider [list-models-failing]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [list-models-failing]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-list-models-failing-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-list-models-failing-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Disallowed list-models blocks discovery but not explicit allow-lists", + "description": "A provider whose allowed_requests excludes list_models performs no live discovery: a wildcard key surfaces no catalog rows, while a key with an explicit allow-list still surfaces its models.", + "item": [ + { + "id": "catwiring-list-models-disabled-01-add-provider", + "name": "01. add provider [list-models-disabled]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [list-models-disabled]\";", + "pm.test(\"01. add provider [list-models-disabled]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-list-models-disabled-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false,\"allowed_requests\":{\"chat_completion\":true,\"chat_completion_stream\":true}}}" + } + } + }, + { + "id": "catwiring-list-models-disabled-02-add-key", + "name": "02. add key k1 [list-models-disabled]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [list-models-disabled]\";", + "pm.test(\"02. add key k1 [list-models-disabled]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-list-models-disabled-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-list-models-disabled-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"list-models-disabled-k1-{{run_id}}\",\"name\":\"catwiring-mc-list-models-disabled-k1-{{run_id}}\",\"value\":\"env.OPENAI_API_KEY\",\"models\":[\"*\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-list-models-disabled-03-assert-models", + "name": "03. wildcard key surfaces nothing without discovery [list-models-disabled]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 4000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [list-models-disabled]\";", + "function assertNow() {", + " var providerName = 'catwiring-list-models-disabled-' + pm.variables.get('run_id');", + " var expectSubset = [];", + " var expectSuperset = [];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = true;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. wildcard key surfaces nothing without discovery [list-models-disabled]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. wildcard key surfaces nothing without discovery [list-models-disabled]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-list-models-disabled-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-list-models-disabled-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "id": "catwiring-list-models-disabled-04-add-key", + "name": "04. add key k2 [list-models-disabled]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [list-models-disabled]\";", + "pm.test(\"04. add key k2 [list-models-disabled]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-list-models-disabled-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-list-models-disabled-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"list-models-disabled-k2-{{run_id}}\",\"name\":\"catwiring-mc-list-models-disabled-k2-{{run_id}}\",\"value\":\"env.OPENAI_API_KEY\",\"models\":[\"gpt-4o-mini\"],\"enabled\":true}" + } + } + }, + { + "id": "catwiring-list-models-disabled-05-assert-models", + "name": "05. explicit allow-list still surfaces [list-models-disabled]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [list-models-disabled]\";", + "function assertNow() {", + " var providerName = 'catwiring-list-models-disabled-' + pm.variables.get('run_id');", + " var expectSubset = [\"gpt-4o-mini\"];", + " var expectSuperset = [];", + " var expectAbsent = [];", + " var expectAbsentVars = [];", + " var expectEmpty = false;", + " var expectNonEmpty = false;", + " if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + " var body = pm.response.json();", + " var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + " if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + " if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + " expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + " });", + " expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + " });", + " expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. explicit allow-list still surfaces [list-models-disabled]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. explicit allow-list still surfaces [list-models-disabled]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models?provider=catwiring-list-models-disabled-{{run_id}}&limit=1000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models" + ], + "query": [ + { + "key": "provider", + "value": "catwiring-list-models-disabled-{{run_id}}" + }, + { + "key": "limit", + "value": "1000" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "catwiring-list-models-disabled-cleanup", + "name": "cleanup: delete provider [list-models-disabled]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [list-models-disabled]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-list-models-disabled-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-list-models-disabled-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + } + ] +} diff --git a/tests/e2e/api/collections/bifrost-routing-wiring.postman_collection.json b/tests/e2e/api/collections/bifrost-routing-wiring.postman_collection.json new file mode 100644 index 0000000000..a2ff2efa2a --- /dev/null +++ b/tests/e2e/api/collections/bifrost-routing-wiring.postman_collection.json @@ -0,0 +1,17721 @@ +{ + "info": { + "_postman_id": "bifrost-routing-wiring", + "name": "Bifrost Routing Wiring (Governance x Catalog)", + "description": "Governance × model-catalog routing wiring. Each scenario stands up an isolated, run-namespaced custom provider backed by real OpenAI, gates it with keys and a virtual key, then drives inference and asserts the route via extra_fields.routing_info (success), the error message (rejection), and the stored log. Machine-generated by runners/build-routing-wiring.mjs — do not hand-edit.", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" + }, + "variable": [ + { + "key": "base_url", + "value": "http://localhost:8080", + "type": "string" + }, + { + "key": "run_id", + "value": "", + "type": "string" + }, + { + "key": "__purge_target", + "value": "", + "type": "string" + }, + { + "key": "__purge_queue", + "value": "", + "type": "string" + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "if (!pm.collectionVariables.get('run_id')) {", + " var seed = pm.variables.get('e2e_seed_prefix') || pm.environment.get('e2e_seed_prefix') || 'local';", + " var nonce = Date.now().toString(36) + '-' + Math.floor(Math.random() * 1000000).toString(36);", + " pm.collectionVariables.set('run_id', seed + '-' + nonce);", + " console.log('run_id = ' + pm.collectionVariables.get('run_id'));", + "}" + ] + } + } + ], + "item": [ + { + "id": "setup-clear-providers", + "name": "Setup: clear providers", + "item": [ + { + "id": "setup-list-providers", + "name": "setup: list providers to clear", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('setup: list providers', function () { pm.expect(pm.response.code, pm.response.text()).to.equal(200); });", + "if (pm.response.code !== 200) { return; }", + "var provs = ((pm.response.json() || {}).providers || []).map(function (p) { return p.name; }).filter(Boolean);", + "if (provs.length === 0) { pm.collectionVariables.set('__purge_target', ''); pm.collectionVariables.set('__purge_queue', ''); return; }", + "pm.collectionVariables.set('__purge_target', provs[0]);", + "pm.collectionVariables.set('__purge_queue', provs.slice(1).join(','));", + "pm.execution.setNextRequest(\"setup: delete provider\");" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + } + } + }, + { + "id": "setup-delete-provider", + "name": "setup: delete provider", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "if (!pm.collectionVariables.get('__purge_target')) { pm.execution.skipRequest(); }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('setup: delete provider', function () { pm.expect([200, 204, 404], 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code); });", + "var queue = (pm.collectionVariables.get('__purge_queue') || '').split(',').filter(Boolean);", + "if (queue.length) {", + " pm.collectionVariables.set('__purge_target', queue[0]);", + " pm.collectionVariables.set('__purge_queue', queue.slice(1).join(','));", + " pm.execution.setNextRequest(\"setup: delete provider\");", + "} else {", + " pm.collectionVariables.set('__purge_target', '');", + "}" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/{{__purge_target}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "{{__purge_target}}" + ] + } + } + } + ] + }, + { + "name": "VK allows model, key allows — request routes", + "description": "A VK whose allowed_models includes the request, over a key that allows it, routes successfully and records the key.", + "item": [ + { + "id": "rt-vk-allows-model-01-add-provider", + "name": "01. add provider [vk-allows-model]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [vk-allows-model]\";", + "pm.test(\"01. add provider [vk-allows-model]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-vk-allows-model-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-vk-allows-model-02-add-key", + "name": "02. add key k1 [vk-allows-model]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [vk-allows-model]\";", + "pm.test(\"02. add key k1 [vk-allows-model]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-vk-allows-model-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-vk-allows-model-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"k1-{{run_id}}\",\"name\":\"catwiring-rt-vk-allows-model-k1-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\"}" + } + } + }, + { + "id": "rt-vk-allows-model-03-create-vk", + "name": "03. create vk [vk-allows-model]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [vk-allows-model]\";", + "var ok = pm.response.code === 200 || pm.response.code === 201;", + "pm.test(\"03. create vk [vk-allows-model]\", function () {", + " pm.expect([200, 201], 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (ok) {", + " var vk = (pm.response.json() || {}).virtual_key || {};", + " pm.collectionVariables.set('vkval_vk-allows-model', vk.value || '');", + " pm.collectionVariables.set('vkid_vk-allows-model', vk.id || '');", + "} else { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"name\":\"catwiring-rtvk-vk-allows-model-{{run_id}}\",\"is_active\":true,\"provider_configs\":[{\"provider\":\"catwiring-rt-vk-allows-model-{{run_id}}\",\"key_ids\":[\"*\"],\"allowed_models\":[\"gpt-4o-mini\"],\"blacklisted_models\":[],\"weight\":1}]}" + } + } + }, + { + "id": "rt-vk-allows-model-04-route", + "name": "04. allowed model routes (routing_info.key) [vk-allows-model]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 1000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [vk-allows-model]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('route status ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " if (!body.choices || body.choices.length === 0) { throw new Error('no choices'); }", + " var ri = (body.extra_fields || {}).routing_info || {};", + " var providerName = 'catwiring-rt-vk-allows-model-' + pm.variables.get('run_id');", + " if (ri.provider !== providerName) { throw new Error('routing_info.provider=' + ri.provider + ' expected ' + providerName); }", + " var expectedKey = 'catwiring-rt-vk-allows-model-k1-' + pm.variables.get('run_id');", + " if (ri.key !== expectedKey) { throw new Error('routing_info.key=' + ri.key + ' expected ' + expectedKey); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. allowed model routes (routing_info.key) [vk-allows-model]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. allowed model routes (routing_info.key) [vk-allows-model]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_vk-allows-model}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-vk-allows-model-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-vk-allows-model-05-assert-log", + "name": "05. log records VK + selected key [vk-allows-model]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [vk-allows-model]\";", + "function assertNow() {", + " var providerName = 'catwiring-rt-vk-allows-model-' + pm.variables.get('run_id');", + " if (pm.response.code !== 200) { throw new Error('logs status ' + pm.response.code); }", + " var logs = (pm.response.json() || {}).logs || [];", + " var model = \"gpt-4o-mini\";", + " var row = null;", + " for (var i = 0; i < logs.length; i++) {", + " if (logs[i].provider === providerName && logs[i].model === model) { row = logs[i]; break; }", + " }", + " if (!row) { throw new Error('no log row for ' + providerName + '/' + model + ' in ' + logs.length + ' rows'); }", + " if (row.status !== \"success\") { throw new Error('log status=' + row.status); }", + " var expectedKey = 'catwiring-rt-vk-allows-model-k1-' + pm.variables.get('run_id');", + " if (row.selected_key_name !== expectedKey) { throw new Error('selected_key_name=' + row.selected_key_name + ' expected ' + expectedKey); }", + " if (!row.virtual_key_id) { throw new Error('log row missing virtual_key_id'); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. log records VK + selected key [vk-allows-model]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. log records VK + selected key [vk-allows-model]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/logs?virtual_key_ids={{vkid_vk-allows-model}}&limit=10", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "logs" + ], + "query": [ + { + "key": "virtual_key_ids", + "value": "{{vkid_vk-allows-model}}" + }, + { + "key": "limit", + "value": "10" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-vk-allows-model-cleanup-vk", + "name": "cleanup: delete vk [vk-allows-model]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete vk [vk-allows-model]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys/{{vkid_vk-allows-model}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys", + "{{vkid_vk-allows-model}}" + ] + } + } + }, + { + "id": "rt-vk-allows-model-cleanup-provider-self", + "name": "cleanup: delete provider [vk-allows-model]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [vk-allows-model]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-vk-allows-model-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-vk-allows-model-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "VK key restriction pins routing to the allowed key", + "description": "A VK whose key_ids names only k1 routes through k1 even though k2 also serves the model; routing_info.key and the log's selected_key_name are k1.", + "item": [ + { + "id": "rt-vk-key-restriction-01-add-provider", + "name": "01. add provider [vk-key-restriction]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [vk-key-restriction]\";", + "pm.test(\"01. add provider [vk-key-restriction]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-vk-key-restriction-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-vk-key-restriction-02-add-key", + "name": "02. add key k1 [vk-key-restriction]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [vk-key-restriction]\";", + "pm.test(\"02. add key k1 [vk-key-restriction]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-vk-key-restriction-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-vk-key-restriction-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"k1-{{run_id}}\",\"name\":\"catwiring-rt-vk-key-restriction-k1-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\"}" + } + } + }, + { + "id": "rt-vk-key-restriction-03-add-key", + "name": "03. add key k2 [vk-key-restriction]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [vk-key-restriction]\";", + "pm.test(\"03. add key k2 [vk-key-restriction]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-vk-key-restriction-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-vk-key-restriction-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"k2-{{run_id}}\",\"name\":\"catwiring-rt-vk-key-restriction-k2-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\"}" + } + } + }, + { + "id": "rt-vk-key-restriction-04-create-vk", + "name": "04. create vk [vk-key-restriction]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [vk-key-restriction]\";", + "var ok = pm.response.code === 200 || pm.response.code === 201;", + "pm.test(\"04. create vk [vk-key-restriction]\", function () {", + " pm.expect([200, 201], 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (ok) {", + " var vk = (pm.response.json() || {}).virtual_key || {};", + " pm.collectionVariables.set('vkval_vk-key-restriction', vk.value || '');", + " pm.collectionVariables.set('vkid_vk-key-restriction', vk.id || '');", + "} else { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"name\":\"catwiring-rtvk-vk-key-restriction-{{run_id}}\",\"is_active\":true,\"provider_configs\":[{\"provider\":\"catwiring-rt-vk-key-restriction-{{run_id}}\",\"key_ids\":[\"k1-{{run_id}}\"],\"allowed_models\":[\"*\"],\"blacklisted_models\":[],\"weight\":1}]}" + } + } + }, + { + "id": "rt-vk-key-restriction-05-route", + "name": "05. routes via the VK-permitted key (routing_info.key=k1) [vk-key-restriction]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 1000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [vk-key-restriction]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('route status ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " if (!body.choices || body.choices.length === 0) { throw new Error('no choices'); }", + " var ri = (body.extra_fields || {}).routing_info || {};", + " var providerName = 'catwiring-rt-vk-key-restriction-' + pm.variables.get('run_id');", + " if (ri.provider !== providerName) { throw new Error('routing_info.provider=' + ri.provider + ' expected ' + providerName); }", + " var expectedKey = 'catwiring-rt-vk-key-restriction-k1-' + pm.variables.get('run_id');", + " if (ri.key !== expectedKey) { throw new Error('routing_info.key=' + ri.key + ' expected ' + expectedKey); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. routes via the VK-permitted key (routing_info.key=k1) [vk-key-restriction]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. routes via the VK-permitted key (routing_info.key=k1) [vk-key-restriction]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_vk-key-restriction}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-vk-key-restriction-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-vk-key-restriction-06-assert-log", + "name": "06. log selected_key_name is k1 [vk-key-restriction]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [vk-key-restriction]\";", + "function assertNow() {", + " var providerName = 'catwiring-rt-vk-key-restriction-' + pm.variables.get('run_id');", + " if (pm.response.code !== 200) { throw new Error('logs status ' + pm.response.code); }", + " var logs = (pm.response.json() || {}).logs || [];", + " var model = \"gpt-4o-mini\";", + " var row = null;", + " for (var i = 0; i < logs.length; i++) {", + " if (logs[i].provider === providerName && logs[i].model === model) { row = logs[i]; break; }", + " }", + " if (!row) { throw new Error('no log row for ' + providerName + '/' + model + ' in ' + logs.length + ' rows'); }", + " if (row.status !== \"success\") { throw new Error('log status=' + row.status); }", + " var expectedKey = 'catwiring-rt-vk-key-restriction-k1-' + pm.variables.get('run_id');", + " if (row.selected_key_name !== expectedKey) { throw new Error('selected_key_name=' + row.selected_key_name + ' expected ' + expectedKey); }", + " if (!row.virtual_key_id) { throw new Error('log row missing virtual_key_id'); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"06. log selected_key_name is k1 [vk-key-restriction]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"06. log selected_key_name is k1 [vk-key-restriction]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/logs?virtual_key_ids={{vkid_vk-key-restriction}}&limit=10", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "logs" + ], + "query": [ + { + "key": "virtual_key_ids", + "value": "{{vkid_vk-key-restriction}}" + }, + { + "key": "limit", + "value": "10" + } + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-vk-key-restriction-cleanup-vk", + "name": "cleanup: delete vk [vk-key-restriction]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete vk [vk-key-restriction]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys/{{vkid_vk-key-restriction}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys", + "{{vkid_vk-key-restriction}}" + ] + } + } + }, + { + "id": "rt-vk-key-restriction-cleanup-provider-self", + "name": "cleanup: delete provider [vk-key-restriction]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [vk-key-restriction]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-vk-key-restriction-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-vk-key-restriction-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "VK pinned to a disabled key is unroutable even with an enabled sibling", + "description": "The VK restricts to k1, which is disabled; k2 is enabled but VK-excluded, so key selection finds no usable key and rejects with 400.", + "item": [ + { + "id": "rt-vk-disabled-key-01-add-provider", + "name": "01. add provider [vk-disabled-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [vk-disabled-key]\";", + "pm.test(\"01. add provider [vk-disabled-key]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-vk-disabled-key-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-vk-disabled-key-02-add-key", + "name": "02. add key k1 [vk-disabled-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [vk-disabled-key]\";", + "pm.test(\"02. add key k1 [vk-disabled-key]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-vk-disabled-key-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-vk-disabled-key-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"k1-{{run_id}}\",\"name\":\"catwiring-rt-vk-disabled-key-k1-{{run_id}}\",\"models\":[\"*\"],\"enabled\":false,\"value\":\"env.OPENAI_API_KEY\"}" + } + } + }, + { + "id": "rt-vk-disabled-key-03-add-key", + "name": "03. add key k2 [vk-disabled-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [vk-disabled-key]\";", + "pm.test(\"03. add key k2 [vk-disabled-key]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-vk-disabled-key-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-vk-disabled-key-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"k2-{{run_id}}\",\"name\":\"catwiring-rt-vk-disabled-key-k2-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\"}" + } + } + }, + { + "id": "rt-vk-disabled-key-04-create-vk", + "name": "04. create vk [vk-disabled-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [vk-disabled-key]\";", + "var ok = pm.response.code === 200 || pm.response.code === 201;", + "pm.test(\"04. create vk [vk-disabled-key]\", function () {", + " pm.expect([200, 201], 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (ok) {", + " var vk = (pm.response.json() || {}).virtual_key || {};", + " pm.collectionVariables.set('vkval_vk-disabled-key', vk.value || '');", + " pm.collectionVariables.set('vkid_vk-disabled-key', vk.id || '');", + "} else { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"name\":\"catwiring-rtvk-vk-disabled-key-{{run_id}}\",\"is_active\":true,\"provider_configs\":[{\"provider\":\"catwiring-rt-vk-disabled-key-{{run_id}}\",\"key_ids\":[\"k1-{{run_id}}\"],\"allowed_models\":[\"*\"],\"blacklisted_models\":[],\"weight\":1}]}" + } + } + }, + { + "id": "rt-vk-disabled-key-05-route", + "name": "05. disabled+VK-restricted key yields no usable key (400) [vk-disabled-key]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 1000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [vk-disabled-key]\";", + "function assertNow() {", + " if (pm.response.code !== 400) { throw new Error('expected status 400 got ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " var msg = (body.error && (body.error.message || body.error)) || body.message || '';", + " if (String(msg).indexOf(\"no keys found\") < 0) { throw new Error('error message=' + msg); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. disabled+VK-restricted key yields no usable key (400) [vk-disabled-key]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. disabled+VK-restricted key yields no usable key (400) [vk-disabled-key]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_vk-disabled-key}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-vk-disabled-key-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-vk-disabled-key-cleanup-vk", + "name": "cleanup: delete vk [vk-disabled-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete vk [vk-disabled-key]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys/{{vkid_vk-disabled-key}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys", + "{{vkid_vk-disabled-key}}" + ] + } + } + }, + { + "id": "rt-vk-disabled-key-cleanup-provider-self", + "name": "cleanup: delete provider [vk-disabled-key]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [vk-disabled-key]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-vk-disabled-key-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-vk-disabled-key-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Both keys serve under weighted key selection", + "description": "Two enabled, equal-weight keys on one provider (no VK). Over a batch of requests, core's weighted key selection routes through both keys.", + "item": [ + { + "id": "rt-multi-key-distribution-01-add-provider", + "name": "01. add provider [multi-key-distribution]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [multi-key-distribution]\";", + "pm.test(\"01. add provider [multi-key-distribution]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-multi-key-distribution-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-multi-key-distribution-02-add-key", + "name": "02. add key k1 [multi-key-distribution]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [multi-key-distribution]\";", + "pm.test(\"02. add key k1 [multi-key-distribution]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-multi-key-distribution-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-multi-key-distribution-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"k1-{{run_id}}\",\"name\":\"catwiring-rt-multi-key-distribution-k1-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\",\"weight\":1}" + } + } + }, + { + "id": "rt-multi-key-distribution-03-add-key", + "name": "03. add key k2 [multi-key-distribution]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [multi-key-distribution]\";", + "pm.test(\"03. add key k2 [multi-key-distribution]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-multi-key-distribution-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-multi-key-distribution-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"k2-{{run_id}}\",\"name\":\"catwiring-rt-multi-key-distribution-k2-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\",\"weight\":1}" + } + } + }, + { + "id": "rt-multi-key-distribution-04-dist-sample", + "name": "04. sample 1/8 (gpt-4o-mini) [multi-key-distribution]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [multi-key-distribution]\";", + "pm.collectionVariables.set('dist_multi-key-distribution', '');", + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + " var cur = pm.collectionVariables.get('dist_multi-key-distribution') || '';", + " var set = cur ? cur.split(',') : [];", + " if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_multi-key-distribution', set.join(',')); }", + "}", + "pm.test(\"04. sample 1/8 (gpt-4o-mini) [multi-key-distribution]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-multi-key-distribution-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-multi-key-distribution-05-dist-sample", + "name": "05. sample 2/8 (gpt-4o-mini) [multi-key-distribution]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [multi-key-distribution]\";", + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + " var cur = pm.collectionVariables.get('dist_multi-key-distribution') || '';", + " var set = cur ? cur.split(',') : [];", + " if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_multi-key-distribution', set.join(',')); }", + "}", + "pm.test(\"05. sample 2/8 (gpt-4o-mini) [multi-key-distribution]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-multi-key-distribution-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-multi-key-distribution-06-dist-sample", + "name": "06. sample 3/8 (gpt-4o-mini) [multi-key-distribution]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [multi-key-distribution]\";", + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + " var cur = pm.collectionVariables.get('dist_multi-key-distribution') || '';", + " var set = cur ? cur.split(',') : [];", + " if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_multi-key-distribution', set.join(',')); }", + "}", + "pm.test(\"06. sample 3/8 (gpt-4o-mini) [multi-key-distribution]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-multi-key-distribution-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-multi-key-distribution-07-dist-sample", + "name": "07. sample 4/8 (gpt-4o-mini) [multi-key-distribution]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [multi-key-distribution]\";", + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + " var cur = pm.collectionVariables.get('dist_multi-key-distribution') || '';", + " var set = cur ? cur.split(',') : [];", + " if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_multi-key-distribution', set.join(',')); }", + "}", + "pm.test(\"07. sample 4/8 (gpt-4o-mini) [multi-key-distribution]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-multi-key-distribution-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-multi-key-distribution-08-dist-sample", + "name": "08. sample 5/8 (gpt-4o-mini) [multi-key-distribution]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [multi-key-distribution]\";", + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + " var cur = pm.collectionVariables.get('dist_multi-key-distribution') || '';", + " var set = cur ? cur.split(',') : [];", + " if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_multi-key-distribution', set.join(',')); }", + "}", + "pm.test(\"08. sample 5/8 (gpt-4o-mini) [multi-key-distribution]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-multi-key-distribution-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-multi-key-distribution-09-dist-sample", + "name": "09. sample 6/8 (gpt-4o-mini) [multi-key-distribution]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [multi-key-distribution]\";", + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + " var cur = pm.collectionVariables.get('dist_multi-key-distribution') || '';", + " var set = cur ? cur.split(',') : [];", + " if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_multi-key-distribution', set.join(',')); }", + "}", + "pm.test(\"09. sample 6/8 (gpt-4o-mini) [multi-key-distribution]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-multi-key-distribution-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-multi-key-distribution-10-dist-sample", + "name": "10. sample 7/8 (gpt-4o-mini) [multi-key-distribution]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [multi-key-distribution]\";", + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + " var cur = pm.collectionVariables.get('dist_multi-key-distribution') || '';", + " var set = cur ? cur.split(',') : [];", + " if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_multi-key-distribution', set.join(',')); }", + "}", + "pm.test(\"10. sample 7/8 (gpt-4o-mini) [multi-key-distribution]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-multi-key-distribution-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-multi-key-distribution-11-dist-sample", + "name": "11. sample 8/8 (gpt-4o-mini) [multi-key-distribution]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [multi-key-distribution]\";", + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + " var cur = pm.collectionVariables.get('dist_multi-key-distribution') || '';", + " var set = cur ? cur.split(',') : [];", + " if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_multi-key-distribution', set.join(',')); }", + "}", + "pm.test(\"11. sample 8/8 (gpt-4o-mini) [multi-key-distribution]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-multi-key-distribution-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-multi-key-distribution-12-dist-assert", + "name": "12. both keys served across 8 samples [multi-key-distribution]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [multi-key-distribution]\";", + "var ok = true, errMsg = '';", + "try {", + " var seen = (pm.collectionVariables.get('dist_multi-key-distribution') || '').split(',').filter(Boolean);", + " var mustServe = ['catwiring-rt-multi-key-distribution-k1-' + pm.variables.get('run_id'), 'catwiring-rt-multi-key-distribution-k2-' + pm.variables.get('run_id')];", + " mustServe.forEach(function (e) { if (seen.indexOf(e) < 0) throw new Error('key ' + e + ' never served; observed ' + JSON.stringify(seen)); });", + "} catch (e) { ok = false; errMsg = e.message; }", + "pm.test(\"12. both keys served across 8 samples [multi-key-distribution]\", function () { if (!ok) throw new Error(errMsg); });", + "if (!ok) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/health", + "host": [ + "{{base_url}}" + ], + "path": [ + "health" + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-multi-key-distribution-cleanup-provider-self", + "name": "cleanup: delete provider [multi-key-distribution]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [multi-key-distribution]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-multi-key-distribution-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-multi-key-distribution-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "A weightless VK is an allow-list (no LB), confirmed by the routing log trail", + "description": "Two providers on a VK with NO weights: A's key serves gpt-4o-mini, B's serves only gpt-4o. Routing the bare gpt-4o-mini, governance filters by capability (excludes B) and — having no weighted configs — skips load balancing, routing to A. The routing_engine_logs record the allow-list decisions.", + "item": [ + { + "id": "rt-weightless-vk-allowlist-via-logs-01-add-provider", + "name": "01. add provider [weightless-vk-allowlist-via-logs]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [weightless-vk-allowlist-via-logs]\";", + "pm.test(\"01. add provider [weightless-vk-allowlist-via-logs]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-weightless-vk-allowlist-via-logs-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-weightless-vk-allowlist-via-logs-02-add-key", + "name": "02. add key ka [weightless-vk-allowlist-via-logs]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [weightless-vk-allowlist-via-logs]\";", + "pm.test(\"02. add key ka [weightless-vk-allowlist-via-logs]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-weightless-vk-allowlist-via-logs-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-weightless-vk-allowlist-via-logs-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"ka-{{run_id}}\",\"name\":\"catwiring-rt-weightless-vk-allowlist-via-logs-ka-{{run_id}}\",\"models\":[\"gpt-4o-mini\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\"}" + } + } + }, + { + "id": "rt-weightless-vk-allowlist-via-logs-03-add-provider", + "name": "03. add provider (b) [weightless-vk-allowlist-via-logs]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [weightless-vk-allowlist-via-logs]\";", + "pm.test(\"03. add provider (b) [weightless-vk-allowlist-via-logs]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-weightless-vk-allowlist-via-logs-b-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-weightless-vk-allowlist-via-logs-04-add-key", + "name": "04. add key kb (b) [weightless-vk-allowlist-via-logs]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [weightless-vk-allowlist-via-logs]\";", + "pm.test(\"04. add key kb (b) [weightless-vk-allowlist-via-logs]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-weightless-vk-allowlist-via-logs-b-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-weightless-vk-allowlist-via-logs-b-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"kb-{{run_id}}\",\"name\":\"catwiring-rt-weightless-vk-allowlist-via-logs-kb-{{run_id}}\",\"models\":[\"gpt-4o\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\"}" + } + } + }, + { + "id": "rt-weightless-vk-allowlist-via-logs-05-create-vk", + "name": "05. create vk [weightless-vk-allowlist-via-logs]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [weightless-vk-allowlist-via-logs]\";", + "var ok = pm.response.code === 200 || pm.response.code === 201;", + "pm.test(\"05. create vk [weightless-vk-allowlist-via-logs]\", function () {", + " pm.expect([200, 201], 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (ok) {", + " var vk = (pm.response.json() || {}).virtual_key || {};", + " pm.collectionVariables.set('vkval_weightless-vk-allowlist-via-logs', vk.value || '');", + " pm.collectionVariables.set('vkid_weightless-vk-allowlist-via-logs', vk.id || '');", + "} else { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"name\":\"catwiring-rtvk-weightless-vk-allowlist-via-logs-{{run_id}}\",\"is_active\":true,\"provider_configs\":[{\"provider\":\"catwiring-rt-weightless-vk-allowlist-via-logs-{{run_id}}\",\"key_ids\":[\"*\"],\"allowed_models\":[\"*\"],\"blacklisted_models\":[]},{\"provider\":\"catwiring-rt-weightless-vk-allowlist-via-logs-b-{{run_id}}\",\"key_ids\":[\"*\"],\"allowed_models\":[\"*\"],\"blacklisted_models\":[]}]}" + } + } + }, + { + "id": "rt-weightless-vk-allowlist-via-logs-06-route", + "name": "06. bare model routes to the only capable provider (allow-list, no LB) [weightless-vk-allowlist-via-logs]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 3000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [weightless-vk-allowlist-via-logs]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('route status ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " if (!body.choices || body.choices.length === 0) { throw new Error('no choices'); }", + " var ri = (body.extra_fields || {}).routing_info || {};", + " var allowedProviders = ['catwiring-rt-weightless-vk-allowlist-via-logs-' + pm.variables.get('run_id')];", + " if (allowedProviders.indexOf(ri.provider) < 0) { throw new Error('routing_info.provider=' + ri.provider + ' not in ' + JSON.stringify(allowedProviders)); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"06. bare model routes to the only capable provider (allow-list, no LB) [weightless-vk-allowlist-via-logs]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"06. bare model routes to the only capable provider (allow-list, no LB) [weightless-vk-allowlist-via-logs]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_weightless-vk-allowlist-via-logs}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-weightless-vk-allowlist-via-logs-07-capture-log", + "name": "07. capture routing log id [weightless-vk-allowlist-via-logs]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [weightless-vk-allowlist-via-logs]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('logs status ' + pm.response.code); }", + " var logs = (pm.response.json() || {}).logs || [];", + " if (!logs.length || !logs[0].id) { throw new Error('no log row yet for VK'); }", + " pm.collectionVariables.set('logid_weightless-vk-allowlist-via-logs', logs[0].id);", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"07. capture routing log id [weightless-vk-allowlist-via-logs]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"07. capture routing log id [weightless-vk-allowlist-via-logs]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/logs?virtual_key_ids={{vkid_weightless-vk-allowlist-via-logs}}&limit=1", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "logs" + ], + "query": [ + { + "key": "virtual_key_ids", + "value": "{{vkid_weightless-vk-allowlist-via-logs}}" + }, + { + "key": "limit", + "value": "1" + } + ] + } + } + }, + { + "id": "rt-weightless-vk-allowlist-via-logs-08-assert-trail", + "name": "08. log trail shows allow-list filtering and LB skipped [weightless-vk-allowlist-via-logs]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [weightless-vk-allowlist-via-logs]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('log detail status ' + pm.response.code); }", + " var row = pm.response.json() || {};", + " var trail = row.routing_engine_logs || '';", + " var expected = [\"not in allowed models list\",\"No weighted configs\",\"skipping load balancing\"];", + " expected.forEach(function (s) {", + " if (String(trail).indexOf(s) < 0) { throw new Error('routing_engine_logs missing ' + JSON.stringify(s) + '; got ' + JSON.stringify(trail)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"08. log trail shows allow-list filtering and LB skipped [weightless-vk-allowlist-via-logs]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"08. log trail shows allow-list filtering and LB skipped [weightless-vk-allowlist-via-logs]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/logs/{{logid_weightless-vk-allowlist-via-logs}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "logs", + "{{logid_weightless-vk-allowlist-via-logs}}" + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-weightless-vk-allowlist-via-logs-cleanup-vk", + "name": "cleanup: delete vk [weightless-vk-allowlist-via-logs]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete vk [weightless-vk-allowlist-via-logs]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys/{{vkid_weightless-vk-allowlist-via-logs}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys", + "{{vkid_weightless-vk-allowlist-via-logs}}" + ] + } + } + }, + { + "id": "rt-weightless-vk-allowlist-via-logs-cleanup-provider-self", + "name": "cleanup: delete provider [weightless-vk-allowlist-via-logs]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [weightless-vk-allowlist-via-logs]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-weightless-vk-allowlist-via-logs-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-weightless-vk-allowlist-via-logs-{{run_id}}" + ] + } + } + }, + { + "id": "rt-weightless-vk-allowlist-via-logs-cleanup-provider-b", + "name": "cleanup: delete provider b [weightless-vk-allowlist-via-logs]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider b [weightless-vk-allowlist-via-logs]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-weightless-vk-allowlist-via-logs-b-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-weightless-vk-allowlist-via-logs-b-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Governance load-balances a bare model across VK providers", + "description": "A VK with two weighted providers, routing a bare (un-prefixed) model, has governance pick one of them; the log detail's routing_engine_logs records the load-balancing decision.", + "item": [ + { + "id": "rt-governance-lb-distributes-01-add-provider", + "name": "01. add provider [governance-lb-distributes]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [governance-lb-distributes]\";", + "pm.test(\"01. add provider [governance-lb-distributes]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-governance-lb-distributes-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-governance-lb-distributes-02-add-key", + "name": "02. add key ka [governance-lb-distributes]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [governance-lb-distributes]\";", + "pm.test(\"02. add key ka [governance-lb-distributes]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-governance-lb-distributes-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-governance-lb-distributes-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"ka-{{run_id}}\",\"name\":\"catwiring-rt-governance-lb-distributes-ka-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\",\"weight\":1}" + } + } + }, + { + "id": "rt-governance-lb-distributes-03-add-provider", + "name": "03. add provider (b) [governance-lb-distributes]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [governance-lb-distributes]\";", + "pm.test(\"03. add provider (b) [governance-lb-distributes]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-governance-lb-distributes-b-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-governance-lb-distributes-04-add-key", + "name": "04. add key kb (b) [governance-lb-distributes]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [governance-lb-distributes]\";", + "pm.test(\"04. add key kb (b) [governance-lb-distributes]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-governance-lb-distributes-b-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-governance-lb-distributes-b-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"kb-{{run_id}}\",\"name\":\"catwiring-rt-governance-lb-distributes-kb-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\",\"weight\":1}" + } + } + }, + { + "id": "rt-governance-lb-distributes-05-create-vk", + "name": "05. create vk [governance-lb-distributes]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [governance-lb-distributes]\";", + "var ok = pm.response.code === 200 || pm.response.code === 201;", + "pm.test(\"05. create vk [governance-lb-distributes]\", function () {", + " pm.expect([200, 201], 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (ok) {", + " var vk = (pm.response.json() || {}).virtual_key || {};", + " pm.collectionVariables.set('vkval_governance-lb-distributes', vk.value || '');", + " pm.collectionVariables.set('vkid_governance-lb-distributes', vk.id || '');", + "} else { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"name\":\"catwiring-rtvk-governance-lb-distributes-{{run_id}}\",\"is_active\":true,\"provider_configs\":[{\"provider\":\"catwiring-rt-governance-lb-distributes-{{run_id}}\",\"key_ids\":[\"*\"],\"allowed_models\":[\"*\"],\"blacklisted_models\":[],\"weight\":1},{\"provider\":\"catwiring-rt-governance-lb-distributes-b-{{run_id}}\",\"key_ids\":[\"*\"],\"allowed_models\":[\"*\"],\"blacklisted_models\":[],\"weight\":1}]}" + } + } + }, + { + "id": "rt-governance-lb-distributes-06-route", + "name": "06. bare model routes via a governance-selected provider [governance-lb-distributes]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [governance-lb-distributes]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('route status ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " if (!body.choices || body.choices.length === 0) { throw new Error('no choices'); }", + " var ri = (body.extra_fields || {}).routing_info || {};", + " var allowedProviders = ['catwiring-rt-governance-lb-distributes-' + pm.variables.get('run_id'), 'catwiring-rt-governance-lb-distributes-b-' + pm.variables.get('run_id')];", + " if (allowedProviders.indexOf(ri.provider) < 0) { throw new Error('routing_info.provider=' + ri.provider + ' not in ' + JSON.stringify(allowedProviders)); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"06. bare model routes via a governance-selected provider [governance-lb-distributes]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"06. bare model routes via a governance-selected provider [governance-lb-distributes]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_governance-lb-distributes}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-governance-lb-distributes-07-capture-log", + "name": "07. capture routing log id [governance-lb-distributes]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [governance-lb-distributes]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('logs status ' + pm.response.code); }", + " var logs = (pm.response.json() || {}).logs || [];", + " if (!logs.length || !logs[0].id) { throw new Error('no log row yet for VK'); }", + " pm.collectionVariables.set('logid_governance-lb-distributes', logs[0].id);", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"07. capture routing log id [governance-lb-distributes]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"07. capture routing log id [governance-lb-distributes]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/logs?virtual_key_ids={{vkid_governance-lb-distributes}}&limit=1", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "logs" + ], + "query": [ + { + "key": "virtual_key_ids", + "value": "{{vkid_governance-lb-distributes}}" + }, + { + "key": "limit", + "value": "1" + } + ] + } + } + }, + { + "id": "rt-governance-lb-distributes-08-assert-trail", + "name": "08. log detail records the LB decision trail [governance-lb-distributes]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [governance-lb-distributes]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('log detail status ' + pm.response.code); }", + " var row = pm.response.json() || {};", + " var trail = row.routing_engine_logs || '';", + " var expected = [\"Load balancing model\",\"Selected provider\"];", + " expected.forEach(function (s) {", + " if (String(trail).indexOf(s) < 0) { throw new Error('routing_engine_logs missing ' + JSON.stringify(s) + '; got ' + JSON.stringify(trail)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"08. log detail records the LB decision trail [governance-lb-distributes]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"08. log detail records the LB decision trail [governance-lb-distributes]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/logs/{{logid_governance-lb-distributes}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "logs", + "{{logid_governance-lb-distributes}}" + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-governance-lb-distributes-cleanup-vk", + "name": "cleanup: delete vk [governance-lb-distributes]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete vk [governance-lb-distributes]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys/{{vkid_governance-lb-distributes}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys", + "{{vkid_governance-lb-distributes}}" + ] + } + } + }, + { + "id": "rt-governance-lb-distributes-cleanup-provider-self", + "name": "cleanup: delete provider [governance-lb-distributes]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [governance-lb-distributes]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-governance-lb-distributes-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-governance-lb-distributes-{{run_id}}" + ] + } + } + }, + { + "id": "rt-governance-lb-distributes-cleanup-provider-b", + "name": "cleanup: delete provider b [governance-lb-distributes]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider b [governance-lb-distributes]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-governance-lb-distributes-b-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-governance-lb-distributes-b-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Routing the resolved model directly (not the alias) is rejected", + "description": "A key gates the alias name, not the resolved id. Routing the alias resolves and succeeds; routing the resolved model id directly fails the gate — alias targets are not auto-added to the key's Models.", + "item": [ + { + "id": "rt-reverse-alias-gate-01-add-provider", + "name": "01. add provider [reverse-alias-gate]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [reverse-alias-gate]\";", + "pm.test(\"01. add provider [reverse-alias-gate]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-reverse-alias-gate-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-reverse-alias-gate-02-add-key", + "name": "02. add key k1 [reverse-alias-gate]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [reverse-alias-gate]\";", + "pm.test(\"02. add key k1 [reverse-alias-gate]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-reverse-alias-gate-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-reverse-alias-gate-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"k1-{{run_id}}\",\"name\":\"catwiring-rt-reverse-alias-gate-k1-{{run_id}}\",\"models\":[\"catwiring-alias-{{run_id}}\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\",\"aliases\":{\"catwiring-alias-{{run_id}}\":\"gpt-4o-mini\"}}" + } + } + }, + { + "id": "rt-reverse-alias-gate-03-route", + "name": "03. alias routes (resolves to the model id) [reverse-alias-gate]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 1000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [reverse-alias-gate]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('route status ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " if (!body.choices || body.choices.length === 0) { throw new Error('no choices'); }", + " var ri = (body.extra_fields || {}).routing_info || {};", + " var providerName = 'catwiring-rt-reverse-alias-gate-' + pm.variables.get('run_id');", + " if (ri.provider !== providerName) { throw new Error('routing_info.provider=' + ri.provider + ' expected ' + providerName); }", + " if (!ri.resolved_key_alias || ri.resolved_key_alias.model_id !== \"gpt-4o-mini\") { throw new Error('resolved_key_alias=' + JSON.stringify(ri.resolved_key_alias)); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. alias routes (resolves to the model id) [reverse-alias-gate]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. alias routes (resolves to the model id) [reverse-alias-gate]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-reverse-alias-gate-{{run_id}}/catwiring-alias-{{run_id}}\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-reverse-alias-gate-04-route", + "name": "04. resolved id routed directly → 400 (not in Models) [reverse-alias-gate]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [reverse-alias-gate]\";", + "function assertNow() {", + " if (pm.response.code !== 400) { throw new Error('expected status 400 got ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " var msg = (body.error && (body.error.message || body.error)) || body.message || '';", + " if (String(msg).indexOf(\"no keys found that support model\") < 0) { throw new Error('error message=' + msg); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. resolved id routed directly → 400 (not in Models) [reverse-alias-gate]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. resolved id routed directly → 400 (not in Models) [reverse-alias-gate]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-reverse-alias-gate-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-reverse-alias-gate-cleanup-provider-self", + "name": "cleanup: delete provider [reverse-alias-gate]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [reverse-alias-gate]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-reverse-alias-gate-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-reverse-alias-gate-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "An alias defined on two keys resolves to the last key", + "description": "Two keys define the same alias to different models. The alias resolves to the last-defined key's target and is served by that key.", + "item": [ + { + "id": "rt-alias-collision-last-wins-01-add-provider", + "name": "01. add provider [alias-collision-last-wins]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [alias-collision-last-wins]\";", + "pm.test(\"01. add provider [alias-collision-last-wins]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-alias-collision-last-wins-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-alias-collision-last-wins-02-add-key", + "name": "02. add key k1 [alias-collision-last-wins]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [alias-collision-last-wins]\";", + "pm.test(\"02. add key k1 [alias-collision-last-wins]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-alias-collision-last-wins-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-alias-collision-last-wins-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"k1-{{run_id}}\",\"name\":\"catwiring-rt-alias-collision-last-wins-k1-{{run_id}}\",\"models\":[\"catwiring-dup-{{run_id}}\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\",\"aliases\":{\"catwiring-dup-{{run_id}}\":\"gpt-4o\"}}" + } + } + }, + { + "id": "rt-alias-collision-last-wins-03-add-key", + "name": "03. add key k2 [alias-collision-last-wins]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [alias-collision-last-wins]\";", + "pm.test(\"03. add key k2 [alias-collision-last-wins]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-alias-collision-last-wins-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-alias-collision-last-wins-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"k2-{{run_id}}\",\"name\":\"catwiring-rt-alias-collision-last-wins-k2-{{run_id}}\",\"models\":[\"catwiring-dup-{{run_id}}\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\",\"aliases\":{\"catwiring-dup-{{run_id}}\":\"gpt-4o-mini\"}}" + } + } + }, + { + "id": "rt-alias-collision-last-wins-04-route", + "name": "04. alias resolves to the last key (k2 → gpt-4o-mini) [alias-collision-last-wins]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 1000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [alias-collision-last-wins]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('route status ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " if (!body.choices || body.choices.length === 0) { throw new Error('no choices'); }", + " var ri = (body.extra_fields || {}).routing_info || {};", + " var providerName = 'catwiring-rt-alias-collision-last-wins-' + pm.variables.get('run_id');", + " if (ri.provider !== providerName) { throw new Error('routing_info.provider=' + ri.provider + ' expected ' + providerName); }", + " var expectedKey = 'catwiring-rt-alias-collision-last-wins-k2-' + pm.variables.get('run_id');", + " if (ri.key !== expectedKey) { throw new Error('routing_info.key=' + ri.key + ' expected ' + expectedKey); }", + " if (!ri.resolved_key_alias || ri.resolved_key_alias.model_id !== \"gpt-4o-mini\") { throw new Error('resolved_key_alias=' + JSON.stringify(ri.resolved_key_alias)); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. alias resolves to the last key (k2 → gpt-4o-mini) [alias-collision-last-wins]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. alias resolves to the last key (k2 → gpt-4o-mini) [alias-collision-last-wins]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-alias-collision-last-wins-{{run_id}}/catwiring-dup-{{run_id}}\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-alias-collision-last-wins-cleanup-provider-self", + "name": "cleanup: delete provider [alias-collision-last-wins]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [alias-collision-last-wins]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-alias-collision-last-wins-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-alias-collision-last-wins-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "A request resolves to an alias whose name differs only in case", + "description": "A key defines a mixed-case alias and gates the mixed-case name. Routing the lowercased form finds no exact-case alias but resolves through a case-insensitive fallback to the same target.", + "item": [ + { + "id": "rt-alias-case-insensitive-fallback-01-add-provider", + "name": "01. add provider [alias-case-insensitive-fallback]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [alias-case-insensitive-fallback]\";", + "pm.test(\"01. add provider [alias-case-insensitive-fallback]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-alias-case-insensitive-fallback-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-alias-case-insensitive-fallback-02-add-key", + "name": "02. add key k1 [alias-case-insensitive-fallback]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [alias-case-insensitive-fallback]\";", + "pm.test(\"02. add key k1 [alias-case-insensitive-fallback]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-alias-case-insensitive-fallback-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-alias-case-insensitive-fallback-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"k1-{{run_id}}\",\"name\":\"catwiring-rt-alias-case-insensitive-fallback-k1-{{run_id}}\",\"models\":[\"CatWiring-CI-{{run_id}}\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\",\"aliases\":{\"CatWiring-CI-{{run_id}}\":\"gpt-4o-mini\"}}" + } + } + }, + { + "id": "rt-alias-case-insensitive-fallback-03-route", + "name": "03. lowercased request resolves via case-insensitive fallback [alias-case-insensitive-fallback]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 1000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [alias-case-insensitive-fallback]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('route status ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " if (!body.choices || body.choices.length === 0) { throw new Error('no choices'); }", + " var ri = (body.extra_fields || {}).routing_info || {};", + " var providerName = 'catwiring-rt-alias-case-insensitive-fallback-' + pm.variables.get('run_id');", + " if (ri.provider !== providerName) { throw new Error('routing_info.provider=' + ri.provider + ' expected ' + providerName); }", + " var expectedKey = 'catwiring-rt-alias-case-insensitive-fallback-k1-' + pm.variables.get('run_id');", + " if (ri.key !== expectedKey) { throw new Error('routing_info.key=' + ri.key + ' expected ' + expectedKey); }", + " if (!ri.resolved_key_alias || ri.resolved_key_alias.model_id !== \"gpt-4o-mini\") { throw new Error('resolved_key_alias=' + JSON.stringify(ri.resolved_key_alias)); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. lowercased request resolves via case-insensitive fallback [alias-case-insensitive-fallback]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. lowercased request resolves via case-insensitive fallback [alias-case-insensitive-fallback]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-alias-case-insensitive-fallback-{{run_id}}/catwiring-ci-{{run_id}}\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-alias-case-insensitive-fallback-cleanup-provider-self", + "name": "cleanup: delete provider [alias-case-insensitive-fallback]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [alias-case-insensitive-fallback]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-alias-case-insensitive-fallback-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-alias-case-insensitive-fallback-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Alias resolves at routing with no governance", + "description": "Pure model-catalog case (no VK): a key alias routes an inference request to the underlying model; routing_info.resolved_key_alias records the resolution.", + "item": [ + { + "id": "rt-alias-routing-no-vk-01-add-provider", + "name": "01. add provider [alias-routing-no-vk]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [alias-routing-no-vk]\";", + "pm.test(\"01. add provider [alias-routing-no-vk]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-alias-routing-no-vk-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-alias-routing-no-vk-02-add-key", + "name": "02. add key k1 [alias-routing-no-vk]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [alias-routing-no-vk]\";", + "pm.test(\"02. add key k1 [alias-routing-no-vk]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-alias-routing-no-vk-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-alias-routing-no-vk-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"k1-{{run_id}}\",\"name\":\"catwiring-rt-alias-routing-no-vk-k1-{{run_id}}\",\"models\":[\"catwiring-alias-{{run_id}}\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\",\"aliases\":{\"catwiring-alias-{{run_id}}\":\"gpt-4o-mini\"}}" + } + } + }, + { + "id": "rt-alias-routing-no-vk-03-route", + "name": "03. alias routes to underlying model (no VK) [alias-routing-no-vk]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 1000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [alias-routing-no-vk]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('route status ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " if (!body.choices || body.choices.length === 0) { throw new Error('no choices'); }", + " var ri = (body.extra_fields || {}).routing_info || {};", + " var providerName = 'catwiring-rt-alias-routing-no-vk-' + pm.variables.get('run_id');", + " if (ri.provider !== providerName) { throw new Error('routing_info.provider=' + ri.provider + ' expected ' + providerName); }", + " if (!ri.resolved_key_alias || ri.resolved_key_alias.model_id !== \"gpt-4o-mini\") { throw new Error('resolved_key_alias=' + JSON.stringify(ri.resolved_key_alias)); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. alias routes to underlying model (no VK) [alias-routing-no-vk]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. alias routes to underlying model (no VK) [alias-routing-no-vk]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-alias-routing-no-vk-{{run_id}}/catwiring-alias-{{run_id}}\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-alias-routing-no-vk-cleanup-provider-self", + "name": "cleanup: delete provider [alias-routing-no-vk]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [alias-routing-no-vk]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-alias-routing-no-vk-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-alias-routing-no-vk-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Key blacklist gates routing with no governance", + "description": "Pure model-catalog case (no VK): a key that allows all models but blacklists one rejects that model at key selection.", + "item": [ + { + "id": "rt-blacklist-gate-no-vk-01-add-provider", + "name": "01. add provider [blacklist-gate-no-vk]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [blacklist-gate-no-vk]\";", + "pm.test(\"01. add provider [blacklist-gate-no-vk]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-blacklist-gate-no-vk-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-blacklist-gate-no-vk-02-add-key", + "name": "02. add key k1 [blacklist-gate-no-vk]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [blacklist-gate-no-vk]\";", + "pm.test(\"02. add key k1 [blacklist-gate-no-vk]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-blacklist-gate-no-vk-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-blacklist-gate-no-vk-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"k1-{{run_id}}\",\"name\":\"catwiring-rt-blacklist-gate-no-vk-k1-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\",\"blacklisted_models\":[\"gpt-4o\"]}" + } + } + }, + { + "id": "rt-blacklist-gate-no-vk-03-route", + "name": "03. blacklisted model rejected by key gate (400) [blacklist-gate-no-vk]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [blacklist-gate-no-vk]\";", + "function assertNow() {", + " if (pm.response.code !== 400) { throw new Error('expected status 400 got ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " var msg = (body.error && (body.error.message || body.error)) || body.message || '';", + " if (String(msg).indexOf(\"no keys found that support model\") < 0) { throw new Error('error message=' + msg); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. blacklisted model rejected by key gate (400) [blacklist-gate-no-vk]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. blacklisted model rejected by key gate (400) [blacklist-gate-no-vk]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-blacklist-gate-no-vk-{{run_id}}/gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-blacklist-gate-no-vk-cleanup-provider-self", + "name": "cleanup: delete provider [blacklist-gate-no-vk]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [blacklist-gate-no-vk]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-blacklist-gate-no-vk-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-blacklist-gate-no-vk-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Disabled key makes a model unroutable with no governance", + "description": "Pure model-catalog case (no VK): the only key for a model is disabled, so key selection finds nothing.", + "item": [ + { + "id": "rt-disabled-key-gate-no-vk-01-add-provider", + "name": "01. add provider [disabled-key-gate-no-vk]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [disabled-key-gate-no-vk]\";", + "pm.test(\"01. add provider [disabled-key-gate-no-vk]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-disabled-key-gate-no-vk-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-disabled-key-gate-no-vk-02-add-key", + "name": "02. add key k1 [disabled-key-gate-no-vk]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [disabled-key-gate-no-vk]\";", + "pm.test(\"02. add key k1 [disabled-key-gate-no-vk]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-disabled-key-gate-no-vk-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-disabled-key-gate-no-vk-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"k1-{{run_id}}\",\"name\":\"catwiring-rt-disabled-key-gate-no-vk-k1-{{run_id}}\",\"models\":[\"*\"],\"enabled\":false,\"value\":\"env.OPENAI_API_KEY\"}" + } + } + }, + { + "id": "rt-disabled-key-gate-no-vk-03-route", + "name": "03. disabled key yields no route (400) [disabled-key-gate-no-vk]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [disabled-key-gate-no-vk]\";", + "function assertNow() {", + " if (pm.response.code !== 400) { throw new Error('expected status 400 got ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " var msg = (body.error && (body.error.message || body.error)) || body.message || '';", + " if (String(msg).indexOf(\"no keys found\") < 0) { throw new Error('error message=' + msg); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. disabled key yields no route (400) [disabled-key-gate-no-vk]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. disabled key yields no route (400) [disabled-key-gate-no-vk]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-disabled-key-gate-no-vk-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-disabled-key-gate-no-vk-cleanup-provider-self", + "name": "cleanup: delete provider [disabled-key-gate-no-vk]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [disabled-key-gate-no-vk]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-disabled-key-gate-no-vk-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-disabled-key-gate-no-vk-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Azure provider routes a deployment", + "description": "A standard azure provider (value + azure_key_config via env.) routes a model whose deployment matches the name. Serial-only (global standard provider).", + "item": [ + { + "id": "rt-route-azure-01-add-provider", + "name": "01. add provider [route-azure]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [route-azure]\";", + "pm.test(\"01. add provider [route-azure]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"azure\"}" + } + } + }, + { + "id": "rt-route-azure-02-add-key", + "name": "02. add key kaz [route-azure]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [route-azure]\";", + "pm.test(\"02. add key kaz [route-azure]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/azure/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "azure", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"kaz-{{run_id}}\",\"name\":\"catwiring-rt-route-azure-kaz-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.AZURE_API_KEY\",\"azure_key_config\":{\"endpoint\":\"env.AZURE_ENDPOINT\"}}" + } + } + }, + { + "id": "rt-route-azure-03-route", + "name": "03. azure routes its deployment (200) [route-azure]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 1000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [route-azure]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('route status ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " if (!body.choices || body.choices.length === 0) { throw new Error('no choices'); }", + " var ri = (body.extra_fields || {}).routing_info || {};", + " var providerName = \"azure\";", + " if (ri.provider !== providerName) { throw new Error('routing_info.provider=' + ri.provider + ' expected ' + providerName); }", + " var expectedKey = 'catwiring-rt-route-azure-kaz-' + pm.variables.get('run_id');", + " if (ri.key !== expectedKey) { throw new Error('routing_info.key=' + ri.key + ' expected ' + expectedKey); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. azure routes its deployment (200) [route-azure]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. azure routes its deployment (200) [route-azure]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"azure/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-route-azure-cleanup-provider-self", + "name": "cleanup: delete provider [route-azure]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [route-azure]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/azure", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "azure" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Bedrock routes Claude via a key alias to its inference profile", + "description": "A standard bedrock provider whose key aliases the common name claude-sonnet-4-5 to the cross-region inference-profile id. Routing the friendly name resolves to the wire id. Serial-only (global standard provider).", + "item": [ + { + "id": "rt-route-bedrock-01-add-provider", + "name": "01. add provider [route-bedrock]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [route-bedrock]\";", + "pm.test(\"01. add provider [route-bedrock]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"bedrock\"}" + } + } + }, + { + "id": "rt-route-bedrock-02-add-key", + "name": "02. add key kbr [route-bedrock]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [route-bedrock]\";", + "pm.test(\"02. add key kbr [route-bedrock]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/bedrock/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "bedrock", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"kbr-{{run_id}}\",\"name\":\"catwiring-rt-route-bedrock-kbr-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"bedrock_key_config\":{\"access_key\":\"env.AWS_ACCESS_KEY_ID\",\"secret_key\":\"env.AWS_SECRET_ACCESS_KEY\",\"region\":\"env.AWS_REGION\"},\"aliases\":{\"claude-sonnet-4-5\":\"us.anthropic.claude-sonnet-4-5-20250929-v1:0\"}}" + } + } + }, + { + "id": "rt-route-bedrock-03-route", + "name": "03. bedrock alias resolves to inference profile (200) [route-bedrock]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 1000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [route-bedrock]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('route status ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " if (!body.choices || body.choices.length === 0) { throw new Error('no choices'); }", + " var ri = (body.extra_fields || {}).routing_info || {};", + " var providerName = \"bedrock\";", + " if (ri.provider !== providerName) { throw new Error('routing_info.provider=' + ri.provider + ' expected ' + providerName); }", + " var expectedKey = 'catwiring-rt-route-bedrock-kbr-' + pm.variables.get('run_id');", + " if (ri.key !== expectedKey) { throw new Error('routing_info.key=' + ri.key + ' expected ' + expectedKey); }", + " if (!ri.resolved_key_alias || ri.resolved_key_alias.model_id !== \"us.anthropic.claude-sonnet-4-5-20250929-v1:0\") { throw new Error('resolved_key_alias=' + JSON.stringify(ri.resolved_key_alias)); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. bedrock alias resolves to inference profile (200) [route-bedrock]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. bedrock alias resolves to inference profile (200) [route-bedrock]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"bedrock/claude-sonnet-4-5\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-route-bedrock-cleanup-provider-self", + "name": "cleanup: delete provider [route-bedrock]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [route-bedrock]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/bedrock", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "bedrock" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Governance LBs one openai model across OpenAI and Azure", + "description": "OpenAI (custom) and Azure (standard) both serve gpt-4o-mini. A VK over both, routing the bare model, has governance distribute across them. Serial-only (Azure is a global standard provider).", + "item": [ + { + "id": "rt-cross-provider-same-model-openai-azure-01-add-provider", + "name": "01. add provider [cross-provider-same-model-openai-azure]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [cross-provider-same-model-openai-azure]\";", + "pm.test(\"01. add provider [cross-provider-same-model-openai-azure]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-cross-provider-same-model-openai-azure-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-cross-provider-same-model-openai-azure-02-add-key", + "name": "02. add key ko [cross-provider-same-model-openai-azure]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [cross-provider-same-model-openai-azure]\";", + "pm.test(\"02. add key ko [cross-provider-same-model-openai-azure]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-cross-provider-same-model-openai-azure-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-cross-provider-same-model-openai-azure-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"ko-{{run_id}}\",\"name\":\"catwiring-rt-cross-provider-same-model-openai-azure-ko-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\",\"weight\":1}" + } + } + }, + { + "id": "rt-cross-provider-same-model-openai-azure-03-add-provider", + "name": "03. add provider (az) [cross-provider-same-model-openai-azure]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [cross-provider-same-model-openai-azure]\";", + "pm.test(\"03. add provider (az) [cross-provider-same-model-openai-azure]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"azure\"}" + } + } + }, + { + "id": "rt-cross-provider-same-model-openai-azure-04-add-key", + "name": "04. add key kaz (az) [cross-provider-same-model-openai-azure]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [cross-provider-same-model-openai-azure]\";", + "pm.test(\"04. add key kaz (az) [cross-provider-same-model-openai-azure]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/azure/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "azure", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"kaz-{{run_id}}\",\"name\":\"catwiring-rt-cross-provider-same-model-openai-azure-kaz-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.AZURE_API_KEY\",\"azure_key_config\":{\"endpoint\":\"env.AZURE_ENDPOINT\"},\"weight\":1}" + } + } + }, + { + "id": "rt-cross-provider-same-model-openai-azure-05-create-vk", + "name": "05. create vk [cross-provider-same-model-openai-azure]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [cross-provider-same-model-openai-azure]\";", + "var ok = pm.response.code === 200 || pm.response.code === 201;", + "pm.test(\"05. create vk [cross-provider-same-model-openai-azure]\", function () {", + " pm.expect([200, 201], 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (ok) {", + " var vk = (pm.response.json() || {}).virtual_key || {};", + " pm.collectionVariables.set('vkval_cross-provider-same-model-openai-azure', vk.value || '');", + " pm.collectionVariables.set('vkid_cross-provider-same-model-openai-azure', vk.id || '');", + "} else { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"name\":\"catwiring-rtvk-cross-provider-same-model-openai-azure-{{run_id}}\",\"is_active\":true,\"provider_configs\":[{\"provider\":\"catwiring-rt-cross-provider-same-model-openai-azure-{{run_id}}\",\"key_ids\":[\"*\"],\"allowed_models\":[\"*\"],\"blacklisted_models\":[],\"weight\":1},{\"provider\":\"azure\",\"key_ids\":[\"*\"],\"allowed_models\":[\"*\"],\"blacklisted_models\":[],\"weight\":1}]}" + } + } + }, + { + "id": "rt-cross-provider-same-model-openai-azure-06-route", + "name": "06. bare model routes via openai or azure (200) [cross-provider-same-model-openai-azure]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 3000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [cross-provider-same-model-openai-azure]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('route status ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " if (!body.choices || body.choices.length === 0) { throw new Error('no choices'); }", + " var ri = (body.extra_fields || {}).routing_info || {};", + " var allowedProviders = ['catwiring-rt-cross-provider-same-model-openai-azure-' + pm.variables.get('run_id'), \"azure\"];", + " if (allowedProviders.indexOf(ri.provider) < 0) { throw new Error('routing_info.provider=' + ri.provider + ' not in ' + JSON.stringify(allowedProviders)); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"06. bare model routes via openai or azure (200) [cross-provider-same-model-openai-azure]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"06. bare model routes via openai or azure (200) [cross-provider-same-model-openai-azure]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_cross-provider-same-model-openai-azure}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-cross-provider-same-model-openai-azure-07-capture-log", + "name": "07. capture routing log id [cross-provider-same-model-openai-azure]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [cross-provider-same-model-openai-azure]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('logs status ' + pm.response.code); }", + " var logs = (pm.response.json() || {}).logs || [];", + " if (!logs.length || !logs[0].id) { throw new Error('no log row yet for VK'); }", + " pm.collectionVariables.set('logid_cross-provider-same-model-openai-azure', logs[0].id);", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"07. capture routing log id [cross-provider-same-model-openai-azure]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"07. capture routing log id [cross-provider-same-model-openai-azure]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/logs?virtual_key_ids={{vkid_cross-provider-same-model-openai-azure}}&limit=1", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "logs" + ], + "query": [ + { + "key": "virtual_key_ids", + "value": "{{vkid_cross-provider-same-model-openai-azure}}" + }, + { + "key": "limit", + "value": "1" + } + ] + } + } + }, + { + "id": "rt-cross-provider-same-model-openai-azure-08-assert-trail", + "name": "08. log detail records openai/azure LB trail [cross-provider-same-model-openai-azure]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [cross-provider-same-model-openai-azure]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('log detail status ' + pm.response.code); }", + " var row = pm.response.json() || {};", + " var trail = row.routing_engine_logs || '';", + " var expected = [\"Load balancing model gpt-4o-mini\",\"Selected provider\"];", + " expected.forEach(function (s) {", + " if (String(trail).indexOf(s) < 0) { throw new Error('routing_engine_logs missing ' + JSON.stringify(s) + '; got ' + JSON.stringify(trail)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"08. log detail records openai/azure LB trail [cross-provider-same-model-openai-azure]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"08. log detail records openai/azure LB trail [cross-provider-same-model-openai-azure]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/logs/{{logid_cross-provider-same-model-openai-azure}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "logs", + "{{logid_cross-provider-same-model-openai-azure}}" + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-cross-provider-same-model-openai-azure-cleanup-vk", + "name": "cleanup: delete vk [cross-provider-same-model-openai-azure]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete vk [cross-provider-same-model-openai-azure]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys/{{vkid_cross-provider-same-model-openai-azure}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys", + "{{vkid_cross-provider-same-model-openai-azure}}" + ] + } + } + }, + { + "id": "rt-cross-provider-same-model-openai-azure-cleanup-provider-self", + "name": "cleanup: delete provider [cross-provider-same-model-openai-azure]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [cross-provider-same-model-openai-azure]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-cross-provider-same-model-openai-azure-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-cross-provider-same-model-openai-azure-{{run_id}}" + ] + } + } + }, + { + "id": "rt-cross-provider-same-model-openai-azure-cleanup-provider-az", + "name": "cleanup: delete provider az [cross-provider-same-model-openai-azure]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider az [cross-provider-same-model-openai-azure]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/azure", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "azure" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Anthropic provider routes its model", + "description": "A custom provider backed by anthropic (key via env.) routes a claude model.", + "item": [ + { + "id": "rt-route-anthropic-01-add-provider", + "name": "01. add provider [route-anthropic]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [route-anthropic]\";", + "pm.test(\"01. add provider [route-anthropic]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-route-anthropic-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"anthropic\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-route-anthropic-02-add-key", + "name": "02. add key ka [route-anthropic]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [route-anthropic]\";", + "pm.test(\"02. add key ka [route-anthropic]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-route-anthropic-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-route-anthropic-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"ka-{{run_id}}\",\"name\":\"catwiring-rt-route-anthropic-ka-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.ANTHROPIC_API_KEY\"}" + } + } + }, + { + "id": "rt-route-anthropic-03-route", + "name": "03. anthropic provider routes claude model (200) [route-anthropic]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 1000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [route-anthropic]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('route status ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " if (!body.choices || body.choices.length === 0) { throw new Error('no choices'); }", + " var ri = (body.extra_fields || {}).routing_info || {};", + " var providerName = 'catwiring-rt-route-anthropic-' + pm.variables.get('run_id');", + " if (ri.provider !== providerName) { throw new Error('routing_info.provider=' + ri.provider + ' expected ' + providerName); }", + " var expectedKey = 'catwiring-rt-route-anthropic-ka-' + pm.variables.get('run_id');", + " if (ri.key !== expectedKey) { throw new Error('routing_info.key=' + ri.key + ' expected ' + expectedKey); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. anthropic provider routes claude model (200) [route-anthropic]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. anthropic provider routes claude model (200) [route-anthropic]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-route-anthropic-{{run_id}}/claude-haiku-4-5\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-route-anthropic-cleanup-provider-self", + "name": "cleanup: delete provider [route-anthropic]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [route-anthropic]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-route-anthropic-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-route-anthropic-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Gemini provider routes its model", + "description": "A custom provider backed by gemini (key via env.) routes a gemini model.", + "item": [ + { + "id": "rt-route-gemini-01-add-provider", + "name": "01. add provider [route-gemini]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [route-gemini]\";", + "pm.test(\"01. add provider [route-gemini]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-route-gemini-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"gemini\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-route-gemini-02-add-key", + "name": "02. add key kg [route-gemini]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [route-gemini]\";", + "pm.test(\"02. add key kg [route-gemini]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-route-gemini-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-route-gemini-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"kg-{{run_id}}\",\"name\":\"catwiring-rt-route-gemini-kg-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.GEMINI_API_KEY\"}" + } + } + }, + { + "id": "rt-route-gemini-03-route", + "name": "03. gemini provider routes gemini model (200) [route-gemini]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 1000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [route-gemini]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('route status ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " if (!body.choices || body.choices.length === 0) { throw new Error('no choices'); }", + " var ri = (body.extra_fields || {}).routing_info || {};", + " var providerName = 'catwiring-rt-route-gemini-' + pm.variables.get('run_id');", + " if (ri.provider !== providerName) { throw new Error('routing_info.provider=' + ri.provider + ' expected ' + providerName); }", + " var expectedKey = 'catwiring-rt-route-gemini-kg-' + pm.variables.get('run_id');", + " if (ri.key !== expectedKey) { throw new Error('routing_info.key=' + ri.key + ' expected ' + expectedKey); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. gemini provider routes gemini model (200) [route-gemini]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. gemini provider routes gemini model (200) [route-gemini]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-route-gemini-{{run_id}}/gemini-2.5-flash\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-route-gemini-cleanup-provider-self", + "name": "cleanup: delete provider [route-gemini]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [route-gemini]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-route-gemini-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-route-gemini-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "VK explicit allowed-model the key can't serve → 400 (not 403)", + "description": "Unlike the wildcard case (which 403s via the catalog-aware check), an EXPLICIT allowed_models entry is string-matched by governance and passes; key selection then fails → 400 no keys. Pins the explicit-vs-wildcard split: explicit lists are string-matched by governance, wildcards go through the catalog-aware check.", + "item": [ + { + "id": "rt-vk-explicit-allowed-key-cant-serve-01-add-provider", + "name": "01. add provider [vk-explicit-allowed-key-cant-serve]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [vk-explicit-allowed-key-cant-serve]\";", + "pm.test(\"01. add provider [vk-explicit-allowed-key-cant-serve]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-vk-explicit-allowed-key-cant-serve-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-vk-explicit-allowed-key-cant-serve-02-add-key", + "name": "02. add key k1 [vk-explicit-allowed-key-cant-serve]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [vk-explicit-allowed-key-cant-serve]\";", + "pm.test(\"02. add key k1 [vk-explicit-allowed-key-cant-serve]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-vk-explicit-allowed-key-cant-serve-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-vk-explicit-allowed-key-cant-serve-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"k1-{{run_id}}\",\"name\":\"catwiring-rt-vk-explicit-allowed-key-cant-serve-k1-{{run_id}}\",\"models\":[\"gpt-4o-mini\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\"}" + } + } + }, + { + "id": "rt-vk-explicit-allowed-key-cant-serve-03-create-vk", + "name": "03. create vk [vk-explicit-allowed-key-cant-serve]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [vk-explicit-allowed-key-cant-serve]\";", + "var ok = pm.response.code === 200 || pm.response.code === 201;", + "pm.test(\"03. create vk [vk-explicit-allowed-key-cant-serve]\", function () {", + " pm.expect([200, 201], 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (ok) {", + " var vk = (pm.response.json() || {}).virtual_key || {};", + " pm.collectionVariables.set('vkval_vk-explicit-allowed-key-cant-serve', vk.value || '');", + " pm.collectionVariables.set('vkid_vk-explicit-allowed-key-cant-serve', vk.id || '');", + "} else { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"name\":\"catwiring-rtvk-vk-explicit-allowed-key-cant-serve-{{run_id}}\",\"is_active\":true,\"provider_configs\":[{\"provider\":\"catwiring-rt-vk-explicit-allowed-key-cant-serve-{{run_id}}\",\"key_ids\":[\"*\"],\"allowed_models\":[\"gpt-4o\"],\"blacklisted_models\":[],\"weight\":1}]}" + } + } + }, + { + "id": "rt-vk-explicit-allowed-key-cant-serve-04-route", + "name": "04. explicit allowed model the key can't serve → 400 [vk-explicit-allowed-key-cant-serve]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [vk-explicit-allowed-key-cant-serve]\";", + "function assertNow() {", + " if (pm.response.code !== 400) { throw new Error('expected status 400 got ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " var msg = (body.error && (body.error.message || body.error)) || body.message || '';", + " if (String(msg).indexOf(\"no keys found that support model\") < 0) { throw new Error('error message=' + msg); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. explicit allowed model the key can't serve → 400 [vk-explicit-allowed-key-cant-serve]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. explicit allowed model the key can't serve → 400 [vk-explicit-allowed-key-cant-serve]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_vk-explicit-allowed-key-cant-serve}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-vk-explicit-allowed-key-cant-serve-{{run_id}}/gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-vk-explicit-allowed-key-cant-serve-cleanup-vk", + "name": "cleanup: delete vk [vk-explicit-allowed-key-cant-serve]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete vk [vk-explicit-allowed-key-cant-serve]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys/{{vkid_vk-explicit-allowed-key-cant-serve}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys", + "{{vkid_vk-explicit-allowed-key-cant-serve}}" + ] + } + } + }, + { + "id": "rt-vk-explicit-allowed-key-cant-serve-cleanup-provider-self", + "name": "cleanup: delete provider [vk-explicit-allowed-key-cant-serve]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [vk-explicit-allowed-key-cant-serve]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-vk-explicit-allowed-key-cant-serve-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-vk-explicit-allowed-key-cant-serve-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Within a provider, the capable key wins over a higher-weight incapable key", + "description": "Pure model-catalog/core (no VK): k1 weight 99 allows only gpt-4o; k2 weight 1 allows gpt-4o-mini. Routing gpt-4o-mini always uses k2 — key capability filters before weighting.", + "item": [ + { + "id": "rt-key-gate-beats-key-weight-01-add-provider", + "name": "01. add provider [key-gate-beats-key-weight]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [key-gate-beats-key-weight]\";", + "pm.test(\"01. add provider [key-gate-beats-key-weight]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-key-gate-beats-key-weight-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-key-gate-beats-key-weight-02-add-key", + "name": "02. add key k1 [key-gate-beats-key-weight]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [key-gate-beats-key-weight]\";", + "pm.test(\"02. add key k1 [key-gate-beats-key-weight]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-key-gate-beats-key-weight-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-key-gate-beats-key-weight-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"k1-{{run_id}}\",\"name\":\"catwiring-rt-key-gate-beats-key-weight-k1-{{run_id}}\",\"models\":[\"gpt-4o\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\",\"weight\":99}" + } + } + }, + { + "id": "rt-key-gate-beats-key-weight-03-add-key", + "name": "03. add key k2 [key-gate-beats-key-weight]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [key-gate-beats-key-weight]\";", + "pm.test(\"03. add key k2 [key-gate-beats-key-weight]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-key-gate-beats-key-weight-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-key-gate-beats-key-weight-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"k2-{{run_id}}\",\"name\":\"catwiring-rt-key-gate-beats-key-weight-k2-{{run_id}}\",\"models\":[\"gpt-4o-mini\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\",\"weight\":1}" + } + } + }, + { + "id": "rt-key-gate-beats-key-weight-04-dist-sample", + "name": "04. sample 1/10 (gpt-4o-mini) [key-gate-beats-key-weight]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [key-gate-beats-key-weight]\";", + "pm.collectionVariables.set('dist_key-gate-beats-key-weight', '');", + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + " var cur = pm.collectionVariables.get('dist_key-gate-beats-key-weight') || '';", + " var set = cur ? cur.split(',') : [];", + " if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_key-gate-beats-key-weight', set.join(',')); }", + "}", + "pm.test(\"04. sample 1/10 (gpt-4o-mini) [key-gate-beats-key-weight]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-key-gate-beats-key-weight-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-key-gate-beats-key-weight-05-dist-sample", + "name": "05. sample 2/10 (gpt-4o-mini) [key-gate-beats-key-weight]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [key-gate-beats-key-weight]\";", + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + " var cur = pm.collectionVariables.get('dist_key-gate-beats-key-weight') || '';", + " var set = cur ? cur.split(',') : [];", + " if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_key-gate-beats-key-weight', set.join(',')); }", + "}", + "pm.test(\"05. sample 2/10 (gpt-4o-mini) [key-gate-beats-key-weight]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-key-gate-beats-key-weight-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-key-gate-beats-key-weight-06-dist-sample", + "name": "06. sample 3/10 (gpt-4o-mini) [key-gate-beats-key-weight]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [key-gate-beats-key-weight]\";", + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + " var cur = pm.collectionVariables.get('dist_key-gate-beats-key-weight') || '';", + " var set = cur ? cur.split(',') : [];", + " if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_key-gate-beats-key-weight', set.join(',')); }", + "}", + "pm.test(\"06. sample 3/10 (gpt-4o-mini) [key-gate-beats-key-weight]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-key-gate-beats-key-weight-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-key-gate-beats-key-weight-07-dist-sample", + "name": "07. sample 4/10 (gpt-4o-mini) [key-gate-beats-key-weight]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [key-gate-beats-key-weight]\";", + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + " var cur = pm.collectionVariables.get('dist_key-gate-beats-key-weight') || '';", + " var set = cur ? cur.split(',') : [];", + " if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_key-gate-beats-key-weight', set.join(',')); }", + "}", + "pm.test(\"07. sample 4/10 (gpt-4o-mini) [key-gate-beats-key-weight]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-key-gate-beats-key-weight-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-key-gate-beats-key-weight-08-dist-sample", + "name": "08. sample 5/10 (gpt-4o-mini) [key-gate-beats-key-weight]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [key-gate-beats-key-weight]\";", + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + " var cur = pm.collectionVariables.get('dist_key-gate-beats-key-weight') || '';", + " var set = cur ? cur.split(',') : [];", + " if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_key-gate-beats-key-weight', set.join(',')); }", + "}", + "pm.test(\"08. sample 5/10 (gpt-4o-mini) [key-gate-beats-key-weight]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-key-gate-beats-key-weight-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-key-gate-beats-key-weight-09-dist-sample", + "name": "09. sample 6/10 (gpt-4o-mini) [key-gate-beats-key-weight]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [key-gate-beats-key-weight]\";", + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + " var cur = pm.collectionVariables.get('dist_key-gate-beats-key-weight') || '';", + " var set = cur ? cur.split(',') : [];", + " if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_key-gate-beats-key-weight', set.join(',')); }", + "}", + "pm.test(\"09. sample 6/10 (gpt-4o-mini) [key-gate-beats-key-weight]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-key-gate-beats-key-weight-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-key-gate-beats-key-weight-10-dist-sample", + "name": "10. sample 7/10 (gpt-4o-mini) [key-gate-beats-key-weight]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [key-gate-beats-key-weight]\";", + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + " var cur = pm.collectionVariables.get('dist_key-gate-beats-key-weight') || '';", + " var set = cur ? cur.split(',') : [];", + " if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_key-gate-beats-key-weight', set.join(',')); }", + "}", + "pm.test(\"10. sample 7/10 (gpt-4o-mini) [key-gate-beats-key-weight]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-key-gate-beats-key-weight-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-key-gate-beats-key-weight-11-dist-sample", + "name": "11. sample 8/10 (gpt-4o-mini) [key-gate-beats-key-weight]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [key-gate-beats-key-weight]\";", + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + " var cur = pm.collectionVariables.get('dist_key-gate-beats-key-weight') || '';", + " var set = cur ? cur.split(',') : [];", + " if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_key-gate-beats-key-weight', set.join(',')); }", + "}", + "pm.test(\"11. sample 8/10 (gpt-4o-mini) [key-gate-beats-key-weight]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-key-gate-beats-key-weight-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-key-gate-beats-key-weight-12-dist-sample", + "name": "12. sample 9/10 (gpt-4o-mini) [key-gate-beats-key-weight]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [key-gate-beats-key-weight]\";", + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + " var cur = pm.collectionVariables.get('dist_key-gate-beats-key-weight') || '';", + " var set = cur ? cur.split(',') : [];", + " if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_key-gate-beats-key-weight', set.join(',')); }", + "}", + "pm.test(\"12. sample 9/10 (gpt-4o-mini) [key-gate-beats-key-weight]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-key-gate-beats-key-weight-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-key-gate-beats-key-weight-13-dist-sample", + "name": "13. sample 10/10 (gpt-4o-mini) [key-gate-beats-key-weight]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [key-gate-beats-key-weight]\";", + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + " var cur = pm.collectionVariables.get('dist_key-gate-beats-key-weight') || '';", + " var set = cur ? cur.split(',') : [];", + " if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_key-gate-beats-key-weight', set.join(',')); }", + "}", + "pm.test(\"13. sample 10/10 (gpt-4o-mini) [key-gate-beats-key-weight]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-key-gate-beats-key-weight-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-key-gate-beats-key-weight-14-dist-assert", + "name": "14. all requests use the capable 1%-weight key (k2) [key-gate-beats-key-weight]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [key-gate-beats-key-weight]\";", + "var ok = true, errMsg = '';", + "try {", + " var seen = (pm.collectionVariables.get('dist_key-gate-beats-key-weight') || '').split(',').filter(Boolean);", + " var only = ['catwiring-rt-key-gate-beats-key-weight-k2-' + pm.variables.get('run_id')];", + " if (seen.length === 0) throw new Error('no key observed across samples');", + " seen.forEach(function (e) { if (only.indexOf(e) < 0) throw new Error('key ' + e + ' served but not in expectOnly ' + JSON.stringify(only)); });", + " var never = ['catwiring-rt-key-gate-beats-key-weight-k1-' + pm.variables.get('run_id')];", + " seen.forEach(function (e) { if (never.indexOf(e) >= 0) throw new Error('key ' + e + ' served but was expectNever; observed ' + JSON.stringify(seen)); });", + "} catch (e) { ok = false; errMsg = e.message; }", + "pm.test(\"14. all requests use the capable 1%-weight key (k2) [key-gate-beats-key-weight]\", function () { if (!ok) throw new Error(errMsg); });", + "if (!ok) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/health", + "host": [ + "{{base_url}}" + ], + "path": [ + "health" + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-key-gate-beats-key-weight-cleanup-provider-self", + "name": "cleanup: delete provider [key-gate-beats-key-weight]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [key-gate-beats-key-weight]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-key-gate-beats-key-weight-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-key-gate-beats-key-weight-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "A model blacklisted on one key still routes via a sibling key", + "description": "Pure model-catalog/core (no VK): k1 allows all but blacklists gpt-4o-mini; k2 allows all. The model is blocked only on k1, so the provider still serves it via k2 (blacklist is per-key, not provider-wide unless all keys block).", + "item": [ + { + "id": "rt-key-blacklist-intersection-01-add-provider", + "name": "01. add provider [key-blacklist-intersection]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [key-blacklist-intersection]\";", + "pm.test(\"01. add provider [key-blacklist-intersection]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-key-blacklist-intersection-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-key-blacklist-intersection-02-add-key", + "name": "02. add key k1 [key-blacklist-intersection]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [key-blacklist-intersection]\";", + "pm.test(\"02. add key k1 [key-blacklist-intersection]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-key-blacklist-intersection-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-key-blacklist-intersection-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"k1-{{run_id}}\",\"name\":\"catwiring-rt-key-blacklist-intersection-k1-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\",\"blacklisted_models\":[\"gpt-4o-mini\"]}" + } + } + }, + { + "id": "rt-key-blacklist-intersection-03-add-key", + "name": "03. add key k2 [key-blacklist-intersection]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [key-blacklist-intersection]\";", + "pm.test(\"03. add key k2 [key-blacklist-intersection]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-key-blacklist-intersection-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-key-blacklist-intersection-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"k2-{{run_id}}\",\"name\":\"catwiring-rt-key-blacklist-intersection-k2-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\"}" + } + } + }, + { + "id": "rt-key-blacklist-intersection-04-dist-sample", + "name": "04. sample 1/10 (gpt-4o-mini) [key-blacklist-intersection]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [key-blacklist-intersection]\";", + "pm.collectionVariables.set('dist_key-blacklist-intersection', '');", + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + " var cur = pm.collectionVariables.get('dist_key-blacklist-intersection') || '';", + " var set = cur ? cur.split(',') : [];", + " if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_key-blacklist-intersection', set.join(',')); }", + "}", + "pm.test(\"04. sample 1/10 (gpt-4o-mini) [key-blacklist-intersection]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-key-blacklist-intersection-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-key-blacklist-intersection-05-dist-sample", + "name": "05. sample 2/10 (gpt-4o-mini) [key-blacklist-intersection]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [key-blacklist-intersection]\";", + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + " var cur = pm.collectionVariables.get('dist_key-blacklist-intersection') || '';", + " var set = cur ? cur.split(',') : [];", + " if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_key-blacklist-intersection', set.join(',')); }", + "}", + "pm.test(\"05. sample 2/10 (gpt-4o-mini) [key-blacklist-intersection]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-key-blacklist-intersection-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-key-blacklist-intersection-06-dist-sample", + "name": "06. sample 3/10 (gpt-4o-mini) [key-blacklist-intersection]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [key-blacklist-intersection]\";", + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + " var cur = pm.collectionVariables.get('dist_key-blacklist-intersection') || '';", + " var set = cur ? cur.split(',') : [];", + " if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_key-blacklist-intersection', set.join(',')); }", + "}", + "pm.test(\"06. sample 3/10 (gpt-4o-mini) [key-blacklist-intersection]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-key-blacklist-intersection-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-key-blacklist-intersection-07-dist-sample", + "name": "07. sample 4/10 (gpt-4o-mini) [key-blacklist-intersection]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [key-blacklist-intersection]\";", + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + " var cur = pm.collectionVariables.get('dist_key-blacklist-intersection') || '';", + " var set = cur ? cur.split(',') : [];", + " if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_key-blacklist-intersection', set.join(',')); }", + "}", + "pm.test(\"07. sample 4/10 (gpt-4o-mini) [key-blacklist-intersection]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-key-blacklist-intersection-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-key-blacklist-intersection-08-dist-sample", + "name": "08. sample 5/10 (gpt-4o-mini) [key-blacklist-intersection]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [key-blacklist-intersection]\";", + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + " var cur = pm.collectionVariables.get('dist_key-blacklist-intersection') || '';", + " var set = cur ? cur.split(',') : [];", + " if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_key-blacklist-intersection', set.join(',')); }", + "}", + "pm.test(\"08. sample 5/10 (gpt-4o-mini) [key-blacklist-intersection]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-key-blacklist-intersection-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-key-blacklist-intersection-09-dist-sample", + "name": "09. sample 6/10 (gpt-4o-mini) [key-blacklist-intersection]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [key-blacklist-intersection]\";", + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + " var cur = pm.collectionVariables.get('dist_key-blacklist-intersection') || '';", + " var set = cur ? cur.split(',') : [];", + " if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_key-blacklist-intersection', set.join(',')); }", + "}", + "pm.test(\"09. sample 6/10 (gpt-4o-mini) [key-blacklist-intersection]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-key-blacklist-intersection-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-key-blacklist-intersection-10-dist-sample", + "name": "10. sample 7/10 (gpt-4o-mini) [key-blacklist-intersection]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [key-blacklist-intersection]\";", + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + " var cur = pm.collectionVariables.get('dist_key-blacklist-intersection') || '';", + " var set = cur ? cur.split(',') : [];", + " if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_key-blacklist-intersection', set.join(',')); }", + "}", + "pm.test(\"10. sample 7/10 (gpt-4o-mini) [key-blacklist-intersection]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-key-blacklist-intersection-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-key-blacklist-intersection-11-dist-sample", + "name": "11. sample 8/10 (gpt-4o-mini) [key-blacklist-intersection]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [key-blacklist-intersection]\";", + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + " var cur = pm.collectionVariables.get('dist_key-blacklist-intersection') || '';", + " var set = cur ? cur.split(',') : [];", + " if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_key-blacklist-intersection', set.join(',')); }", + "}", + "pm.test(\"11. sample 8/10 (gpt-4o-mini) [key-blacklist-intersection]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-key-blacklist-intersection-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-key-blacklist-intersection-12-dist-sample", + "name": "12. sample 9/10 (gpt-4o-mini) [key-blacklist-intersection]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [key-blacklist-intersection]\";", + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + " var cur = pm.collectionVariables.get('dist_key-blacklist-intersection') || '';", + " var set = cur ? cur.split(',') : [];", + " if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_key-blacklist-intersection', set.join(',')); }", + "}", + "pm.test(\"12. sample 9/10 (gpt-4o-mini) [key-blacklist-intersection]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-key-blacklist-intersection-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-key-blacklist-intersection-13-dist-sample", + "name": "13. sample 10/10 (gpt-4o-mini) [key-blacklist-intersection]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [key-blacklist-intersection]\";", + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + " var cur = pm.collectionVariables.get('dist_key-blacklist-intersection') || '';", + " var set = cur ? cur.split(',') : [];", + " if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_key-blacklist-intersection', set.join(',')); }", + "}", + "pm.test(\"13. sample 10/10 (gpt-4o-mini) [key-blacklist-intersection]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-key-blacklist-intersection-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-key-blacklist-intersection-14-dist-assert", + "name": "14. model routes via the non-blacklisting sibling (k2) [key-blacklist-intersection]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete provider [key-blacklist-intersection]\";", + "var ok = true, errMsg = '';", + "try {", + " var seen = (pm.collectionVariables.get('dist_key-blacklist-intersection') || '').split(',').filter(Boolean);", + " var only = ['catwiring-rt-key-blacklist-intersection-k2-' + pm.variables.get('run_id')];", + " if (seen.length === 0) throw new Error('no key observed across samples');", + " seen.forEach(function (e) { if (only.indexOf(e) < 0) throw new Error('key ' + e + ' served but not in expectOnly ' + JSON.stringify(only)); });", + " var never = ['catwiring-rt-key-blacklist-intersection-k1-' + pm.variables.get('run_id')];", + " seen.forEach(function (e) { if (never.indexOf(e) >= 0) throw new Error('key ' + e + ' served but was expectNever; observed ' + JSON.stringify(seen)); });", + "} catch (e) { ok = false; errMsg = e.message; }", + "pm.test(\"14. model routes via the non-blacklisting sibling (k2) [key-blacklist-intersection]\", function () { if (!ok) throw new Error(errMsg); });", + "if (!ok) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/health", + "host": [ + "{{base_url}}" + ], + "path": [ + "health" + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-key-blacklist-intersection-cleanup-provider-self", + "name": "cleanup: delete provider [key-blacklist-intersection]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [key-blacklist-intersection]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-key-blacklist-intersection-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-key-blacklist-intersection-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "LB excludes only the incapable provider; the rest still split", + "description": "Three providers on a VK: A can't serve the model (key gate), B and C can. Routing the bare model, A is excluded and B/C both still serve over the batch.", + "item": [ + { + "id": "rt-lb-partial-exclusion-3-providers-01-add-provider", + "name": "01. add provider [lb-partial-exclusion-3-providers]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [lb-partial-exclusion-3-providers]\";", + "pm.test(\"01. add provider [lb-partial-exclusion-3-providers]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-lb-partial-exclusion-3-providers-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-lb-partial-exclusion-3-providers-02-add-key", + "name": "02. add key ka [lb-partial-exclusion-3-providers]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [lb-partial-exclusion-3-providers]\";", + "pm.test(\"02. add key ka [lb-partial-exclusion-3-providers]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-lb-partial-exclusion-3-providers-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-lb-partial-exclusion-3-providers-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"ka-{{run_id}}\",\"name\":\"catwiring-rt-lb-partial-exclusion-3-providers-ka-{{run_id}}\",\"models\":[\"gpt-4o\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\",\"weight\":1}" + } + } + }, + { + "id": "rt-lb-partial-exclusion-3-providers-03-add-provider", + "name": "03. add provider (b) [lb-partial-exclusion-3-providers]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [lb-partial-exclusion-3-providers]\";", + "pm.test(\"03. add provider (b) [lb-partial-exclusion-3-providers]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-lb-partial-exclusion-3-providers-b-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-lb-partial-exclusion-3-providers-04-add-key", + "name": "04. add key kb (b) [lb-partial-exclusion-3-providers]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [lb-partial-exclusion-3-providers]\";", + "pm.test(\"04. add key kb (b) [lb-partial-exclusion-3-providers]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-lb-partial-exclusion-3-providers-b-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-lb-partial-exclusion-3-providers-b-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"kb-{{run_id}}\",\"name\":\"catwiring-rt-lb-partial-exclusion-3-providers-kb-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\",\"weight\":1}" + } + } + }, + { + "id": "rt-lb-partial-exclusion-3-providers-05-add-provider", + "name": "05. add provider (c) [lb-partial-exclusion-3-providers]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [lb-partial-exclusion-3-providers]\";", + "pm.test(\"05. add provider (c) [lb-partial-exclusion-3-providers]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-lb-partial-exclusion-3-providers-c-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-lb-partial-exclusion-3-providers-06-add-key", + "name": "06. add key kc (c) [lb-partial-exclusion-3-providers]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [lb-partial-exclusion-3-providers]\";", + "pm.test(\"06. add key kc (c) [lb-partial-exclusion-3-providers]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-lb-partial-exclusion-3-providers-c-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-lb-partial-exclusion-3-providers-c-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"kc-{{run_id}}\",\"name\":\"catwiring-rt-lb-partial-exclusion-3-providers-kc-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\",\"weight\":1}" + } + } + }, + { + "id": "rt-lb-partial-exclusion-3-providers-07-create-vk", + "name": "07. create vk [lb-partial-exclusion-3-providers]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-partial-exclusion-3-providers]\";", + "var ok = pm.response.code === 200 || pm.response.code === 201;", + "pm.test(\"07. create vk [lb-partial-exclusion-3-providers]\", function () {", + " pm.expect([200, 201], 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (ok) {", + " var vk = (pm.response.json() || {}).virtual_key || {};", + " pm.collectionVariables.set('vkval_lb-partial-exclusion-3-providers', vk.value || '');", + " pm.collectionVariables.set('vkid_lb-partial-exclusion-3-providers', vk.id || '');", + "} else { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"name\":\"catwiring-rtvk-lb-partial-exclusion-3-providers-{{run_id}}\",\"is_active\":true,\"provider_configs\":[{\"provider\":\"catwiring-rt-lb-partial-exclusion-3-providers-{{run_id}}\",\"key_ids\":[\"*\"],\"allowed_models\":[\"*\"],\"blacklisted_models\":[],\"weight\":1},{\"provider\":\"catwiring-rt-lb-partial-exclusion-3-providers-b-{{run_id}}\",\"key_ids\":[\"*\"],\"allowed_models\":[\"*\"],\"blacklisted_models\":[],\"weight\":1},{\"provider\":\"catwiring-rt-lb-partial-exclusion-3-providers-c-{{run_id}}\",\"key_ids\":[\"*\"],\"allowed_models\":[\"*\"],\"blacklisted_models\":[],\"weight\":1}]}" + } + } + }, + { + "id": "rt-lb-partial-exclusion-3-providers-08-pdist-sample", + "name": "08. sample 1/12 (gpt-4o-mini) [lb-partial-exclusion-3-providers]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-partial-exclusion-3-providers]\";", + "pm.collectionVariables.set('pdist_lb-partial-exclusion-3-providers', '');", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-partial-exclusion-3-providers') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-partial-exclusion-3-providers', set.join(',')); }", + "}", + "pm.test(\"08. sample 1/12 (gpt-4o-mini) [lb-partial-exclusion-3-providers]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-partial-exclusion-3-providers}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-partial-exclusion-3-providers-09-pdist-sample", + "name": "09. sample 2/12 (gpt-4o-mini) [lb-partial-exclusion-3-providers]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-partial-exclusion-3-providers]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-partial-exclusion-3-providers') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-partial-exclusion-3-providers', set.join(',')); }", + "}", + "pm.test(\"09. sample 2/12 (gpt-4o-mini) [lb-partial-exclusion-3-providers]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-partial-exclusion-3-providers}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-partial-exclusion-3-providers-10-pdist-sample", + "name": "10. sample 3/12 (gpt-4o-mini) [lb-partial-exclusion-3-providers]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-partial-exclusion-3-providers]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-partial-exclusion-3-providers') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-partial-exclusion-3-providers', set.join(',')); }", + "}", + "pm.test(\"10. sample 3/12 (gpt-4o-mini) [lb-partial-exclusion-3-providers]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-partial-exclusion-3-providers}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-partial-exclusion-3-providers-11-pdist-sample", + "name": "11. sample 4/12 (gpt-4o-mini) [lb-partial-exclusion-3-providers]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-partial-exclusion-3-providers]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-partial-exclusion-3-providers') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-partial-exclusion-3-providers', set.join(',')); }", + "}", + "pm.test(\"11. sample 4/12 (gpt-4o-mini) [lb-partial-exclusion-3-providers]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-partial-exclusion-3-providers}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-partial-exclusion-3-providers-12-pdist-sample", + "name": "12. sample 5/12 (gpt-4o-mini) [lb-partial-exclusion-3-providers]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-partial-exclusion-3-providers]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-partial-exclusion-3-providers') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-partial-exclusion-3-providers', set.join(',')); }", + "}", + "pm.test(\"12. sample 5/12 (gpt-4o-mini) [lb-partial-exclusion-3-providers]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-partial-exclusion-3-providers}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-partial-exclusion-3-providers-13-pdist-sample", + "name": "13. sample 6/12 (gpt-4o-mini) [lb-partial-exclusion-3-providers]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-partial-exclusion-3-providers]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-partial-exclusion-3-providers') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-partial-exclusion-3-providers', set.join(',')); }", + "}", + "pm.test(\"13. sample 6/12 (gpt-4o-mini) [lb-partial-exclusion-3-providers]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-partial-exclusion-3-providers}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-partial-exclusion-3-providers-14-pdist-sample", + "name": "14. sample 7/12 (gpt-4o-mini) [lb-partial-exclusion-3-providers]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-partial-exclusion-3-providers]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-partial-exclusion-3-providers') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-partial-exclusion-3-providers', set.join(',')); }", + "}", + "pm.test(\"14. sample 7/12 (gpt-4o-mini) [lb-partial-exclusion-3-providers]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-partial-exclusion-3-providers}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-partial-exclusion-3-providers-15-pdist-sample", + "name": "15. sample 8/12 (gpt-4o-mini) [lb-partial-exclusion-3-providers]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-partial-exclusion-3-providers]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-partial-exclusion-3-providers') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-partial-exclusion-3-providers', set.join(',')); }", + "}", + "pm.test(\"15. sample 8/12 (gpt-4o-mini) [lb-partial-exclusion-3-providers]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-partial-exclusion-3-providers}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-partial-exclusion-3-providers-16-pdist-sample", + "name": "16. sample 9/12 (gpt-4o-mini) [lb-partial-exclusion-3-providers]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-partial-exclusion-3-providers]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-partial-exclusion-3-providers') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-partial-exclusion-3-providers', set.join(',')); }", + "}", + "pm.test(\"16. sample 9/12 (gpt-4o-mini) [lb-partial-exclusion-3-providers]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-partial-exclusion-3-providers}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-partial-exclusion-3-providers-17-pdist-sample", + "name": "17. sample 10/12 (gpt-4o-mini) [lb-partial-exclusion-3-providers]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-partial-exclusion-3-providers]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-partial-exclusion-3-providers') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-partial-exclusion-3-providers', set.join(',')); }", + "}", + "pm.test(\"17. sample 10/12 (gpt-4o-mini) [lb-partial-exclusion-3-providers]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-partial-exclusion-3-providers}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-partial-exclusion-3-providers-18-pdist-sample", + "name": "18. sample 11/12 (gpt-4o-mini) [lb-partial-exclusion-3-providers]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-partial-exclusion-3-providers]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-partial-exclusion-3-providers') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-partial-exclusion-3-providers', set.join(',')); }", + "}", + "pm.test(\"18. sample 11/12 (gpt-4o-mini) [lb-partial-exclusion-3-providers]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-partial-exclusion-3-providers}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-partial-exclusion-3-providers-19-pdist-sample", + "name": "19. sample 12/12 (gpt-4o-mini) [lb-partial-exclusion-3-providers]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-partial-exclusion-3-providers]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-partial-exclusion-3-providers') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-partial-exclusion-3-providers', set.join(',')); }", + "}", + "pm.test(\"19. sample 12/12 (gpt-4o-mini) [lb-partial-exclusion-3-providers]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-partial-exclusion-3-providers}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-partial-exclusion-3-providers-20-pdist-assert", + "name": "20. incapable provider excluded; B and C both serve [lb-partial-exclusion-3-providers]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-partial-exclusion-3-providers]\";", + "var ok = true, errMsg = '';", + "try {", + " var seen = (pm.collectionVariables.get('pdist_lb-partial-exclusion-3-providers') || '').split(',').filter(Boolean);", + " var only = ['catwiring-rt-lb-partial-exclusion-3-providers-b-' + pm.variables.get('run_id'), 'catwiring-rt-lb-partial-exclusion-3-providers-c-' + pm.variables.get('run_id')];", + " var never = ['catwiring-rt-lb-partial-exclusion-3-providers-' + pm.variables.get('run_id')];", + " var all = ['catwiring-rt-lb-partial-exclusion-3-providers-b-' + pm.variables.get('run_id'), 'catwiring-rt-lb-partial-exclusion-3-providers-c-' + pm.variables.get('run_id')];", + " if (seen.length === 0) throw new Error('no provider observed across samples');", + " seen.forEach(function (p) {", + " if (only.length && only.indexOf(p) < 0) throw new Error('provider ' + p + ' served but not in expectOnly ' + JSON.stringify(only));", + " if (never.indexOf(p) >= 0) throw new Error('provider ' + p + ' served but was expectNever; observed ' + JSON.stringify(seen));", + " });", + " all.forEach(function (p) { if (seen.indexOf(p) < 0) throw new Error('provider ' + p + ' expected to serve but did not; observed ' + JSON.stringify(seen)); });", + "} catch (e) { ok = false; errMsg = e.message; }", + "pm.test(\"20. incapable provider excluded; B and C both serve [lb-partial-exclusion-3-providers]\", function () { if (!ok) throw new Error(errMsg); });", + "if (!ok) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/health", + "host": [ + "{{base_url}}" + ], + "path": [ + "health" + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-lb-partial-exclusion-3-providers-cleanup-vk", + "name": "cleanup: delete vk [lb-partial-exclusion-3-providers]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete vk [lb-partial-exclusion-3-providers]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys/{{vkid_lb-partial-exclusion-3-providers}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys", + "{{vkid_lb-partial-exclusion-3-providers}}" + ] + } + } + }, + { + "id": "rt-lb-partial-exclusion-3-providers-cleanup-provider-self", + "name": "cleanup: delete provider [lb-partial-exclusion-3-providers]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [lb-partial-exclusion-3-providers]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-lb-partial-exclusion-3-providers-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-lb-partial-exclusion-3-providers-{{run_id}}" + ] + } + } + }, + { + "id": "rt-lb-partial-exclusion-3-providers-cleanup-provider-b", + "name": "cleanup: delete provider b [lb-partial-exclusion-3-providers]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider b [lb-partial-exclusion-3-providers]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-lb-partial-exclusion-3-providers-b-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-lb-partial-exclusion-3-providers-b-{{run_id}}" + ] + } + } + }, + { + "id": "rt-lb-partial-exclusion-3-providers-cleanup-provider-c", + "name": "cleanup: delete provider c [lb-partial-exclusion-3-providers]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider c [lb-partial-exclusion-3-providers]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-lb-partial-exclusion-3-providers-c-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-lb-partial-exclusion-3-providers-c-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "LB skips a 99%-weight provider that can't serve the model (key gate)", + "description": "A is weighted 99% but its key only allows gpt-4o; B is weighted 1% and allows all. Routing the bare gpt-4o-mini, capability filtering excludes A entirely, so every request lands on B regardless of weight.", + "item": [ + { + "id": "rt-lb-skips-model-gated-provider-01-add-provider", + "name": "01. add provider [lb-skips-model-gated-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [lb-skips-model-gated-provider]\";", + "pm.test(\"01. add provider [lb-skips-model-gated-provider]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-lb-skips-model-gated-provider-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-lb-skips-model-gated-provider-02-add-key", + "name": "02. add key ka [lb-skips-model-gated-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [lb-skips-model-gated-provider]\";", + "pm.test(\"02. add key ka [lb-skips-model-gated-provider]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-lb-skips-model-gated-provider-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-lb-skips-model-gated-provider-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"ka-{{run_id}}\",\"name\":\"catwiring-rt-lb-skips-model-gated-provider-ka-{{run_id}}\",\"models\":[\"gpt-4o\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\",\"weight\":1}" + } + } + }, + { + "id": "rt-lb-skips-model-gated-provider-03-add-provider", + "name": "03. add provider (b) [lb-skips-model-gated-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [lb-skips-model-gated-provider]\";", + "pm.test(\"03. add provider (b) [lb-skips-model-gated-provider]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-lb-skips-model-gated-provider-b-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-lb-skips-model-gated-provider-04-add-key", + "name": "04. add key kb (b) [lb-skips-model-gated-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [lb-skips-model-gated-provider]\";", + "pm.test(\"04. add key kb (b) [lb-skips-model-gated-provider]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-lb-skips-model-gated-provider-b-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-lb-skips-model-gated-provider-b-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"kb-{{run_id}}\",\"name\":\"catwiring-rt-lb-skips-model-gated-provider-kb-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\",\"weight\":1}" + } + } + }, + { + "id": "rt-lb-skips-model-gated-provider-05-create-vk", + "name": "05. create vk [lb-skips-model-gated-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-model-gated-provider]\";", + "var ok = pm.response.code === 200 || pm.response.code === 201;", + "pm.test(\"05. create vk [lb-skips-model-gated-provider]\", function () {", + " pm.expect([200, 201], 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (ok) {", + " var vk = (pm.response.json() || {}).virtual_key || {};", + " pm.collectionVariables.set('vkval_lb-skips-model-gated-provider', vk.value || '');", + " pm.collectionVariables.set('vkid_lb-skips-model-gated-provider', vk.id || '');", + "} else { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"name\":\"catwiring-rtvk-lb-skips-model-gated-provider-{{run_id}}\",\"is_active\":true,\"provider_configs\":[{\"provider\":\"catwiring-rt-lb-skips-model-gated-provider-{{run_id}}\",\"key_ids\":[\"*\"],\"allowed_models\":[\"*\"],\"blacklisted_models\":[],\"weight\":99},{\"provider\":\"catwiring-rt-lb-skips-model-gated-provider-b-{{run_id}}\",\"key_ids\":[\"*\"],\"allowed_models\":[\"*\"],\"blacklisted_models\":[],\"weight\":1}]}" + } + } + }, + { + "id": "rt-lb-skips-model-gated-provider-06-pdist-sample", + "name": "06. sample 1/10 (gpt-4o-mini) [lb-skips-model-gated-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-model-gated-provider]\";", + "pm.collectionVariables.set('pdist_lb-skips-model-gated-provider', '');", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-model-gated-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-model-gated-provider', set.join(',')); }", + "}", + "pm.test(\"06. sample 1/10 (gpt-4o-mini) [lb-skips-model-gated-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-model-gated-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-model-gated-provider-07-pdist-sample", + "name": "07. sample 2/10 (gpt-4o-mini) [lb-skips-model-gated-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-model-gated-provider]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-model-gated-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-model-gated-provider', set.join(',')); }", + "}", + "pm.test(\"07. sample 2/10 (gpt-4o-mini) [lb-skips-model-gated-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-model-gated-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-model-gated-provider-08-pdist-sample", + "name": "08. sample 3/10 (gpt-4o-mini) [lb-skips-model-gated-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-model-gated-provider]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-model-gated-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-model-gated-provider', set.join(',')); }", + "}", + "pm.test(\"08. sample 3/10 (gpt-4o-mini) [lb-skips-model-gated-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-model-gated-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-model-gated-provider-09-pdist-sample", + "name": "09. sample 4/10 (gpt-4o-mini) [lb-skips-model-gated-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-model-gated-provider]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-model-gated-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-model-gated-provider', set.join(',')); }", + "}", + "pm.test(\"09. sample 4/10 (gpt-4o-mini) [lb-skips-model-gated-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-model-gated-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-model-gated-provider-10-pdist-sample", + "name": "10. sample 5/10 (gpt-4o-mini) [lb-skips-model-gated-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-model-gated-provider]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-model-gated-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-model-gated-provider', set.join(',')); }", + "}", + "pm.test(\"10. sample 5/10 (gpt-4o-mini) [lb-skips-model-gated-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-model-gated-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-model-gated-provider-11-pdist-sample", + "name": "11. sample 6/10 (gpt-4o-mini) [lb-skips-model-gated-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-model-gated-provider]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-model-gated-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-model-gated-provider', set.join(',')); }", + "}", + "pm.test(\"11. sample 6/10 (gpt-4o-mini) [lb-skips-model-gated-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-model-gated-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-model-gated-provider-12-pdist-sample", + "name": "12. sample 7/10 (gpt-4o-mini) [lb-skips-model-gated-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-model-gated-provider]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-model-gated-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-model-gated-provider', set.join(',')); }", + "}", + "pm.test(\"12. sample 7/10 (gpt-4o-mini) [lb-skips-model-gated-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-model-gated-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-model-gated-provider-13-pdist-sample", + "name": "13. sample 8/10 (gpt-4o-mini) [lb-skips-model-gated-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-model-gated-provider]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-model-gated-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-model-gated-provider', set.join(',')); }", + "}", + "pm.test(\"13. sample 8/10 (gpt-4o-mini) [lb-skips-model-gated-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-model-gated-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-model-gated-provider-14-pdist-sample", + "name": "14. sample 9/10 (gpt-4o-mini) [lb-skips-model-gated-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-model-gated-provider]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-model-gated-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-model-gated-provider', set.join(',')); }", + "}", + "pm.test(\"14. sample 9/10 (gpt-4o-mini) [lb-skips-model-gated-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-model-gated-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-model-gated-provider-15-pdist-sample", + "name": "15. sample 10/10 (gpt-4o-mini) [lb-skips-model-gated-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-model-gated-provider]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-model-gated-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-model-gated-provider', set.join(',')); }", + "}", + "pm.test(\"15. sample 10/10 (gpt-4o-mini) [lb-skips-model-gated-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-model-gated-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-model-gated-provider-16-pdist-assert", + "name": "16. all requests go to the 1% provider (99% provider can't serve the model) [lb-skips-model-gated-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-model-gated-provider]\";", + "var ok = true, errMsg = '';", + "try {", + " var seen = (pm.collectionVariables.get('pdist_lb-skips-model-gated-provider') || '').split(',').filter(Boolean);", + " var only = ['catwiring-rt-lb-skips-model-gated-provider-b-' + pm.variables.get('run_id')];", + " var never = ['catwiring-rt-lb-skips-model-gated-provider-' + pm.variables.get('run_id')];", + " var all = [];", + " if (seen.length === 0) throw new Error('no provider observed across samples');", + " seen.forEach(function (p) {", + " if (only.length && only.indexOf(p) < 0) throw new Error('provider ' + p + ' served but not in expectOnly ' + JSON.stringify(only));", + " if (never.indexOf(p) >= 0) throw new Error('provider ' + p + ' served but was expectNever; observed ' + JSON.stringify(seen));", + " });", + " all.forEach(function (p) { if (seen.indexOf(p) < 0) throw new Error('provider ' + p + ' expected to serve but did not; observed ' + JSON.stringify(seen)); });", + "} catch (e) { ok = false; errMsg = e.message; }", + "pm.test(\"16. all requests go to the 1% provider (99% provider can't serve the model) [lb-skips-model-gated-provider]\", function () { if (!ok) throw new Error(errMsg); });", + "if (!ok) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/health", + "host": [ + "{{base_url}}" + ], + "path": [ + "health" + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-lb-skips-model-gated-provider-cleanup-vk", + "name": "cleanup: delete vk [lb-skips-model-gated-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete vk [lb-skips-model-gated-provider]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys/{{vkid_lb-skips-model-gated-provider}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys", + "{{vkid_lb-skips-model-gated-provider}}" + ] + } + } + }, + { + "id": "rt-lb-skips-model-gated-provider-cleanup-provider-self", + "name": "cleanup: delete provider [lb-skips-model-gated-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [lb-skips-model-gated-provider]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-lb-skips-model-gated-provider-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-lb-skips-model-gated-provider-{{run_id}}" + ] + } + } + }, + { + "id": "rt-lb-skips-model-gated-provider-cleanup-provider-b", + "name": "cleanup: delete provider b [lb-skips-model-gated-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider b [lb-skips-model-gated-provider]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-lb-skips-model-gated-provider-b-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-lb-skips-model-gated-provider-b-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "LB skips a 99%-weight provider that blacklists the model", + "description": "A (99%) allows all models but blacklists gpt-4o-mini on its key; B (1%) allows all. The blacklist removes A from the candidates, so every request lands on B.", + "item": [ + { + "id": "rt-lb-skips-blacklisted-provider-01-add-provider", + "name": "01. add provider [lb-skips-blacklisted-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [lb-skips-blacklisted-provider]\";", + "pm.test(\"01. add provider [lb-skips-blacklisted-provider]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-lb-skips-blacklisted-provider-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-lb-skips-blacklisted-provider-02-add-key", + "name": "02. add key ka [lb-skips-blacklisted-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [lb-skips-blacklisted-provider]\";", + "pm.test(\"02. add key ka [lb-skips-blacklisted-provider]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-lb-skips-blacklisted-provider-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-lb-skips-blacklisted-provider-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"ka-{{run_id}}\",\"name\":\"catwiring-rt-lb-skips-blacklisted-provider-ka-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\",\"blacklisted_models\":[\"gpt-4o-mini\"],\"weight\":1}" + } + } + }, + { + "id": "rt-lb-skips-blacklisted-provider-03-add-provider", + "name": "03. add provider (b) [lb-skips-blacklisted-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [lb-skips-blacklisted-provider]\";", + "pm.test(\"03. add provider (b) [lb-skips-blacklisted-provider]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-lb-skips-blacklisted-provider-b-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-lb-skips-blacklisted-provider-04-add-key", + "name": "04. add key kb (b) [lb-skips-blacklisted-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [lb-skips-blacklisted-provider]\";", + "pm.test(\"04. add key kb (b) [lb-skips-blacklisted-provider]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-lb-skips-blacklisted-provider-b-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-lb-skips-blacklisted-provider-b-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"kb-{{run_id}}\",\"name\":\"catwiring-rt-lb-skips-blacklisted-provider-kb-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\",\"weight\":1}" + } + } + }, + { + "id": "rt-lb-skips-blacklisted-provider-05-create-vk", + "name": "05. create vk [lb-skips-blacklisted-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-blacklisted-provider]\";", + "var ok = pm.response.code === 200 || pm.response.code === 201;", + "pm.test(\"05. create vk [lb-skips-blacklisted-provider]\", function () {", + " pm.expect([200, 201], 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (ok) {", + " var vk = (pm.response.json() || {}).virtual_key || {};", + " pm.collectionVariables.set('vkval_lb-skips-blacklisted-provider', vk.value || '');", + " pm.collectionVariables.set('vkid_lb-skips-blacklisted-provider', vk.id || '');", + "} else { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"name\":\"catwiring-rtvk-lb-skips-blacklisted-provider-{{run_id}}\",\"is_active\":true,\"provider_configs\":[{\"provider\":\"catwiring-rt-lb-skips-blacklisted-provider-{{run_id}}\",\"key_ids\":[\"*\"],\"allowed_models\":[\"*\"],\"blacklisted_models\":[],\"weight\":99},{\"provider\":\"catwiring-rt-lb-skips-blacklisted-provider-b-{{run_id}}\",\"key_ids\":[\"*\"],\"allowed_models\":[\"*\"],\"blacklisted_models\":[],\"weight\":1}]}" + } + } + }, + { + "id": "rt-lb-skips-blacklisted-provider-06-pdist-sample", + "name": "06. sample 1/10 (gpt-4o-mini) [lb-skips-blacklisted-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-blacklisted-provider]\";", + "pm.collectionVariables.set('pdist_lb-skips-blacklisted-provider', '');", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-blacklisted-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-blacklisted-provider', set.join(',')); }", + "}", + "pm.test(\"06. sample 1/10 (gpt-4o-mini) [lb-skips-blacklisted-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-blacklisted-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-blacklisted-provider-07-pdist-sample", + "name": "07. sample 2/10 (gpt-4o-mini) [lb-skips-blacklisted-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-blacklisted-provider]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-blacklisted-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-blacklisted-provider', set.join(',')); }", + "}", + "pm.test(\"07. sample 2/10 (gpt-4o-mini) [lb-skips-blacklisted-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-blacklisted-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-blacklisted-provider-08-pdist-sample", + "name": "08. sample 3/10 (gpt-4o-mini) [lb-skips-blacklisted-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-blacklisted-provider]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-blacklisted-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-blacklisted-provider', set.join(',')); }", + "}", + "pm.test(\"08. sample 3/10 (gpt-4o-mini) [lb-skips-blacklisted-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-blacklisted-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-blacklisted-provider-09-pdist-sample", + "name": "09. sample 4/10 (gpt-4o-mini) [lb-skips-blacklisted-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-blacklisted-provider]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-blacklisted-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-blacklisted-provider', set.join(',')); }", + "}", + "pm.test(\"09. sample 4/10 (gpt-4o-mini) [lb-skips-blacklisted-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-blacklisted-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-blacklisted-provider-10-pdist-sample", + "name": "10. sample 5/10 (gpt-4o-mini) [lb-skips-blacklisted-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-blacklisted-provider]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-blacklisted-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-blacklisted-provider', set.join(',')); }", + "}", + "pm.test(\"10. sample 5/10 (gpt-4o-mini) [lb-skips-blacklisted-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-blacklisted-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-blacklisted-provider-11-pdist-sample", + "name": "11. sample 6/10 (gpt-4o-mini) [lb-skips-blacklisted-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-blacklisted-provider]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-blacklisted-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-blacklisted-provider', set.join(',')); }", + "}", + "pm.test(\"11. sample 6/10 (gpt-4o-mini) [lb-skips-blacklisted-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-blacklisted-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-blacklisted-provider-12-pdist-sample", + "name": "12. sample 7/10 (gpt-4o-mini) [lb-skips-blacklisted-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-blacklisted-provider]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-blacklisted-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-blacklisted-provider', set.join(',')); }", + "}", + "pm.test(\"12. sample 7/10 (gpt-4o-mini) [lb-skips-blacklisted-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-blacklisted-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-blacklisted-provider-13-pdist-sample", + "name": "13. sample 8/10 (gpt-4o-mini) [lb-skips-blacklisted-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-blacklisted-provider]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-blacklisted-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-blacklisted-provider', set.join(',')); }", + "}", + "pm.test(\"13. sample 8/10 (gpt-4o-mini) [lb-skips-blacklisted-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-blacklisted-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-blacklisted-provider-14-pdist-sample", + "name": "14. sample 9/10 (gpt-4o-mini) [lb-skips-blacklisted-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-blacklisted-provider]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-blacklisted-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-blacklisted-provider', set.join(',')); }", + "}", + "pm.test(\"14. sample 9/10 (gpt-4o-mini) [lb-skips-blacklisted-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-blacklisted-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-blacklisted-provider-15-pdist-sample", + "name": "15. sample 10/10 (gpt-4o-mini) [lb-skips-blacklisted-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-blacklisted-provider]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-blacklisted-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-blacklisted-provider', set.join(',')); }", + "}", + "pm.test(\"15. sample 10/10 (gpt-4o-mini) [lb-skips-blacklisted-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-blacklisted-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-blacklisted-provider-16-pdist-assert", + "name": "16. all requests go to the 1% provider (99% provider blacklists the model) [lb-skips-blacklisted-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-blacklisted-provider]\";", + "var ok = true, errMsg = '';", + "try {", + " var seen = (pm.collectionVariables.get('pdist_lb-skips-blacklisted-provider') || '').split(',').filter(Boolean);", + " var only = ['catwiring-rt-lb-skips-blacklisted-provider-b-' + pm.variables.get('run_id')];", + " var never = ['catwiring-rt-lb-skips-blacklisted-provider-' + pm.variables.get('run_id')];", + " var all = [];", + " if (seen.length === 0) throw new Error('no provider observed across samples');", + " seen.forEach(function (p) {", + " if (only.length && only.indexOf(p) < 0) throw new Error('provider ' + p + ' served but not in expectOnly ' + JSON.stringify(only));", + " if (never.indexOf(p) >= 0) throw new Error('provider ' + p + ' served but was expectNever; observed ' + JSON.stringify(seen));", + " });", + " all.forEach(function (p) { if (seen.indexOf(p) < 0) throw new Error('provider ' + p + ' expected to serve but did not; observed ' + JSON.stringify(seen)); });", + "} catch (e) { ok = false; errMsg = e.message; }", + "pm.test(\"16. all requests go to the 1% provider (99% provider blacklists the model) [lb-skips-blacklisted-provider]\", function () { if (!ok) throw new Error(errMsg); });", + "if (!ok) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/health", + "host": [ + "{{base_url}}" + ], + "path": [ + "health" + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-lb-skips-blacklisted-provider-cleanup-vk", + "name": "cleanup: delete vk [lb-skips-blacklisted-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete vk [lb-skips-blacklisted-provider]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys/{{vkid_lb-skips-blacklisted-provider}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys", + "{{vkid_lb-skips-blacklisted-provider}}" + ] + } + } + }, + { + "id": "rt-lb-skips-blacklisted-provider-cleanup-provider-self", + "name": "cleanup: delete provider [lb-skips-blacklisted-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [lb-skips-blacklisted-provider]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-lb-skips-blacklisted-provider-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-lb-skips-blacklisted-provider-{{run_id}}" + ] + } + } + }, + { + "id": "rt-lb-skips-blacklisted-provider-cleanup-provider-b", + "name": "cleanup: delete provider b [lb-skips-blacklisted-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider b [lb-skips-blacklisted-provider]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-lb-skips-blacklisted-provider-b-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-lb-skips-blacklisted-provider-b-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "LB skips a 99%-weight provider whose only key is disabled", + "description": "A (99%) has its only key disabled; B (1%) is enabled. With no usable key, A is excluded, so every request lands on B.", + "item": [ + { + "id": "rt-lb-skips-disabled-key-provider-01-add-provider", + "name": "01. add provider [lb-skips-disabled-key-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [lb-skips-disabled-key-provider]\";", + "pm.test(\"01. add provider [lb-skips-disabled-key-provider]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-lb-skips-disabled-key-provider-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-lb-skips-disabled-key-provider-02-add-key", + "name": "02. add key ka [lb-skips-disabled-key-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [lb-skips-disabled-key-provider]\";", + "pm.test(\"02. add key ka [lb-skips-disabled-key-provider]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-lb-skips-disabled-key-provider-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-lb-skips-disabled-key-provider-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"ka-{{run_id}}\",\"name\":\"catwiring-rt-lb-skips-disabled-key-provider-ka-{{run_id}}\",\"models\":[\"*\"],\"enabled\":false,\"value\":\"env.OPENAI_API_KEY\",\"weight\":1}" + } + } + }, + { + "id": "rt-lb-skips-disabled-key-provider-03-add-provider", + "name": "03. add provider (b) [lb-skips-disabled-key-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [lb-skips-disabled-key-provider]\";", + "pm.test(\"03. add provider (b) [lb-skips-disabled-key-provider]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-lb-skips-disabled-key-provider-b-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-lb-skips-disabled-key-provider-04-add-key", + "name": "04. add key kb (b) [lb-skips-disabled-key-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [lb-skips-disabled-key-provider]\";", + "pm.test(\"04. add key kb (b) [lb-skips-disabled-key-provider]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-lb-skips-disabled-key-provider-b-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-lb-skips-disabled-key-provider-b-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"kb-{{run_id}}\",\"name\":\"catwiring-rt-lb-skips-disabled-key-provider-kb-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\",\"weight\":1}" + } + } + }, + { + "id": "rt-lb-skips-disabled-key-provider-05-create-vk", + "name": "05. create vk [lb-skips-disabled-key-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-disabled-key-provider]\";", + "var ok = pm.response.code === 200 || pm.response.code === 201;", + "pm.test(\"05. create vk [lb-skips-disabled-key-provider]\", function () {", + " pm.expect([200, 201], 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (ok) {", + " var vk = (pm.response.json() || {}).virtual_key || {};", + " pm.collectionVariables.set('vkval_lb-skips-disabled-key-provider', vk.value || '');", + " pm.collectionVariables.set('vkid_lb-skips-disabled-key-provider', vk.id || '');", + "} else { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"name\":\"catwiring-rtvk-lb-skips-disabled-key-provider-{{run_id}}\",\"is_active\":true,\"provider_configs\":[{\"provider\":\"catwiring-rt-lb-skips-disabled-key-provider-{{run_id}}\",\"key_ids\":[\"*\"],\"allowed_models\":[\"*\"],\"blacklisted_models\":[],\"weight\":99},{\"provider\":\"catwiring-rt-lb-skips-disabled-key-provider-b-{{run_id}}\",\"key_ids\":[\"*\"],\"allowed_models\":[\"*\"],\"blacklisted_models\":[],\"weight\":1}]}" + } + } + }, + { + "id": "rt-lb-skips-disabled-key-provider-06-pdist-sample", + "name": "06. sample 1/10 (gpt-4o-mini) [lb-skips-disabled-key-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-disabled-key-provider]\";", + "pm.collectionVariables.set('pdist_lb-skips-disabled-key-provider', '');", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-disabled-key-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-disabled-key-provider', set.join(',')); }", + "}", + "pm.test(\"06. sample 1/10 (gpt-4o-mini) [lb-skips-disabled-key-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-disabled-key-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-disabled-key-provider-07-pdist-sample", + "name": "07. sample 2/10 (gpt-4o-mini) [lb-skips-disabled-key-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-disabled-key-provider]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-disabled-key-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-disabled-key-provider', set.join(',')); }", + "}", + "pm.test(\"07. sample 2/10 (gpt-4o-mini) [lb-skips-disabled-key-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-disabled-key-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-disabled-key-provider-08-pdist-sample", + "name": "08. sample 3/10 (gpt-4o-mini) [lb-skips-disabled-key-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-disabled-key-provider]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-disabled-key-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-disabled-key-provider', set.join(',')); }", + "}", + "pm.test(\"08. sample 3/10 (gpt-4o-mini) [lb-skips-disabled-key-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-disabled-key-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-disabled-key-provider-09-pdist-sample", + "name": "09. sample 4/10 (gpt-4o-mini) [lb-skips-disabled-key-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-disabled-key-provider]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-disabled-key-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-disabled-key-provider', set.join(',')); }", + "}", + "pm.test(\"09. sample 4/10 (gpt-4o-mini) [lb-skips-disabled-key-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-disabled-key-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-disabled-key-provider-10-pdist-sample", + "name": "10. sample 5/10 (gpt-4o-mini) [lb-skips-disabled-key-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-disabled-key-provider]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-disabled-key-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-disabled-key-provider', set.join(',')); }", + "}", + "pm.test(\"10. sample 5/10 (gpt-4o-mini) [lb-skips-disabled-key-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-disabled-key-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-disabled-key-provider-11-pdist-sample", + "name": "11. sample 6/10 (gpt-4o-mini) [lb-skips-disabled-key-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-disabled-key-provider]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-disabled-key-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-disabled-key-provider', set.join(',')); }", + "}", + "pm.test(\"11. sample 6/10 (gpt-4o-mini) [lb-skips-disabled-key-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-disabled-key-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-disabled-key-provider-12-pdist-sample", + "name": "12. sample 7/10 (gpt-4o-mini) [lb-skips-disabled-key-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-disabled-key-provider]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-disabled-key-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-disabled-key-provider', set.join(',')); }", + "}", + "pm.test(\"12. sample 7/10 (gpt-4o-mini) [lb-skips-disabled-key-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-disabled-key-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-disabled-key-provider-13-pdist-sample", + "name": "13. sample 8/10 (gpt-4o-mini) [lb-skips-disabled-key-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-disabled-key-provider]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-disabled-key-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-disabled-key-provider', set.join(',')); }", + "}", + "pm.test(\"13. sample 8/10 (gpt-4o-mini) [lb-skips-disabled-key-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-disabled-key-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-disabled-key-provider-14-pdist-sample", + "name": "14. sample 9/10 (gpt-4o-mini) [lb-skips-disabled-key-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-disabled-key-provider]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-disabled-key-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-disabled-key-provider', set.join(',')); }", + "}", + "pm.test(\"14. sample 9/10 (gpt-4o-mini) [lb-skips-disabled-key-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-disabled-key-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-disabled-key-provider-15-pdist-sample", + "name": "15. sample 10/10 (gpt-4o-mini) [lb-skips-disabled-key-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-disabled-key-provider]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_lb-skips-disabled-key-provider') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_lb-skips-disabled-key-provider', set.join(',')); }", + "}", + "pm.test(\"15. sample 10/10 (gpt-4o-mini) [lb-skips-disabled-key-provider]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_lb-skips-disabled-key-provider}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-lb-skips-disabled-key-provider-16-pdist-assert", + "name": "16. all requests go to the 1% provider (99% provider key disabled) [lb-skips-disabled-key-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [lb-skips-disabled-key-provider]\";", + "var ok = true, errMsg = '';", + "try {", + " var seen = (pm.collectionVariables.get('pdist_lb-skips-disabled-key-provider') || '').split(',').filter(Boolean);", + " var only = ['catwiring-rt-lb-skips-disabled-key-provider-b-' + pm.variables.get('run_id')];", + " var never = ['catwiring-rt-lb-skips-disabled-key-provider-' + pm.variables.get('run_id')];", + " var all = [];", + " if (seen.length === 0) throw new Error('no provider observed across samples');", + " seen.forEach(function (p) {", + " if (only.length && only.indexOf(p) < 0) throw new Error('provider ' + p + ' served but not in expectOnly ' + JSON.stringify(only));", + " if (never.indexOf(p) >= 0) throw new Error('provider ' + p + ' served but was expectNever; observed ' + JSON.stringify(seen));", + " });", + " all.forEach(function (p) { if (seen.indexOf(p) < 0) throw new Error('provider ' + p + ' expected to serve but did not; observed ' + JSON.stringify(seen)); });", + "} catch (e) { ok = false; errMsg = e.message; }", + "pm.test(\"16. all requests go to the 1% provider (99% provider key disabled) [lb-skips-disabled-key-provider]\", function () { if (!ok) throw new Error(errMsg); });", + "if (!ok) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/health", + "host": [ + "{{base_url}}" + ], + "path": [ + "health" + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-lb-skips-disabled-key-provider-cleanup-vk", + "name": "cleanup: delete vk [lb-skips-disabled-key-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete vk [lb-skips-disabled-key-provider]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys/{{vkid_lb-skips-disabled-key-provider}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys", + "{{vkid_lb-skips-disabled-key-provider}}" + ] + } + } + }, + { + "id": "rt-lb-skips-disabled-key-provider-cleanup-provider-self", + "name": "cleanup: delete provider [lb-skips-disabled-key-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [lb-skips-disabled-key-provider]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-lb-skips-disabled-key-provider-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-lb-skips-disabled-key-provider-{{run_id}}" + ] + } + } + }, + { + "id": "rt-lb-skips-disabled-key-provider-cleanup-provider-b", + "name": "cleanup: delete provider b [lb-skips-disabled-key-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider b [lb-skips-disabled-key-provider]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-lb-skips-disabled-key-provider-b-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-lb-skips-disabled-key-provider-b-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "VK provider allowlist blocks a different real provider", + "description": "A VK that lists only the openai provider routes openai but rejects an explicit request to the anthropic provider (pruned from the routing allowlist).", + "item": [ + { + "id": "rt-cross-provider-allowlist-01-add-provider", + "name": "01. add provider [cross-provider-allowlist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [cross-provider-allowlist]\";", + "pm.test(\"01. add provider [cross-provider-allowlist]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-cross-provider-allowlist-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-cross-provider-allowlist-02-add-key", + "name": "02. add key ko [cross-provider-allowlist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [cross-provider-allowlist]\";", + "pm.test(\"02. add key ko [cross-provider-allowlist]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-cross-provider-allowlist-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-cross-provider-allowlist-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"ko-{{run_id}}\",\"name\":\"catwiring-rt-cross-provider-allowlist-ko-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\"}" + } + } + }, + { + "id": "rt-cross-provider-allowlist-03-add-provider", + "name": "03. add provider (b) [cross-provider-allowlist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [cross-provider-allowlist]\";", + "pm.test(\"03. add provider (b) [cross-provider-allowlist]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-cross-provider-allowlist-b-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"anthropic\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-cross-provider-allowlist-04-add-key", + "name": "04. add key ka (b) [cross-provider-allowlist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [cross-provider-allowlist]\";", + "pm.test(\"04. add key ka (b) [cross-provider-allowlist]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-cross-provider-allowlist-b-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-cross-provider-allowlist-b-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"ka-{{run_id}}\",\"name\":\"catwiring-rt-cross-provider-allowlist-ka-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.ANTHROPIC_API_KEY\"}" + } + } + }, + { + "id": "rt-cross-provider-allowlist-05-create-vk", + "name": "05. create vk [cross-provider-allowlist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [cross-provider-allowlist]\";", + "var ok = pm.response.code === 200 || pm.response.code === 201;", + "pm.test(\"05. create vk [cross-provider-allowlist]\", function () {", + " pm.expect([200, 201], 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (ok) {", + " var vk = (pm.response.json() || {}).virtual_key || {};", + " pm.collectionVariables.set('vkval_cross-provider-allowlist', vk.value || '');", + " pm.collectionVariables.set('vkid_cross-provider-allowlist', vk.id || '');", + "} else { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"name\":\"catwiring-rtvk-cross-provider-allowlist-{{run_id}}\",\"is_active\":true,\"provider_configs\":[{\"provider\":\"catwiring-rt-cross-provider-allowlist-{{run_id}}\",\"key_ids\":[\"*\"],\"allowed_models\":[\"*\"],\"blacklisted_models\":[],\"weight\":1}]}" + } + } + }, + { + "id": "rt-cross-provider-allowlist-06-route", + "name": "06. VK-allowed provider routes (200) [cross-provider-allowlist]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [cross-provider-allowlist]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('route status ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " if (!body.choices || body.choices.length === 0) { throw new Error('no choices'); }", + " var ri = (body.extra_fields || {}).routing_info || {};", + " var providerName = 'catwiring-rt-cross-provider-allowlist-' + pm.variables.get('run_id');", + " if (ri.provider !== providerName) { throw new Error('routing_info.provider=' + ri.provider + ' expected ' + providerName); }", + " var expectedKey = 'catwiring-rt-cross-provider-allowlist-ko-' + pm.variables.get('run_id');", + " if (ri.key !== expectedKey) { throw new Error('routing_info.key=' + ri.key + ' expected ' + expectedKey); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"06. VK-allowed provider routes (200) [cross-provider-allowlist]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"06. VK-allowed provider routes (200) [cross-provider-allowlist]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_cross-provider-allowlist}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-cross-provider-allowlist-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-cross-provider-allowlist-07-route", + "name": "07. VK-disallowed provider blocked (400) [cross-provider-allowlist]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [cross-provider-allowlist]\";", + "function assertNow() {", + " if (pm.response.code !== 400) { throw new Error('expected status 400 got ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " var msg = (body.error && (body.error.message || body.error)) || body.message || '';", + " if (String(msg).indexOf(\"is not permitted for this request\") < 0) { throw new Error('error message=' + msg); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"07. VK-disallowed provider blocked (400) [cross-provider-allowlist]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"07. VK-disallowed provider blocked (400) [cross-provider-allowlist]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_cross-provider-allowlist}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-cross-provider-allowlist-b-{{run_id}}/claude-haiku-4-5\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-cross-provider-allowlist-cleanup-vk", + "name": "cleanup: delete vk [cross-provider-allowlist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete vk [cross-provider-allowlist]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys/{{vkid_cross-provider-allowlist}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys", + "{{vkid_cross-provider-allowlist}}" + ] + } + } + }, + { + "id": "rt-cross-provider-allowlist-cleanup-provider-self", + "name": "cleanup: delete provider [cross-provider-allowlist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [cross-provider-allowlist]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-cross-provider-allowlist-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-cross-provider-allowlist-{{run_id}}" + ] + } + } + }, + { + "id": "rt-cross-provider-allowlist-cleanup-provider-b", + "name": "cleanup: delete provider b [cross-provider-allowlist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider b [cross-provider-allowlist]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-cross-provider-allowlist-b-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-cross-provider-allowlist-b-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Governance LBs one Claude model across Anthropic, Vertex, and Bedrock", + "description": "Anthropic (custom, native), Vertex (standard, native) and Bedrock (standard, via a key alias to its inference-profile id) all serve claude-sonnet-4-5. A VK over all three, routing the bare model, has governance distribute across the heterogeneous providers; the log detail records the LB trail. Serial-only (Vertex/Bedrock are global standard providers).", + "item": [ + { + "id": "rt-cross-provider-same-model-lb-01-add-provider", + "name": "01. add provider [cross-provider-same-model-lb]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [cross-provider-same-model-lb]\";", + "pm.test(\"01. add provider [cross-provider-same-model-lb]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-cross-provider-same-model-lb-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"anthropic\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-cross-provider-same-model-lb-02-add-key", + "name": "02. add key kan [cross-provider-same-model-lb]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [cross-provider-same-model-lb]\";", + "pm.test(\"02. add key kan [cross-provider-same-model-lb]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-cross-provider-same-model-lb-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-cross-provider-same-model-lb-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"kan-{{run_id}}\",\"name\":\"catwiring-rt-cross-provider-same-model-lb-kan-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.ANTHROPIC_API_KEY\",\"weight\":1}" + } + } + }, + { + "id": "rt-cross-provider-same-model-lb-03-add-provider", + "name": "03. add provider (vtx) [cross-provider-same-model-lb]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [cross-provider-same-model-lb]\";", + "pm.test(\"03. add provider (vtx) [cross-provider-same-model-lb]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"vertex\"}" + } + } + }, + { + "id": "rt-cross-provider-same-model-lb-04-add-key", + "name": "04. add key kvx (vtx) [cross-provider-same-model-lb]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [cross-provider-same-model-lb]\";", + "pm.test(\"04. add key kvx (vtx) [cross-provider-same-model-lb]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/vertex/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "vertex", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"kvx-{{run_id}}\",\"name\":\"catwiring-rt-cross-provider-same-model-lb-kvx-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"vertex_key_config\":{\"project_id\":\"env.VERTEX_PROJECT_ID\",\"project_number\":\"env.VERTEX_PROJECT_NUMBER\",\"region\":\"global\",\"auth_credentials\":\"env.VERTEX_CREDENTIALS\"},\"weight\":1}" + } + } + }, + { + "id": "rt-cross-provider-same-model-lb-05-add-provider", + "name": "05. add provider (bdr) [cross-provider-same-model-lb]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [cross-provider-same-model-lb]\";", + "pm.test(\"05. add provider (bdr) [cross-provider-same-model-lb]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"bedrock\"}" + } + } + }, + { + "id": "rt-cross-provider-same-model-lb-06-add-key", + "name": "06. add key kbd (bdr) [cross-provider-same-model-lb]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [cross-provider-same-model-lb]\";", + "pm.test(\"06. add key kbd (bdr) [cross-provider-same-model-lb]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/bedrock/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "bedrock", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"kbd-{{run_id}}\",\"name\":\"catwiring-rt-cross-provider-same-model-lb-kbd-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"bedrock_key_config\":{\"access_key\":\"env.AWS_ACCESS_KEY_ID\",\"secret_key\":\"env.AWS_SECRET_ACCESS_KEY\",\"region\":\"env.AWS_REGION\"},\"aliases\":{\"claude-sonnet-4-5\":\"us.anthropic.claude-sonnet-4-5-20250929-v1:0\"},\"weight\":1}" + } + } + }, + { + "id": "rt-cross-provider-same-model-lb-07-create-vk", + "name": "07. create vk [cross-provider-same-model-lb]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [cross-provider-same-model-lb]\";", + "var ok = pm.response.code === 200 || pm.response.code === 201;", + "pm.test(\"07. create vk [cross-provider-same-model-lb]\", function () {", + " pm.expect([200, 201], 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (ok) {", + " var vk = (pm.response.json() || {}).virtual_key || {};", + " pm.collectionVariables.set('vkval_cross-provider-same-model-lb', vk.value || '');", + " pm.collectionVariables.set('vkid_cross-provider-same-model-lb', vk.id || '');", + "} else { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"name\":\"catwiring-rtvk-cross-provider-same-model-lb-{{run_id}}\",\"is_active\":true,\"provider_configs\":[{\"provider\":\"catwiring-rt-cross-provider-same-model-lb-{{run_id}}\",\"key_ids\":[\"*\"],\"allowed_models\":[\"*\"],\"blacklisted_models\":[],\"weight\":1},{\"provider\":\"vertex\",\"key_ids\":[\"*\"],\"allowed_models\":[\"*\"],\"blacklisted_models\":[],\"weight\":1},{\"provider\":\"bedrock\",\"key_ids\":[\"*\"],\"allowed_models\":[\"*\"],\"blacklisted_models\":[],\"weight\":1}]}" + } + } + }, + { + "id": "rt-cross-provider-same-model-lb-08-route", + "name": "08. bare claude model routes via anthropic, vertex, or bedrock (200) [cross-provider-same-model-lb]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 3000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [cross-provider-same-model-lb]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('route status ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " if (!body.choices || body.choices.length === 0) { throw new Error('no choices'); }", + " var ri = (body.extra_fields || {}).routing_info || {};", + " var allowedProviders = ['catwiring-rt-cross-provider-same-model-lb-' + pm.variables.get('run_id'), \"vertex\", \"bedrock\"];", + " if (allowedProviders.indexOf(ri.provider) < 0) { throw new Error('routing_info.provider=' + ri.provider + ' not in ' + JSON.stringify(allowedProviders)); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"08. bare claude model routes via anthropic, vertex, or bedrock (200) [cross-provider-same-model-lb]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"08. bare claude model routes via anthropic, vertex, or bedrock (200) [cross-provider-same-model-lb]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_cross-provider-same-model-lb}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"claude-sonnet-4-5\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-cross-provider-same-model-lb-09-capture-log", + "name": "09. capture routing log id [cross-provider-same-model-lb]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [cross-provider-same-model-lb]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('logs status ' + pm.response.code); }", + " var logs = (pm.response.json() || {}).logs || [];", + " if (!logs.length || !logs[0].id) { throw new Error('no log row yet for VK'); }", + " pm.collectionVariables.set('logid_cross-provider-same-model-lb', logs[0].id);", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"09. capture routing log id [cross-provider-same-model-lb]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"09. capture routing log id [cross-provider-same-model-lb]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/logs?virtual_key_ids={{vkid_cross-provider-same-model-lb}}&limit=1", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "logs" + ], + "query": [ + { + "key": "virtual_key_ids", + "value": "{{vkid_cross-provider-same-model-lb}}" + }, + { + "key": "limit", + "value": "1" + } + ] + } + } + }, + { + "id": "rt-cross-provider-same-model-lb-10-assert-trail", + "name": "10. log detail records cross-provider LB trail [cross-provider-same-model-lb]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [cross-provider-same-model-lb]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('log detail status ' + pm.response.code); }", + " var row = pm.response.json() || {};", + " var trail = row.routing_engine_logs || '';", + " var expected = [\"Load balancing model claude-sonnet-4-5\",\"Selected provider\"];", + " expected.forEach(function (s) {", + " if (String(trail).indexOf(s) < 0) { throw new Error('routing_engine_logs missing ' + JSON.stringify(s) + '; got ' + JSON.stringify(trail)); }", + " });", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"10. log detail records cross-provider LB trail [cross-provider-same-model-lb]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"10. log detail records cross-provider LB trail [cross-provider-same-model-lb]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/logs/{{logid_cross-provider-same-model-lb}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "logs", + "{{logid_cross-provider-same-model-lb}}" + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-cross-provider-same-model-lb-cleanup-vk", + "name": "cleanup: delete vk [cross-provider-same-model-lb]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete vk [cross-provider-same-model-lb]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys/{{vkid_cross-provider-same-model-lb}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys", + "{{vkid_cross-provider-same-model-lb}}" + ] + } + } + }, + { + "id": "rt-cross-provider-same-model-lb-cleanup-provider-self", + "name": "cleanup: delete provider [cross-provider-same-model-lb]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [cross-provider-same-model-lb]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-cross-provider-same-model-lb-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-cross-provider-same-model-lb-{{run_id}}" + ] + } + } + }, + { + "id": "rt-cross-provider-same-model-lb-cleanup-provider-vtx", + "name": "cleanup: delete provider vtx [cross-provider-same-model-lb]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider vtx [cross-provider-same-model-lb]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/vertex", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "vertex" + ] + } + } + }, + { + "id": "rt-cross-provider-same-model-lb-cleanup-provider-bdr", + "name": "cleanup: delete provider bdr [cross-provider-same-model-lb]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider bdr [cross-provider-same-model-lb]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/bedrock", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "bedrock" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "A request fails over to a healthy provider via a request-level fallback", + "description": "The primary provider's key is invalid, so its attempt fails upstream; a request-level fallback to a second provider serving the same model succeeds. routing_info marks the fallback and records the original primary provider/model. The /v1 endpoint takes fallbacks in the string \"provider/model\" form.", + "item": [ + { + "id": "rt-fallback-cross-provider-01-add-provider", + "name": "01. add provider [fallback-cross-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [fallback-cross-provider]\";", + "pm.test(\"01. add provider [fallback-cross-provider]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-fallback-cross-provider-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-fallback-cross-provider-02-add-key", + "name": "02. add key kbad [fallback-cross-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [fallback-cross-provider]\";", + "pm.test(\"02. add key kbad [fallback-cross-provider]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-fallback-cross-provider-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-fallback-cross-provider-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"kbad-{{run_id}}\",\"name\":\"catwiring-rt-fallback-cross-provider-kbad-{{run_id}}\",\"models\":[\"gpt-4o-mini\"],\"enabled\":true,\"value\":\"sk-deadbeef-invalid-000\"}" + } + } + }, + { + "id": "rt-fallback-cross-provider-03-add-provider", + "name": "03. add provider (b) [fallback-cross-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [fallback-cross-provider]\";", + "pm.test(\"03. add provider (b) [fallback-cross-provider]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-fallback-cross-provider-b-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-fallback-cross-provider-04-add-key", + "name": "04. add key kgood (b) [fallback-cross-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [fallback-cross-provider]\";", + "pm.test(\"04. add key kgood (b) [fallback-cross-provider]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-fallback-cross-provider-b-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-fallback-cross-provider-b-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"kgood-{{run_id}}\",\"name\":\"catwiring-rt-fallback-cross-provider-kgood-{{run_id}}\",\"models\":[\"gpt-4o-mini\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\"}" + } + } + }, + { + "id": "rt-fallback-cross-provider-05-route", + "name": "05. primary fails; request fallback serves (200, is_fallback) [fallback-cross-provider]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 1000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [fallback-cross-provider]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('route status ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " if (!body.choices || body.choices.length === 0) { throw new Error('no choices'); }", + " var ri = (body.extra_fields || {}).routing_info || {};", + " var allowedProviders = ['catwiring-rt-fallback-cross-provider-b-' + pm.variables.get('run_id')];", + " if (allowedProviders.indexOf(ri.provider) < 0) { throw new Error('routing_info.provider=' + ri.provider + ' not in ' + JSON.stringify(allowedProviders)); }", + " if (Boolean(ri.is_fallback) !== true) { throw new Error('is_fallback=' + ri.is_fallback); }", + " var expectedPrimary = 'catwiring-rt-fallback-cross-provider-' + pm.variables.get('run_id');", + " if (ri.primary_provider !== expectedPrimary) { throw new Error('primary_provider=' + ri.primary_provider + ' expected ' + expectedPrimary); }", + " if (ri.primary_model !== \"gpt-4o-mini\") { throw new Error('primary_model=' + ri.primary_model); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. primary fails; request fallback serves (200, is_fallback) [fallback-cross-provider]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. primary fails; request fallback serves (200, is_fallback) [fallback-cross-provider]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-fallback-cross-provider-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5,\"fallbacks\":[\"catwiring-rt-fallback-cross-provider-b-{{run_id}}/gpt-4o-mini\"]}" + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-fallback-cross-provider-cleanup-provider-self", + "name": "cleanup: delete provider [fallback-cross-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [fallback-cross-provider]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-fallback-cross-provider-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-fallback-cross-provider-{{run_id}}" + ] + } + } + }, + { + "id": "rt-fallback-cross-provider-cleanup-provider-b", + "name": "cleanup: delete provider b [fallback-cross-provider]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider b [fallback-cross-provider]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-fallback-cross-provider-b-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-fallback-cross-provider-b-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Fallbacks are tried in order until one succeeds", + "description": "The primary and the first fallback both have invalid keys; the second fallback succeeds. routing_info reports the surviving provider and still names the original primary.", + "item": [ + { + "id": "rt-fallback-chain-first-healthy-wins-01-add-provider", + "name": "01. add provider [fallback-chain-first-healthy-wins]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [fallback-chain-first-healthy-wins]\";", + "pm.test(\"01. add provider [fallback-chain-first-healthy-wins]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-fallback-chain-first-healthy-wins-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-fallback-chain-first-healthy-wins-02-add-key", + "name": "02. add key kbad1 [fallback-chain-first-healthy-wins]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [fallback-chain-first-healthy-wins]\";", + "pm.test(\"02. add key kbad1 [fallback-chain-first-healthy-wins]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-fallback-chain-first-healthy-wins-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-fallback-chain-first-healthy-wins-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"kbad1-{{run_id}}\",\"name\":\"catwiring-rt-fallback-chain-first-healthy-wins-kbad1-{{run_id}}\",\"models\":[\"gpt-4o-mini\"],\"enabled\":true,\"value\":\"sk-deadbeef-invalid-000\"}" + } + } + }, + { + "id": "rt-fallback-chain-first-healthy-wins-03-add-provider", + "name": "03. add provider (b) [fallback-chain-first-healthy-wins]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [fallback-chain-first-healthy-wins]\";", + "pm.test(\"03. add provider (b) [fallback-chain-first-healthy-wins]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-fallback-chain-first-healthy-wins-b-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-fallback-chain-first-healthy-wins-04-add-key", + "name": "04. add key kbad2 (b) [fallback-chain-first-healthy-wins]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [fallback-chain-first-healthy-wins]\";", + "pm.test(\"04. add key kbad2 (b) [fallback-chain-first-healthy-wins]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-fallback-chain-first-healthy-wins-b-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-fallback-chain-first-healthy-wins-b-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"kbad2-{{run_id}}\",\"name\":\"catwiring-rt-fallback-chain-first-healthy-wins-kbad2-{{run_id}}\",\"models\":[\"gpt-4o-mini\"],\"enabled\":true,\"value\":\"sk-deadbeef-invalid-000\"}" + } + } + }, + { + "id": "rt-fallback-chain-first-healthy-wins-05-add-provider", + "name": "05. add provider (c) [fallback-chain-first-healthy-wins]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [fallback-chain-first-healthy-wins]\";", + "pm.test(\"05. add provider (c) [fallback-chain-first-healthy-wins]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-fallback-chain-first-healthy-wins-c-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-fallback-chain-first-healthy-wins-06-add-key", + "name": "06. add key kgood (c) [fallback-chain-first-healthy-wins]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [fallback-chain-first-healthy-wins]\";", + "pm.test(\"06. add key kgood (c) [fallback-chain-first-healthy-wins]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-fallback-chain-first-healthy-wins-c-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-fallback-chain-first-healthy-wins-c-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"kgood-{{run_id}}\",\"name\":\"catwiring-rt-fallback-chain-first-healthy-wins-kgood-{{run_id}}\",\"models\":[\"gpt-4o-mini\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\"}" + } + } + }, + { + "id": "rt-fallback-chain-first-healthy-wins-07-route", + "name": "07. chain falls through to the only healthy provider (200) [fallback-chain-first-healthy-wins]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 1000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [fallback-chain-first-healthy-wins]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('route status ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " if (!body.choices || body.choices.length === 0) { throw new Error('no choices'); }", + " var ri = (body.extra_fields || {}).routing_info || {};", + " var allowedProviders = ['catwiring-rt-fallback-chain-first-healthy-wins-c-' + pm.variables.get('run_id')];", + " if (allowedProviders.indexOf(ri.provider) < 0) { throw new Error('routing_info.provider=' + ri.provider + ' not in ' + JSON.stringify(allowedProviders)); }", + " if (Boolean(ri.is_fallback) !== true) { throw new Error('is_fallback=' + ri.is_fallback); }", + " var expectedPrimary = 'catwiring-rt-fallback-chain-first-healthy-wins-' + pm.variables.get('run_id');", + " if (ri.primary_provider !== expectedPrimary) { throw new Error('primary_provider=' + ri.primary_provider + ' expected ' + expectedPrimary); }", + " if (ri.primary_model !== \"gpt-4o-mini\") { throw new Error('primary_model=' + ri.primary_model); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"07. chain falls through to the only healthy provider (200) [fallback-chain-first-healthy-wins]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"07. chain falls through to the only healthy provider (200) [fallback-chain-first-healthy-wins]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-fallback-chain-first-healthy-wins-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5,\"fallbacks\":[\"catwiring-rt-fallback-chain-first-healthy-wins-b-{{run_id}}/gpt-4o-mini\",\"catwiring-rt-fallback-chain-first-healthy-wins-c-{{run_id}}/gpt-4o-mini\"]}" + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-fallback-chain-first-healthy-wins-cleanup-provider-self", + "name": "cleanup: delete provider [fallback-chain-first-healthy-wins]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [fallback-chain-first-healthy-wins]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-fallback-chain-first-healthy-wins-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-fallback-chain-first-healthy-wins-{{run_id}}" + ] + } + } + }, + { + "id": "rt-fallback-chain-first-healthy-wins-cleanup-provider-b", + "name": "cleanup: delete provider b [fallback-chain-first-healthy-wins]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider b [fallback-chain-first-healthy-wins]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-fallback-chain-first-healthy-wins-b-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-fallback-chain-first-healthy-wins-b-{{run_id}}" + ] + } + } + }, + { + "id": "rt-fallback-chain-first-healthy-wins-cleanup-provider-c", + "name": "cleanup: delete provider c [fallback-chain-first-healthy-wins]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider c [fallback-chain-first-healthy-wins]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-fallback-chain-first-healthy-wins-c-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-fallback-chain-first-healthy-wins-c-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "A fallback to a provider off the VK allowlist is pruned, not tried", + "description": "Under a VK that permits only the primary provider, a request-level fallback to a healthy off-allowlist provider is pruned before the attempt loop. The primary's invalid key fails with a 401 and the pruned provider — which has a valid key and would otherwise return 200 — never rescues it, proving it was dropped.", + "item": [ + { + "id": "rt-fallback-pruned-by-vk-allowlist-01-add-provider", + "name": "01. add provider [fallback-pruned-by-vk-allowlist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [fallback-pruned-by-vk-allowlist]\";", + "pm.test(\"01. add provider [fallback-pruned-by-vk-allowlist]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-fallback-pruned-by-vk-allowlist-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-fallback-pruned-by-vk-allowlist-02-add-key", + "name": "02. add key kbad [fallback-pruned-by-vk-allowlist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [fallback-pruned-by-vk-allowlist]\";", + "pm.test(\"02. add key kbad [fallback-pruned-by-vk-allowlist]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-fallback-pruned-by-vk-allowlist-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-fallback-pruned-by-vk-allowlist-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"kbad-{{run_id}}\",\"name\":\"catwiring-rt-fallback-pruned-by-vk-allowlist-kbad-{{run_id}}\",\"models\":[\"gpt-4o-mini\"],\"enabled\":true,\"value\":\"sk-deadbeef-invalid-000\"}" + } + } + }, + { + "id": "rt-fallback-pruned-by-vk-allowlist-03-add-provider", + "name": "03. add provider (b) [fallback-pruned-by-vk-allowlist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [fallback-pruned-by-vk-allowlist]\";", + "pm.test(\"03. add provider (b) [fallback-pruned-by-vk-allowlist]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-fallback-pruned-by-vk-allowlist-b-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-fallback-pruned-by-vk-allowlist-04-add-key", + "name": "04. add key kgood (b) [fallback-pruned-by-vk-allowlist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [fallback-pruned-by-vk-allowlist]\";", + "pm.test(\"04. add key kgood (b) [fallback-pruned-by-vk-allowlist]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-fallback-pruned-by-vk-allowlist-b-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-fallback-pruned-by-vk-allowlist-b-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"kgood-{{run_id}}\",\"name\":\"catwiring-rt-fallback-pruned-by-vk-allowlist-kgood-{{run_id}}\",\"models\":[\"gpt-4o-mini\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\"}" + } + } + }, + { + "id": "rt-fallback-pruned-by-vk-allowlist-05-create-vk", + "name": "05. create vk [fallback-pruned-by-vk-allowlist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [fallback-pruned-by-vk-allowlist]\";", + "var ok = pm.response.code === 200 || pm.response.code === 201;", + "pm.test(\"05. create vk [fallback-pruned-by-vk-allowlist]\", function () {", + " pm.expect([200, 201], 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (ok) {", + " var vk = (pm.response.json() || {}).virtual_key || {};", + " pm.collectionVariables.set('vkval_fallback-pruned-by-vk-allowlist', vk.value || '');", + " pm.collectionVariables.set('vkid_fallback-pruned-by-vk-allowlist', vk.id || '');", + "} else { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"name\":\"catwiring-rtvk-fallback-pruned-by-vk-allowlist-{{run_id}}\",\"is_active\":true,\"provider_configs\":[{\"provider\":\"catwiring-rt-fallback-pruned-by-vk-allowlist-{{run_id}}\",\"key_ids\":[\"*\"],\"allowed_models\":[\"*\"],\"blacklisted_models\":[],\"weight\":1}]}" + } + } + }, + { + "id": "rt-fallback-pruned-by-vk-allowlist-06-route", + "name": "06. off-allowlist fallback pruned; request fails on the primary (401) [fallback-pruned-by-vk-allowlist]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 1000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [fallback-pruned-by-vk-allowlist]\";", + "function assertNow() {", + " if (pm.response.code !== 401) { throw new Error('expected status 401 got ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " var msg = (body.error && (body.error.message || body.error)) || body.message || '';", + " if (String(msg).indexOf(\"Incorrect API key\") < 0) { throw new Error('error message=' + msg); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"06. off-allowlist fallback pruned; request fails on the primary (401) [fallback-pruned-by-vk-allowlist]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"06. off-allowlist fallback pruned; request fails on the primary (401) [fallback-pruned-by-vk-allowlist]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_fallback-pruned-by-vk-allowlist}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-fallback-pruned-by-vk-allowlist-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5,\"fallbacks\":[\"catwiring-rt-fallback-pruned-by-vk-allowlist-b-{{run_id}}/gpt-4o-mini\"]}" + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-fallback-pruned-by-vk-allowlist-cleanup-vk", + "name": "cleanup: delete vk [fallback-pruned-by-vk-allowlist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete vk [fallback-pruned-by-vk-allowlist]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys/{{vkid_fallback-pruned-by-vk-allowlist}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys", + "{{vkid_fallback-pruned-by-vk-allowlist}}" + ] + } + } + }, + { + "id": "rt-fallback-pruned-by-vk-allowlist-cleanup-provider-self", + "name": "cleanup: delete provider [fallback-pruned-by-vk-allowlist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [fallback-pruned-by-vk-allowlist]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-fallback-pruned-by-vk-allowlist-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-fallback-pruned-by-vk-allowlist-{{run_id}}" + ] + } + } + }, + { + "id": "rt-fallback-pruned-by-vk-allowlist-cleanup-provider-b", + "name": "cleanup: delete provider b [fallback-pruned-by-vk-allowlist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider b [fallback-pruned-by-vk-allowlist]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-fallback-pruned-by-vk-allowlist-b-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-fallback-pruned-by-vk-allowlist-b-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "A VK with weighted providers auto-attaches the others as fallbacks", + "description": "A VK weights two providers that both serve the model; the dominant-weight provider's key is invalid. With no request-level fallbacks, governance auto-attaches the remaining weighted config as a fallback, so every request still lands on the healthy low-weight provider — whether it was the load-balanced primary or the fallback.", + "item": [ + { + "id": "rt-vk-auto-attached-fallback-01-add-provider", + "name": "01. add provider [vk-auto-attached-fallback]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [vk-auto-attached-fallback]\";", + "pm.test(\"01. add provider [vk-auto-attached-fallback]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-vk-auto-attached-fallback-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-vk-auto-attached-fallback-02-add-key", + "name": "02. add key kbad [vk-auto-attached-fallback]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [vk-auto-attached-fallback]\";", + "pm.test(\"02. add key kbad [vk-auto-attached-fallback]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-vk-auto-attached-fallback-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-vk-auto-attached-fallback-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"kbad-{{run_id}}\",\"name\":\"catwiring-rt-vk-auto-attached-fallback-kbad-{{run_id}}\",\"models\":[\"gpt-4o-mini\"],\"enabled\":true,\"value\":\"sk-deadbeef-invalid-000\"}" + } + } + }, + { + "id": "rt-vk-auto-attached-fallback-03-add-provider", + "name": "03. add provider (b) [vk-auto-attached-fallback]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [vk-auto-attached-fallback]\";", + "pm.test(\"03. add provider (b) [vk-auto-attached-fallback]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-vk-auto-attached-fallback-b-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-vk-auto-attached-fallback-04-add-key", + "name": "04. add key kgood (b) [vk-auto-attached-fallback]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [vk-auto-attached-fallback]\";", + "pm.test(\"04. add key kgood (b) [vk-auto-attached-fallback]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-vk-auto-attached-fallback-b-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-vk-auto-attached-fallback-b-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"kgood-{{run_id}}\",\"name\":\"catwiring-rt-vk-auto-attached-fallback-kgood-{{run_id}}\",\"models\":[\"gpt-4o-mini\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\"}" + } + } + }, + { + "id": "rt-vk-auto-attached-fallback-05-create-vk", + "name": "05. create vk [vk-auto-attached-fallback]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [vk-auto-attached-fallback]\";", + "var ok = pm.response.code === 200 || pm.response.code === 201;", + "pm.test(\"05. create vk [vk-auto-attached-fallback]\", function () {", + " pm.expect([200, 201], 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (ok) {", + " var vk = (pm.response.json() || {}).virtual_key || {};", + " pm.collectionVariables.set('vkval_vk-auto-attached-fallback', vk.value || '');", + " pm.collectionVariables.set('vkid_vk-auto-attached-fallback', vk.id || '');", + "} else { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"name\":\"catwiring-rtvk-vk-auto-attached-fallback-{{run_id}}\",\"is_active\":true,\"provider_configs\":[{\"provider\":\"catwiring-rt-vk-auto-attached-fallback-{{run_id}}\",\"key_ids\":[\"*\"],\"allowed_models\":[\"*\"],\"blacklisted_models\":[],\"weight\":100},{\"provider\":\"catwiring-rt-vk-auto-attached-fallback-b-{{run_id}}\",\"key_ids\":[\"*\"],\"allowed_models\":[\"*\"],\"blacklisted_models\":[],\"weight\":1}]}" + } + } + }, + { + "id": "rt-vk-auto-attached-fallback-06-pdist-sample", + "name": "06. sample 1/6 (gpt-4o-mini) [vk-auto-attached-fallback]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [vk-auto-attached-fallback]\";", + "pm.collectionVariables.set('pdist_vk-auto-attached-fallback', '');", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_vk-auto-attached-fallback') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_vk-auto-attached-fallback', set.join(',')); }", + "}", + "pm.test(\"06. sample 1/6 (gpt-4o-mini) [vk-auto-attached-fallback]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_vk-auto-attached-fallback}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-vk-auto-attached-fallback-07-pdist-sample", + "name": "07. sample 2/6 (gpt-4o-mini) [vk-auto-attached-fallback]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [vk-auto-attached-fallback]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_vk-auto-attached-fallback') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_vk-auto-attached-fallback', set.join(',')); }", + "}", + "pm.test(\"07. sample 2/6 (gpt-4o-mini) [vk-auto-attached-fallback]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_vk-auto-attached-fallback}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-vk-auto-attached-fallback-08-pdist-sample", + "name": "08. sample 3/6 (gpt-4o-mini) [vk-auto-attached-fallback]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [vk-auto-attached-fallback]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_vk-auto-attached-fallback') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_vk-auto-attached-fallback', set.join(',')); }", + "}", + "pm.test(\"08. sample 3/6 (gpt-4o-mini) [vk-auto-attached-fallback]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_vk-auto-attached-fallback}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-vk-auto-attached-fallback-09-pdist-sample", + "name": "09. sample 4/6 (gpt-4o-mini) [vk-auto-attached-fallback]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [vk-auto-attached-fallback]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_vk-auto-attached-fallback') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_vk-auto-attached-fallback', set.join(',')); }", + "}", + "pm.test(\"09. sample 4/6 (gpt-4o-mini) [vk-auto-attached-fallback]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_vk-auto-attached-fallback}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-vk-auto-attached-fallback-10-pdist-sample", + "name": "10. sample 5/6 (gpt-4o-mini) [vk-auto-attached-fallback]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [vk-auto-attached-fallback]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_vk-auto-attached-fallback') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_vk-auto-attached-fallback', set.join(',')); }", + "}", + "pm.test(\"10. sample 5/6 (gpt-4o-mini) [vk-auto-attached-fallback]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_vk-auto-attached-fallback}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-vk-auto-attached-fallback-11-pdist-sample", + "name": "11. sample 6/6 (gpt-4o-mini) [vk-auto-attached-fallback]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [vk-auto-attached-fallback]\";", + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + " var cur = pm.collectionVariables.get('pdist_vk-auto-attached-fallback') || '';", + " var set = cur ? cur.split(',') : [];", + " if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_vk-auto-attached-fallback', set.join(',')); }", + "}", + "pm.test(\"11. sample 6/6 (gpt-4o-mini) [vk-auto-attached-fallback]\", function () {", + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_vk-auto-attached-fallback}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-vk-auto-attached-fallback-12-pdist-assert", + "name": "12. every request lands on the healthy provider via the auto-attached fallback [vk-auto-attached-fallback]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [vk-auto-attached-fallback]\";", + "var ok = true, errMsg = '';", + "try {", + " var seen = (pm.collectionVariables.get('pdist_vk-auto-attached-fallback') || '').split(',').filter(Boolean);", + " var only = ['catwiring-rt-vk-auto-attached-fallback-b-' + pm.variables.get('run_id')];", + " var never = [];", + " var all = [];", + " if (seen.length === 0) throw new Error('no provider observed across samples');", + " seen.forEach(function (p) {", + " if (only.length && only.indexOf(p) < 0) throw new Error('provider ' + p + ' served but not in expectOnly ' + JSON.stringify(only));", + " if (never.indexOf(p) >= 0) throw new Error('provider ' + p + ' served but was expectNever; observed ' + JSON.stringify(seen));", + " });", + " all.forEach(function (p) { if (seen.indexOf(p) < 0) throw new Error('provider ' + p + ' expected to serve but did not; observed ' + JSON.stringify(seen)); });", + "} catch (e) { ok = false; errMsg = e.message; }", + "pm.test(\"12. every request lands on the healthy provider via the auto-attached fallback [vk-auto-attached-fallback]\", function () { if (!ok) throw new Error(errMsg); });", + "if (!ok) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/health", + "host": [ + "{{base_url}}" + ], + "path": [ + "health" + ] + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-vk-auto-attached-fallback-cleanup-vk", + "name": "cleanup: delete vk [vk-auto-attached-fallback]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete vk [vk-auto-attached-fallback]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys/{{vkid_vk-auto-attached-fallback}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys", + "{{vkid_vk-auto-attached-fallback}}" + ] + } + } + }, + { + "id": "rt-vk-auto-attached-fallback-cleanup-provider-self", + "name": "cleanup: delete provider [vk-auto-attached-fallback]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [vk-auto-attached-fallback]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-vk-auto-attached-fallback-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-vk-auto-attached-fallback-{{run_id}}" + ] + } + } + }, + { + "id": "rt-vk-auto-attached-fallback-cleanup-provider-b", + "name": "cleanup: delete provider b [vk-auto-attached-fallback]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider b [vk-auto-attached-fallback]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-vk-auto-attached-fallback-b-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-vk-auto-attached-fallback-b-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Standard (non-custom) openai provider routes", + "description": "A standard openai provider created via the API routes its model. The catalog is datasheet-backed (no live-cache wait). NOTE: standard providers are global singletons — this scenario is NOT run-id-isolated; run serially against a clean instance, not in parallel shards.", + "item": [ + { + "id": "rt-standard-openai-route-01-add-provider", + "name": "01. add provider [standard-openai-route]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [standard-openai-route]\";", + "pm.test(\"01. add provider [standard-openai-route]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"openai\"}" + } + } + }, + { + "id": "rt-standard-openai-route-02-add-key", + "name": "02. add key ko [standard-openai-route]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [standard-openai-route]\";", + "pm.test(\"02. add key ko [standard-openai-route]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/openai/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "openai", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"ko-{{run_id}}\",\"name\":\"catwiring-rt-standard-openai-route-ko-{{run_id}}\",\"models\":[\"gpt-4o-mini\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\"}" + } + } + }, + { + "id": "rt-standard-openai-route-03-route", + "name": "03. standard openai routes its model (200) [standard-openai-route]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [standard-openai-route]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('route status ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " if (!body.choices || body.choices.length === 0) { throw new Error('no choices'); }", + " var ri = (body.extra_fields || {}).routing_info || {};", + " var providerName = \"openai\";", + " if (ri.provider !== providerName) { throw new Error('routing_info.provider=' + ri.provider + ' expected ' + providerName); }", + " var expectedKey = 'catwiring-rt-standard-openai-route-ko-' + pm.variables.get('run_id');", + " if (ri.key !== expectedKey) { throw new Error('routing_info.key=' + ri.key + ' expected ' + expectedKey); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. standard openai routes its model (200) [standard-openai-route]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. standard openai routes its model (200) [standard-openai-route]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"openai/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-standard-openai-route-cleanup-provider-self", + "name": "cleanup: delete provider [standard-openai-route]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [standard-openai-route]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/openai", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "openai" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Governance gates a standard provider", + "description": "A VK over a standard openai provider routes an allowed model and prunes a disallowed one. Serial-only (global provider).", + "item": [ + { + "id": "rt-standard-openai-vk-gate-01-add-provider", + "name": "01. add provider [standard-openai-vk-gate]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [standard-openai-vk-gate]\";", + "pm.test(\"01. add provider [standard-openai-vk-gate]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"openai\"}" + } + } + }, + { + "id": "rt-standard-openai-vk-gate-02-add-key", + "name": "02. add key ko [standard-openai-vk-gate]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [standard-openai-vk-gate]\";", + "pm.test(\"02. add key ko [standard-openai-vk-gate]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/openai/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "openai", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"ko-{{run_id}}\",\"name\":\"catwiring-rt-standard-openai-vk-gate-ko-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\"}" + } + } + }, + { + "id": "rt-standard-openai-vk-gate-03-create-vk", + "name": "03. create vk [standard-openai-vk-gate]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [standard-openai-vk-gate]\";", + "var ok = pm.response.code === 200 || pm.response.code === 201;", + "pm.test(\"03. create vk [standard-openai-vk-gate]\", function () {", + " pm.expect([200, 201], 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (ok) {", + " var vk = (pm.response.json() || {}).virtual_key || {};", + " pm.collectionVariables.set('vkval_standard-openai-vk-gate', vk.value || '');", + " pm.collectionVariables.set('vkid_standard-openai-vk-gate', vk.id || '');", + "} else { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"name\":\"catwiring-rtvk-standard-openai-vk-gate-{{run_id}}\",\"is_active\":true,\"provider_configs\":[{\"provider\":\"openai\",\"key_ids\":[\"*\"],\"allowed_models\":[\"gpt-4o-mini\"],\"blacklisted_models\":[],\"weight\":1}]}" + } + } + }, + { + "id": "rt-standard-openai-vk-gate-04-route", + "name": "04. VK-allowed model routes on standard provider (200) [standard-openai-vk-gate]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [standard-openai-vk-gate]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('route status ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " if (!body.choices || body.choices.length === 0) { throw new Error('no choices'); }", + " var ri = (body.extra_fields || {}).routing_info || {};", + " var providerName = \"openai\";", + " if (ri.provider !== providerName) { throw new Error('routing_info.provider=' + ri.provider + ' expected ' + providerName); }", + " var expectedKey = 'catwiring-rt-standard-openai-vk-gate-ko-' + pm.variables.get('run_id');", + " if (ri.key !== expectedKey) { throw new Error('routing_info.key=' + ri.key + ' expected ' + expectedKey); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. VK-allowed model routes on standard provider (200) [standard-openai-vk-gate]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. VK-allowed model routes on standard provider (200) [standard-openai-vk-gate]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_standard-openai-vk-gate}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"openai/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "id": "rt-standard-openai-vk-gate-05-assert-log", + "name": "05. log records standard-provider route [standard-openai-vk-gate]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 2000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [standard-openai-vk-gate]\";", + "function assertNow() {", + " var providerName = \"openai\";", + " if (pm.response.code !== 200) { throw new Error('logs status ' + pm.response.code); }", + " var logs = (pm.response.json() || {}).logs || [];", + " var model = \"gpt-4o-mini\";", + " var row = null;", + " for (var i = 0; i < logs.length; i++) {", + " if (logs[i].provider === providerName && logs[i].model === model) { row = logs[i]; break; }", + " }", + " if (!row) { throw new Error('no log row for ' + providerName + '/' + model + ' in ' + logs.length + ' rows'); }", + " if (row.status !== \"success\") { throw new Error('log status=' + row.status); }", + " var expectedKey = 'catwiring-rt-standard-openai-vk-gate-ko-' + pm.variables.get('run_id');", + " if (row.selected_key_name !== expectedKey) { throw new Error('selected_key_name=' + row.selected_key_name + ' expected ' + expectedKey); }", + " if (!row.virtual_key_id) { throw new Error('log row missing virtual_key_id'); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. log records standard-provider route [standard-openai-vk-gate]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"05. log records standard-provider route [standard-openai-vk-gate]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/logs?virtual_key_ids={{vkid_standard-openai-vk-gate}}&limit=10", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "logs" + ], + "query": [ + { + "key": "virtual_key_ids", + "value": "{{vkid_standard-openai-vk-gate}}" + }, + { + "key": "limit", + "value": "10" + } + ] + } + } + }, + { + "id": "rt-standard-openai-vk-gate-06-route", + "name": "06. VK-disallowed model pruned (400) [standard-openai-vk-gate]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [standard-openai-vk-gate]\";", + "function assertNow() {", + " if (pm.response.code !== 400) { throw new Error('expected status 400 got ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " var msg = (body.error && (body.error.message || body.error)) || body.message || '';", + " if (String(msg).indexOf(\"is not permitted for this request\") < 0) { throw new Error('error message=' + msg); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"06. VK-disallowed model pruned (400) [standard-openai-vk-gate]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"06. VK-disallowed model pruned (400) [standard-openai-vk-gate]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_standard-openai-vk-gate}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"openai/gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-standard-openai-vk-gate-cleanup-vk", + "name": "cleanup: delete vk [standard-openai-vk-gate]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete vk [standard-openai-vk-gate]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys/{{vkid_standard-openai-vk-gate}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys", + "{{vkid_standard-openai-vk-gate}}" + ] + } + } + }, + { + "id": "rt-standard-openai-vk-gate-cleanup-provider-self", + "name": "cleanup: delete provider [standard-openai-vk-gate]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [standard-openai-vk-gate]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/openai", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "openai" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "VK model whitelist blocks an unlisted model", + "description": "A model outside the VK's allowed_models prunes the (single) provider from the routing allowlist, so core rejects with a 'provider not permitted' 400 (intended).", + "item": [ + { + "id": "rt-vk-model-whitelist-01-add-provider", + "name": "01. add provider [vk-model-whitelist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [vk-model-whitelist]\";", + "pm.test(\"01. add provider [vk-model-whitelist]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-vk-model-whitelist-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-vk-model-whitelist-02-add-key", + "name": "02. add key k1 [vk-model-whitelist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [vk-model-whitelist]\";", + "pm.test(\"02. add key k1 [vk-model-whitelist]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-vk-model-whitelist-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-vk-model-whitelist-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"k1-{{run_id}}\",\"name\":\"catwiring-rt-vk-model-whitelist-k1-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\"}" + } + } + }, + { + "id": "rt-vk-model-whitelist-03-create-vk", + "name": "03. create vk [vk-model-whitelist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [vk-model-whitelist]\";", + "var ok = pm.response.code === 200 || pm.response.code === 201;", + "pm.test(\"03. create vk [vk-model-whitelist]\", function () {", + " pm.expect([200, 201], 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (ok) {", + " var vk = (pm.response.json() || {}).virtual_key || {};", + " pm.collectionVariables.set('vkval_vk-model-whitelist', vk.value || '');", + " pm.collectionVariables.set('vkid_vk-model-whitelist', vk.id || '');", + "} else { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"name\":\"catwiring-rtvk-vk-model-whitelist-{{run_id}}\",\"is_active\":true,\"provider_configs\":[{\"provider\":\"catwiring-rt-vk-model-whitelist-{{run_id}}\",\"key_ids\":[\"*\"],\"allowed_models\":[\"gpt-4o-mini\"],\"blacklisted_models\":[],\"weight\":1}]}" + } + } + }, + { + "id": "rt-vk-model-whitelist-04-route", + "name": "04. unlisted model rejected via empty allowlist (400) [vk-model-whitelist]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [vk-model-whitelist]\";", + "function assertNow() {", + " if (pm.response.code !== 400) { throw new Error('expected status 400 got ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " var msg = (body.error && (body.error.message || body.error)) || body.message || '';", + " if (String(msg).indexOf(\"is not permitted for this request\") < 0) { throw new Error('error message=' + msg); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. unlisted model rejected via empty allowlist (400) [vk-model-whitelist]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. unlisted model rejected via empty allowlist (400) [vk-model-whitelist]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_vk-model-whitelist}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-vk-model-whitelist-{{run_id}}/gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-vk-model-whitelist-cleanup-vk", + "name": "cleanup: delete vk [vk-model-whitelist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete vk [vk-model-whitelist]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys/{{vkid_vk-model-whitelist}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys", + "{{vkid_vk-model-whitelist}}" + ] + } + } + }, + { + "id": "rt-vk-model-whitelist-cleanup-provider-self", + "name": "cleanup: delete provider [vk-model-whitelist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [vk-model-whitelist]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-vk-model-whitelist-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-vk-model-whitelist-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "VK blacklist blocks a model", + "description": "A model in the VK's blacklisted_models prunes the (single) provider from the routing allowlist, so core rejects with a 'provider not permitted' 400 (intended).", + "item": [ + { + "id": "rt-vk-blacklist-01-add-provider", + "name": "01. add provider [vk-blacklist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [vk-blacklist]\";", + "pm.test(\"01. add provider [vk-blacklist]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-vk-blacklist-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-vk-blacklist-02-add-key", + "name": "02. add key k1 [vk-blacklist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [vk-blacklist]\";", + "pm.test(\"02. add key k1 [vk-blacklist]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-vk-blacklist-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-vk-blacklist-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"k1-{{run_id}}\",\"name\":\"catwiring-rt-vk-blacklist-k1-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\"}" + } + } + }, + { + "id": "rt-vk-blacklist-03-create-vk", + "name": "03. create vk [vk-blacklist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [vk-blacklist]\";", + "var ok = pm.response.code === 200 || pm.response.code === 201;", + "pm.test(\"03. create vk [vk-blacklist]\", function () {", + " pm.expect([200, 201], 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (ok) {", + " var vk = (pm.response.json() || {}).virtual_key || {};", + " pm.collectionVariables.set('vkval_vk-blacklist', vk.value || '');", + " pm.collectionVariables.set('vkid_vk-blacklist', vk.id || '');", + "} else { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"name\":\"catwiring-rtvk-vk-blacklist-{{run_id}}\",\"is_active\":true,\"provider_configs\":[{\"provider\":\"catwiring-rt-vk-blacklist-{{run_id}}\",\"key_ids\":[\"*\"],\"allowed_models\":[\"*\"],\"blacklisted_models\":[\"gpt-4o\"],\"weight\":1}]}" + } + } + }, + { + "id": "rt-vk-blacklist-04-route", + "name": "04. blacklisted model rejected via empty allowlist (400) [vk-blacklist]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [vk-blacklist]\";", + "function assertNow() {", + " if (pm.response.code !== 400) { throw new Error('expected status 400 got ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " var msg = (body.error && (body.error.message || body.error)) || body.message || '';", + " if (String(msg).indexOf(\"is not permitted for this request\") < 0) { throw new Error('error message=' + msg); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. blacklisted model rejected via empty allowlist (400) [vk-blacklist]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. blacklisted model rejected via empty allowlist (400) [vk-blacklist]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_vk-blacklist}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-vk-blacklist-{{run_id}}/gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-vk-blacklist-cleanup-vk", + "name": "cleanup: delete vk [vk-blacklist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete vk [vk-blacklist]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys/{{vkid_vk-blacklist}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys", + "{{vkid_vk-blacklist}}" + ] + } + } + }, + { + "id": "rt-vk-blacklist-cleanup-provider-self", + "name": "cleanup: delete provider [vk-blacklist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [vk-blacklist]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-vk-blacklist-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-vk-blacklist-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "VK wildcard is still bounded by the key gate", + "description": "Governance's model check is catalog-aware, so a VK allowed_models=[\"*\"] does not widen past what the key actually gates; a model outside the key's allow-list is blocked by governance.", + "item": [ + { + "id": "rt-vk-wildcard-bounded-by-key-gate-01-add-provider", + "name": "01. add provider [vk-wildcard-bounded-by-key-gate]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [vk-wildcard-bounded-by-key-gate]\";", + "pm.test(\"01. add provider [vk-wildcard-bounded-by-key-gate]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-vk-wildcard-bounded-by-key-gate-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-vk-wildcard-bounded-by-key-gate-02-add-key", + "name": "02. add key k1 [vk-wildcard-bounded-by-key-gate]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [vk-wildcard-bounded-by-key-gate]\";", + "pm.test(\"02. add key k1 [vk-wildcard-bounded-by-key-gate]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-vk-wildcard-bounded-by-key-gate-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-vk-wildcard-bounded-by-key-gate-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"k1-{{run_id}}\",\"name\":\"catwiring-rt-vk-wildcard-bounded-by-key-gate-k1-{{run_id}}\",\"models\":[\"gpt-4o-mini\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\"}" + } + } + }, + { + "id": "rt-vk-wildcard-bounded-by-key-gate-03-create-vk", + "name": "03. create vk [vk-wildcard-bounded-by-key-gate]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [vk-wildcard-bounded-by-key-gate]\";", + "var ok = pm.response.code === 200 || pm.response.code === 201;", + "pm.test(\"03. create vk [vk-wildcard-bounded-by-key-gate]\", function () {", + " pm.expect([200, 201], 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (ok) {", + " var vk = (pm.response.json() || {}).virtual_key || {};", + " pm.collectionVariables.set('vkval_vk-wildcard-bounded-by-key-gate', vk.value || '');", + " pm.collectionVariables.set('vkid_vk-wildcard-bounded-by-key-gate', vk.id || '');", + "} else { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"name\":\"catwiring-rtvk-vk-wildcard-bounded-by-key-gate-{{run_id}}\",\"is_active\":true,\"provider_configs\":[{\"provider\":\"catwiring-rt-vk-wildcard-bounded-by-key-gate-{{run_id}}\",\"key_ids\":[\"*\"],\"allowed_models\":[\"*\"],\"blacklisted_models\":[],\"weight\":1}]}" + } + } + }, + { + "id": "rt-vk-wildcard-bounded-by-key-gate-04-route", + "name": "04. wildcard VK still blocks a model the key does not gate (403) [vk-wildcard-bounded-by-key-gate]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [vk-wildcard-bounded-by-key-gate]\";", + "function assertNow() {", + " if (pm.response.code !== 403) { throw new Error('expected status 403 got ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " var msg = (body.error && (body.error.message || body.error)) || body.message || '';", + " if (String(msg).indexOf(\"is not allowed for this virtual key\") < 0) { throw new Error('error message=' + msg); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. wildcard VK still blocks a model the key does not gate (403) [vk-wildcard-bounded-by-key-gate]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. wildcard VK still blocks a model the key does not gate (403) [vk-wildcard-bounded-by-key-gate]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_vk-wildcard-bounded-by-key-gate}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-vk-wildcard-bounded-by-key-gate-{{run_id}}/gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-vk-wildcard-bounded-by-key-gate-cleanup-vk", + "name": "cleanup: delete vk [vk-wildcard-bounded-by-key-gate]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete vk [vk-wildcard-bounded-by-key-gate]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys/{{vkid_vk-wildcard-bounded-by-key-gate}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys", + "{{vkid_vk-wildcard-bounded-by-key-gate}}" + ] + } + } + }, + { + "id": "rt-vk-wildcard-bounded-by-key-gate-cleanup-provider-self", + "name": "cleanup: delete provider [vk-wildcard-bounded-by-key-gate]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [vk-wildcard-bounded-by-key-gate]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-vk-wildcard-bounded-by-key-gate-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-vk-wildcard-bounded-by-key-gate-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Catalog/key gate blocks a model with no VK (400)", + "description": "Without a virtual key, governance does not run; a model the key does not allow is rejected by core key selection.", + "item": [ + { + "id": "rt-catalog-gate-blocks-no-vk-01-add-provider", + "name": "01. add provider [catalog-gate-blocks-no-vk]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [catalog-gate-blocks-no-vk]\";", + "pm.test(\"01. add provider [catalog-gate-blocks-no-vk]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-catalog-gate-blocks-no-vk-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-catalog-gate-blocks-no-vk-02-add-key", + "name": "02. add key k1 [catalog-gate-blocks-no-vk]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete provider [catalog-gate-blocks-no-vk]\";", + "pm.test(\"02. add key k1 [catalog-gate-blocks-no-vk]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-catalog-gate-blocks-no-vk-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-catalog-gate-blocks-no-vk-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"k1-{{run_id}}\",\"name\":\"catwiring-rt-catalog-gate-blocks-no-vk-k1-{{run_id}}\",\"models\":[\"gpt-4o-mini\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\"}" + } + } + }, + { + "id": "rt-catalog-gate-blocks-no-vk-03-route", + "name": "03. key gate rejects unlisted model (400) [catalog-gate-blocks-no-vk]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete provider [catalog-gate-blocks-no-vk]\";", + "function assertNow() {", + " if (pm.response.code !== 400) { throw new Error('expected status 400 got ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " var msg = (body.error && (body.error.message || body.error)) || body.message || '';", + " if (String(msg).indexOf(\"no keys found that support model\") < 0) { throw new Error('error message=' + msg); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. key gate rejects unlisted model (400) [catalog-gate-blocks-no-vk]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"03. key gate rejects unlisted model (400) [catalog-gate-blocks-no-vk]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-catalog-gate-blocks-no-vk-{{run_id}}/gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-catalog-gate-blocks-no-vk-cleanup-provider-self", + "name": "cleanup: delete provider [catalog-gate-blocks-no-vk]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [catalog-gate-blocks-no-vk]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-catalog-gate-blocks-no-vk-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-catalog-gate-blocks-no-vk-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Alias whitelisted by name resolves", + "description": "When the VK whitelists the alias name, the request routes and resolves to the underlying model id.", + "item": [ + { + "id": "rt-alias-whitelisted-by-name-01-add-provider", + "name": "01. add provider [alias-whitelisted-by-name]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [alias-whitelisted-by-name]\";", + "pm.test(\"01. add provider [alias-whitelisted-by-name]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-alias-whitelisted-by-name-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-alias-whitelisted-by-name-02-add-key", + "name": "02. add key k1 [alias-whitelisted-by-name]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [alias-whitelisted-by-name]\";", + "pm.test(\"02. add key k1 [alias-whitelisted-by-name]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-alias-whitelisted-by-name-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-alias-whitelisted-by-name-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"k1-{{run_id}}\",\"name\":\"catwiring-rt-alias-whitelisted-by-name-k1-{{run_id}}\",\"models\":[\"catwiring-alias-{{run_id}}\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\",\"aliases\":{\"catwiring-alias-{{run_id}}\":\"gpt-4o-mini\"}}" + } + } + }, + { + "id": "rt-alias-whitelisted-by-name-03-create-vk", + "name": "03. create vk [alias-whitelisted-by-name]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [alias-whitelisted-by-name]\";", + "var ok = pm.response.code === 200 || pm.response.code === 201;", + "pm.test(\"03. create vk [alias-whitelisted-by-name]\", function () {", + " pm.expect([200, 201], 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (ok) {", + " var vk = (pm.response.json() || {}).virtual_key || {};", + " pm.collectionVariables.set('vkval_alias-whitelisted-by-name', vk.value || '');", + " pm.collectionVariables.set('vkid_alias-whitelisted-by-name', vk.id || '');", + "} else { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"name\":\"catwiring-rtvk-alias-whitelisted-by-name-{{run_id}}\",\"is_active\":true,\"provider_configs\":[{\"provider\":\"catwiring-rt-alias-whitelisted-by-name-{{run_id}}\",\"key_ids\":[\"*\"],\"allowed_models\":[\"catwiring-alias-{{run_id}}\"],\"blacklisted_models\":[],\"weight\":1}]}" + } + } + }, + { + "id": "rt-alias-whitelisted-by-name-04-route", + "name": "04. alias routes, resolves to model id [alias-whitelisted-by-name]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + "if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < 1000) {} }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [alias-whitelisted-by-name]\";", + "function assertNow() {", + " if (pm.response.code !== 200) { throw new Error('route status ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " if (!body.choices || body.choices.length === 0) { throw new Error('no choices'); }", + " var ri = (body.extra_fields || {}).routing_info || {};", + " var providerName = 'catwiring-rt-alias-whitelisted-by-name-' + pm.variables.get('run_id');", + " if (ri.provider !== providerName) { throw new Error('routing_info.provider=' + ri.provider + ' expected ' + providerName); }", + " if (!ri.resolved_key_alias || ri.resolved_key_alias.model_id !== \"gpt-4o-mini\") { throw new Error('resolved_key_alias=' + JSON.stringify(ri.resolved_key_alias)); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. alias routes, resolves to model id [alias-whitelisted-by-name]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. alias routes, resolves to model id [alias-whitelisted-by-name]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_alias-whitelisted-by-name}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-alias-whitelisted-by-name-{{run_id}}/catwiring-alias-{{run_id}}\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-alias-whitelisted-by-name-cleanup-vk", + "name": "cleanup: delete vk [alias-whitelisted-by-name]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete vk [alias-whitelisted-by-name]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys/{{vkid_alias-whitelisted-by-name}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys", + "{{vkid_alias-whitelisted-by-name}}" + ] + } + } + }, + { + "id": "rt-alias-whitelisted-by-name-cleanup-provider-self", + "name": "cleanup: delete provider [alias-whitelisted-by-name]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [alias-whitelisted-by-name]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-alias-whitelisted-by-name-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-alias-whitelisted-by-name-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "Alias name not whitelisted is blocked", + "description": "The VK whitelists the resolved model id, but the request uses the alias name; the alias isn't in allowed_models, so the provider is pruned from the routing allowlist and core rejects with a 'provider not permitted' 400 (intended).", + "item": [ + { + "id": "rt-alias-vs-whitelist-01-add-provider", + "name": "01. add provider [alias-vs-whitelist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [alias-vs-whitelist]\";", + "pm.test(\"01. add provider [alias-vs-whitelist]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-alias-vs-whitelist-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-alias-vs-whitelist-02-add-key", + "name": "02. add key k1 [alias-vs-whitelist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [alias-vs-whitelist]\";", + "pm.test(\"02. add key k1 [alias-vs-whitelist]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-alias-vs-whitelist-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-alias-vs-whitelist-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"k1-{{run_id}}\",\"name\":\"catwiring-rt-alias-vs-whitelist-k1-{{run_id}}\",\"models\":[\"catwiring-alias-{{run_id}}\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\",\"aliases\":{\"catwiring-alias-{{run_id}}\":\"gpt-4o-mini\"}}" + } + } + }, + { + "id": "rt-alias-vs-whitelist-03-create-vk", + "name": "03. create vk [alias-vs-whitelist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [alias-vs-whitelist]\";", + "var ok = pm.response.code === 200 || pm.response.code === 201;", + "pm.test(\"03. create vk [alias-vs-whitelist]\", function () {", + " pm.expect([200, 201], 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (ok) {", + " var vk = (pm.response.json() || {}).virtual_key || {};", + " pm.collectionVariables.set('vkval_alias-vs-whitelist', vk.value || '');", + " pm.collectionVariables.set('vkid_alias-vs-whitelist', vk.id || '');", + "} else { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"name\":\"catwiring-rtvk-alias-vs-whitelist-{{run_id}}\",\"is_active\":true,\"provider_configs\":[{\"provider\":\"catwiring-rt-alias-vs-whitelist-{{run_id}}\",\"key_ids\":[\"*\"],\"allowed_models\":[\"gpt-4o-mini\"],\"blacklisted_models\":[],\"weight\":1}]}" + } + } + }, + { + "id": "rt-alias-vs-whitelist-04-route", + "name": "04. unlisted alias rejected via empty allowlist (400) [alias-vs-whitelist]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [alias-vs-whitelist]\";", + "function assertNow() {", + " if (pm.response.code !== 400) { throw new Error('expected status 400 got ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " var msg = (body.error && (body.error.message || body.error)) || body.message || '';", + " if (String(msg).indexOf(\"is not permitted for this request\") < 0) { throw new Error('error message=' + msg); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. unlisted alias rejected via empty allowlist (400) [alias-vs-whitelist]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. unlisted alias rejected via empty allowlist (400) [alias-vs-whitelist]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_alias-vs-whitelist}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-alias-vs-whitelist-{{run_id}}/catwiring-alias-{{run_id}}\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-alias-vs-whitelist-cleanup-vk", + "name": "cleanup: delete vk [alias-vs-whitelist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete vk [alias-vs-whitelist]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys/{{vkid_alias-vs-whitelist}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys", + "{{vkid_alias-vs-whitelist}}" + ] + } + } + }, + { + "id": "rt-alias-vs-whitelist-cleanup-provider-self", + "name": "cleanup: delete provider [alias-vs-whitelist]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [alias-vs-whitelist]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-alias-vs-whitelist-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-alias-vs-whitelist-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + }, + { + "name": "VK with no provider configs blocks everything", + "description": "An empty provider_configs is deny-by-default; the provider is not permitted.", + "item": [ + { + "id": "rt-vk-empty-configs-01-add-provider", + "name": "01. add provider [vk-empty-configs]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [vk-empty-configs]\";", + "pm.test(\"01. add provider [vk-empty-configs]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"provider\":\"catwiring-rt-vk-empty-configs-{{run_id}}\",\"custom_provider_config\":{\"base_provider_type\":\"openai\",\"is_key_less\":false}}" + } + } + }, + { + "id": "rt-vk-empty-configs-02-add-key", + "name": "02. add key k1 [vk-empty-configs]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var acceptable = [200,201];", + "var cleanupReq = \"cleanup: delete vk [vk-empty-configs]\";", + "pm.test(\"02. add key k1 [vk-empty-configs]\", function () {", + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-vk-empty-configs-{{run_id}}/keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-vk-empty-configs-{{run_id}}", + "keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"id\":\"k1-{{run_id}}\",\"name\":\"catwiring-rt-vk-empty-configs-k1-{{run_id}}\",\"models\":[\"*\"],\"enabled\":true,\"value\":\"env.OPENAI_API_KEY\"}" + } + } + }, + { + "id": "rt-vk-empty-configs-03-create-vk", + "name": "03. create vk [vk-empty-configs]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var cleanupReq = \"cleanup: delete vk [vk-empty-configs]\";", + "var ok = pm.response.code === 200 || pm.response.code === 201;", + "pm.test(\"03. create vk [vk-empty-configs]\", function () {", + " pm.expect([200, 201], 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (ok) {", + " var vk = (pm.response.json() || {}).virtual_key || {};", + " pm.collectionVariables.set('vkval_vk-empty-configs', vk.value || '');", + " pm.collectionVariables.set('vkid_vk-empty-configs', vk.id || '');", + "} else { pm.execution.setNextRequest(cleanupReq); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"name\":\"catwiring-rtvk-vk-empty-configs-{{run_id}}\",\"is_active\":true,\"provider_configs\":[]}" + } + } + }, + { + "id": "rt-vk-empty-configs-04-route", + "name": "04. deny-by-default blocks request (400 routing allowlist) [vk-empty-configs]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var maxAttempts = 8;", + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + "var cleanupReq = \"cleanup: delete vk [vk-empty-configs]\";", + "function assertNow() {", + " if (pm.response.code !== 400) { throw new Error('expected status 400 got ' + pm.response.code + ' body ' + pm.response.text()); }", + " var body = pm.response.json();", + " var msg = (body.error && (body.error.message || body.error)) || body.message || '';", + " if (String(msg).indexOf(\"is not permitted for this request\") < 0) { throw new Error('error message=' + msg); }", + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. deny-by-default blocks request (400 routing allowlist) [vk-empty-configs]\", function () { pm.expect(true, 'assertion satisfied').to.be.true; });", + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + " pm.test(\"04. deny-by-default blocks request (400 routing allowlist) [vk-empty-configs]\", function () { throw new Error(errMsg); });", + " pm.execution.setNextRequest(cleanupReq);", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkval_vk-empty-configs}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{base_url}}/v1/chat/completions", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\"model\":\"catwiring-rt-vk-empty-configs-{{run_id}}/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok.\"}],\"max_tokens\":5}" + } + } + }, + { + "name": "Cleanup", + "item": [ + { + "id": "rt-vk-empty-configs-cleanup-vk", + "name": "cleanup: delete vk [vk-empty-configs]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete vk [vk-empty-configs]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys/{{vkid_vk-empty-configs}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys", + "{{vkid_vk-empty-configs}}" + ] + } + } + }, + { + "id": "rt-vk-empty-configs-cleanup-provider-self", + "name": "cleanup: delete provider [vk-empty-configs]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"cleanup: delete provider [vk-empty-configs]\", function () {", + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/providers/catwiring-rt-vk-empty-configs-{{run_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "providers", + "catwiring-rt-vk-empty-configs-{{run_id}}" + ] + } + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var required = [];", + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}" + ] + } + } + ] + } + ] +} diff --git a/tests/e2e/api/collections/provider-harness.json b/tests/e2e/api/collections/provider-harness.json index ba55f4881f..c6227a8190 100644 --- a/tests/e2e/api/collections/provider-harness.json +++ b/tests/e2e/api/collections/provider-harness.json @@ -36,7 +36,9 @@ "", "// Only register the content-shape test when the request actually succeeded.", "// Otherwise newman reports it as passing (because of an early return) which is misleading next to a failing status.", - "if (pm.response.code < 400 && (pm.response.headers.get('content-type') || '').indexOf('text/event-stream') === -1 && (pm.response.headers.get('content-type') || '').indexOf('vnd.amazon.eventstream') === -1) {", + "var __u = (pm.request && pm.request.url && pm.request.url.toString()) || '';", + "var __skipShape = (__u.indexOf('/files/') !== -1 && (__u.indexOf('/content?') !== -1 || __u.slice(-8) === '/content')) || __u.indexOf('storage.googleapis.com') !== -1 || __u.indexOf('batchPredictionJobs') !== -1;", + "if (pm.response.code < 400 && !__skipShape && (pm.response.headers.get('content-type') || '').indexOf('text/event-stream') === -1 && (pm.response.headers.get('content-type') || '').indexOf('vnd.amazon.eventstream') === -1) {", " pm.test('Response has content (text or tool_use)', function () {", " var j;", " try { j = pm.response.json(); } catch (e) {", @@ -117,6 +119,9 @@ " } else if (j.background === true && typeof j.id === 'string') {", " shape = 'openai-responses-background';", " hasContent = !!j.id;", + " } else if (j.object === 'file' && typeof j.id === 'string') {", + " shape = 'file-object';", + " hasContent = !!j.id;", " }", " pm.expect(hasContent, 'expected non-empty content (shape=' + shape + ', body=' + JSON.stringify(j).slice(0, 200) + ')').to.be.true;", " });", @@ -136,7 +141,11 @@ { "key": "genaiModel", "value": "gemini-2.5-pro", "type": "string" }, { "key": "vertexModel", "value": "gemini-2.5-pro", "type": "string" }, { "key": "bedrockGuardrailIdentifier", "value": "", "type": "string" }, - { "key": "bedrockGuardrailVersion", "value": "DRAFT", "type": "string" } + { "key": "bedrockGuardrailVersion", "value": "DRAFT", "type": "string" }, + { "key": "vertexGcsBucket", "value": "replace-me-bucket", "type": "string" }, + { "key": "vertexGcsPrefix", "value": "bifrost-e2e/", "type": "string" }, + { "key": "vertexProject", "value": "replace-me-project", "type": "string" }, + { "key": "vertexLocation", "value": "us-central1", "type": "string" } ], "item": [ { @@ -857,7 +866,10 @@ { "name": "8.6.4.A native chat → anthropic/claude-opus-4-7 (thinking)", "request": { "method": "POST", "header": [{ "key": "Content-Type", "value": "application/json" }], "body": { "mode": "raw", "raw": "{\n \"model\": \"anthropic/claude-opus-4-7\",\n \"messages\": [{ \"role\": \"user\", \"content\": \"What is 17 * 23? Think step by step.\" }],\n \"max_tokens\": 2048,\n \"thinking\": { \"type\": \"enabled\", \"budget_tokens\": 1024 }\n}" }, "url": { "raw": "{{baseUrl}}/v1/chat/completions", "host": ["{{baseUrl}}"], "path": ["v1", "chat", "completions"] } } }, { "name": "8.6.4.A native chat → anthropic/claude-opus-4-8 (thinking)", "request": { "method": "POST", "header": [{ "key": "Content-Type", "value": "application/json" }], "body": { "mode": "raw", "raw": "{\n \"model\": \"anthropic/claude-opus-4-8\",\n \"messages\": [{ \"role\": \"user\", \"content\": \"What is 17 * 23? Think step by step.\" }],\n \"max_tokens\": 2048,\n \"thinking\": { \"type\": \"adaptive\" }\n}" }, "url": { "raw": "{{baseUrl}}/v1/chat/completions", "host": ["{{baseUrl}}"], "path": ["v1", "chat", "completions"] } } }, { "name": "8.6.4.A native chat → gemini/gemini-2.5-flash (reasoning_effort translated)", "request": { "method": "POST", "header": [{ "key": "Content-Type", "value": "application/json" }], "body": { "mode": "raw", "raw": "{\n \"model\": \"gemini/gemini-2.5-flash\",\n \"messages\": [{ \"role\": \"user\", \"content\": \"What is 17 * 23? Think step by step.\" }],\n \"reasoning_effort\": \"low\"\n}" }, "url": { "raw": "{{baseUrl}}/v1/chat/completions", "host": ["{{baseUrl}}"], "path": ["v1", "chat", "completions"] } } }, - { "name": "8.6.4.E /anthropic/v1/messages → openai/gpt-5 (thinking translated)", "request": { "method": "POST", "header": [{ "key": "Content-Type", "value": "application/json" }], "body": { "mode": "raw", "raw": "{\n \"model\": \"openai/gpt-5\",\n \"max_tokens\": 2048,\n \"messages\": [{ \"role\": \"user\", \"content\": \"What is 17 * 23? Think step by step.\" }],\n \"thinking\": { \"type\": \"enabled\", \"budget_tokens\": 1024 }\n}" }, "url": { "raw": "{{baseUrl}}/anthropic/v1/messages", "host": ["{{baseUrl}}"], "path": ["anthropic", "v1", "messages"] } } } + { "name": "8.6.4.E /anthropic/v1/messages → openai/gpt-5 (thinking translated)", "request": { "method": "POST", "header": [{ "key": "Content-Type", "value": "application/json" }], "body": { "mode": "raw", "raw": "{\n \"model\": \"openai/gpt-5\",\n \"max_tokens\": 2048,\n \"messages\": [{ \"role\": \"user\", \"content\": \"What is 17 * 23? Think step by step.\" }],\n \"thinking\": { \"type\": \"enabled\", \"budget_tokens\": 1024 }\n}" }, "url": { "raw": "{{baseUrl}}/anthropic/v1/messages", "host": ["{{baseUrl}}"], "path": ["anthropic", "v1", "messages"] } } }, + { "name": "8.6.4.A native chat → vertex/moonshotai/kimi-k2-thinking-maas (reasoning_effort none → dropped)", "request": { "method": "POST", "header": [{ "key": "Content-Type", "value": "application/json" }], "body": { "mode": "raw", "raw": "{\n \"model\": \"vertex/moonshotai/kimi-k2-thinking-maas\",\n \"messages\": [{ \"role\": \"user\", \"content\": \"What is 17 * 23? Think step by step.\" }],\n \"reasoning_effort\": \"none\"\n}" }, "url": { "raw": "{{baseUrl}}/v1/chat/completions", "host": ["{{baseUrl}}"], "path": ["v1", "chat", "completions"] } } }, + { "name": "8.6.4.A native chat → vertex/minimaxai/minimax-m2-maas (reasoning_effort none → dropped)", "request": { "method": "POST", "header": [{ "key": "Content-Type", "value": "application/json" }], "body": { "mode": "raw", "raw": "{\n \"model\": \"vertex/minimaxai/minimax-m2-maas\",\n \"messages\": [{ \"role\": \"user\", \"content\": \"What is 17 * 23? Think step by step.\" }],\n \"reasoning_effort\": \"none\"\n}" }, "url": { "raw": "{{baseUrl}}/v1/chat/completions", "host": ["{{baseUrl}}"], "path": ["v1", "chat", "completions"] } } }, + { "name": "8.6.4.B native /v1/responses → vertex/moonshotai/kimi-k2-thinking-maas (reasoning none → dropped)", "request": { "method": "POST", "header": [{ "key": "Content-Type", "value": "application/json" }], "body": { "mode": "raw", "raw": "{\n \"model\": \"vertex/moonshotai/kimi-k2-thinking-maas\",\n \"input\": \"What is 17 * 23? Think step by step.\",\n \"reasoning\": { \"effort\": \"none\" }\n}" }, "url": { "raw": "{{baseUrl}}/v1/responses", "host": ["{{baseUrl}}"], "path": ["v1", "responses"] } } } ] }, { @@ -1913,132 +1925,849 @@ ] }, { - "name": "12. Backlog Coverage (auto-added missing cases)", - "description": "Comprehensive coverage of features sourced from each provider's docs. Organized by provider. Many entries will fail in environments without the corresponding model/feature provisioned - that's expected; they exist to surface gaps via the failure report.", + "name": "11b. Vertex GCS Files (/openai + native resumable)", + "description": "Vertex stores files in a customer GCS bucket. CRUD runs via the OpenAI drop-in (storage_config.gcs); resumable upload runs via the native API (mint session -> client PUTs bytes straight to GCS -> cleanup). All [PREVIEW]-tagged: needs a gateway Vertex provider key (server-side) + a GCS bucket. Set vertexGcsBucket and run with --env-var include_preview=1. The OpenAI-drop-in rows send no Authorization header (routing is by provider=vertex).", "item": [ { - "name": "OpenAI Backlog", - "item": [ - { "name": "OpenAI: tool_choice specific function", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o-mini\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Pick a color\"}],\n \"tools\": [{\"type\":\"function\",\"function\":{\"name\":\"pick_color\",\"parameters\":{\"type\":\"object\",\"properties\":{\"hex\":{\"type\":\"string\"}}}}}],\n \"tool_choice\": {\"type\":\"function\",\"function\":{\"name\":\"pick_color\"}}\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/chat/completions","host":["{{baseUrl}}"],"path":["openai","v1","chat","completions"]}}}, - { "name": "OpenAI: parallel_tool_calls=false", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o-mini\",\n \"messages\": [{\"role\":\"user\",\"content\":\"What's the weather in NYC and SF?\"}],\n \"tools\": [{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}}}}}],\n \"parallel_tool_calls\": false\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/chat/completions","host":["{{baseUrl}}"],"path":["openai","v1","chat","completions"]}}}, - { "name": "OpenAI: response_format json_object", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o-mini\",\n \"messages\": [{\"role\":\"system\",\"content\":\"Output JSON only.\"},{\"role\":\"user\",\"content\":\"Tokyo population\"}],\n \"response_format\": {\"type\":\"json_object\"}\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/chat/completions","host":["{{baseUrl}}"],"path":["openai","v1","chat","completions"]}}}, - { "name": "OpenAI: logprobs + top_logprobs", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o-mini\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}],\n \"logprobs\": true,\n \"top_logprobs\": 5\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/chat/completions","host":["{{baseUrl}}"],"path":["openai","v1","chat","completions"]}}}, - { "name": "OpenAI: seed for deterministic output", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o-mini\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Pick a number\"}],\n \"seed\": 12345\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/chat/completions","host":["{{baseUrl}}"],"path":["openai","v1","chat","completions"]}}}, - { "name": "OpenAI: stop sequences", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o-mini\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Count: one, two, three, four, five\"}],\n \"stop\": [\"three\"]\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/chat/completions","host":["{{baseUrl}}"],"path":["openai","v1","chat","completions"]}}}, - { "name": "OpenAI: stream_options include_usage", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o-mini\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}],\n \"stream\": true,\n \"stream_options\": {\"include_usage\": true}\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/chat/completions","host":["{{baseUrl}}"],"path":["openai","v1","chat","completions"]}}}, - { "name": "OpenAI: predicted outputs", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o-mini\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Echo: hello world\"}],\n \"prediction\": {\"type\":\"content\",\"content\":\"hello world\"}\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/chat/completions","host":["{{baseUrl}}"],"path":["openai","v1","chat","completions"]}}}, - { "name": "OpenAI: service_tier auto", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o-mini\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}],\n \"service_tier\": \"auto\"\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/chat/completions","host":["{{baseUrl}}"],"path":["openai","v1","chat","completions"]}}}, - { "name": "OpenAI: store + metadata", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o-mini\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}],\n \"store\": true,\n \"metadata\": {\"harness\": \"backlog\"}\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/chat/completions","host":["{{baseUrl}}"],"path":["openai","v1","chat","completions"]}}}, - { "name": "OpenAI Responses: reasoning summary", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"o3-mini\",\n \"input\": \"What's 17*23?\",\n \"reasoning\": {\"summary\": \"auto\"}\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/responses","host":["{{baseUrl}}"],"path":["openai","v1","responses"]}}}, - { "name": "OpenAI Responses streaming: summary_index + obfuscation preserved", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code >= 400) { return; }","pm.test('summary_index and obfuscation survive stream', function () {"," var body = pm.response.text() || '';"," pm.expect(body, 'expected summary_index in SSE body').to.include('\"summary_index\"');"," pm.expect(body, 'expected obfuscation in SSE body').to.include('\"obfuscation\"');","});"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"o3-mini\",\n \"input\": \"What's 17*23?\",\n \"reasoning\": {\"summary\": \"auto\"},\n \"stream\": true,\n \"stream_options\": {\"include_obfuscation\": true}\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/responses","host":["{{baseUrl}}"],"path":["openai","v1","responses"]}}}, - { "name": "OpenAI Responses streaming: assistant phase preserved (gpt-5.3-codex)", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code >= 400) { return; }","pm.test('phase appears on assistant message items', function () {"," var body = pm.response.text() || '';"," var hasPhase = body.indexOf('\"phase\":\"final_answer\"') !== -1 || body.indexOf('\"phase\":\"commentary\"') !== -1;"," pm.expect(hasPhase, 'no phase field in SSE body. First 200 chars: ' + body.slice(0,200)).to.be.true;","});"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-5.3-codex\",\n \"input\": \"Solve 2+2 and explain your steps briefly.\",\n \"stream\": true\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/responses","host":["{{baseUrl}}"],"path":["openai","v1","responses"]}}}, - { "name": "OpenAI Responses: assistant phase input round-trip (gpt-5.3-codex)", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code >= 400) {"," pm.test('phase field accepted on input message', function () {"," var body = (pm.response.text() || '').toLowerCase();"," pm.expect(body, 'unexpected rejection of phase field: ' + body.slice(0,200)).to.not.include('unknown field \"phase\"');"," pm.expect(body).to.not.include('unexpected field \"phase\"');"," });"," return;","}","pm.test('response returned output items', function () {"," var body = pm.response.text() || '';"," pm.expect(body).to.include('\"output\"');","});"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-5.3-codex\",\n \"input\": [\n {\"role\":\"user\",\"content\":\"What's 2+2?\"},\n {\"role\":\"assistant\",\"phase\":\"final_answer\",\"content\":\"4\"},\n {\"role\":\"user\",\"content\":\"Now what's 3+3?\"}\n ]\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/responses","host":["{{baseUrl}}"],"path":["openai","v1","responses"]}}}, - { "name": "OpenAI Responses: background mode", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o\",\n \"input\": \"Hi\",\n \"background\": true\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/responses","host":["{{baseUrl}}"],"path":["openai","v1","responses"]}}}, - { "name": "OpenAI Responses: truncation auto", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o-mini\",\n \"input\": \"Hi\",\n \"truncation\": \"auto\"\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/responses","host":["{{baseUrl}}"],"path":["openai","v1","responses"]}}}, - { "name": "OpenAI Responses: include array", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o-mini\",\n \"input\": \"Hi\",\n \"include\": [\"message.input_image.image_url\"]\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/responses","host":["{{baseUrl}}"],"path":["openai","v1","responses"]}}}, - { "name": "OpenAI Responses: custom tool", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o\",\n \"input\": \"Send Slack message\",\n \"tools\": [{\"type\":\"function\",\"name\":\"send_slack\",\"parameters\":{\"type\":\"object\",\"properties\":{\"channel\":{\"type\":\"string\"},\"message\":{\"type\":\"string\"}}}}]\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/responses","host":["{{baseUrl}}"],"path":["openai","v1","responses"]}}}, - { "name": "OpenAI Responses: token counting", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o-mini\",\n \"input\": \"Count tokens for me\"\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/responses/input_tokens","host":["{{baseUrl}}"],"path":["openai","v1","responses","input_tokens"]}}} + "name": "[PREVIEW] Vertex: upload file (GCS, /openai)", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "file", + "src": "tests/e2e/api/fixtures/sample.jsonl", + "type": "file" + }, + { + "key": "purpose", + "value": "batch", + "type": "text" + }, + { + "key": "provider", + "value": "vertex", + "type": "text" + }, + { + "key": "storage_config[gcs][bucket]", + "value": "{{vertexGcsBucket}}", + "type": "text" + }, + { + "key": "storage_config[gcs][prefix]", + "value": "{{vertexGcsPrefix}}", + "type": "text" + } + ] + }, + "url": { + "raw": "{{baseUrl}}/openai/v1/files", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "files" + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var j = pm.response.json();", + "pm.test('Vertex upload: gcs backend', function () { pm.expect(j.storage_backend).to.eql('gcs'); });", + "pm.test('Vertex upload: has id', function () { pm.expect(j.id).to.be.a('string').and.not.empty; });", + "pm.collectionVariables.set('vertexFileId', j.id);" + ] + } + } ] }, { - "name": "Anthropic Backlog", - "item": [ - { "name": "Anthropic: prompt caching 1h TTL", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"extended-cache-ttl-2025-04-11"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-haiku-4-5\",\n \"max_tokens\": 800,\n \"system\": [{\"type\":\"text\",\"text\":\"long context\",\"cache_control\":{\"type\":\"ephemeral\",\"ttl\":\"1h\"}}],\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, - { "name": "Anthropic: web_fetch tool", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-7\",\n \"max_tokens\": 1024,\n \"messages\": [{\"role\":\"user\",\"content\":\"Fetch https://example.com and summarize\"}],\n \"tools\": [{\"type\":\"web_fetch_20250910\",\"name\":\"web_fetch\",\"max_uses\":2}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, - { "name": "Anthropic: memory tool", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-7\",\n \"max_tokens\": 1024,\n \"messages\": [{\"role\":\"user\",\"content\":\"Remember my name is Akshay\"}],\n \"tools\": [{\"type\":\"memory_20250818\",\"name\":\"memory\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, - { "name": "Anthropic: tool_search BM25", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-7\",\n \"max_tokens\": 1024,\n \"messages\": [{\"role\":\"user\",\"content\":\"List tools matching 'weather'\"}],\n \"tools\": [{\"type\":\"tool_search_tool_bm25\",\"name\":\"tool_search\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, - { "name": "Anthropic: tool_search regex", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-7\",\n \"max_tokens\": 1024,\n \"messages\": [{\"role\":\"user\",\"content\":\"Find tools matching get_.*\"}],\n \"tools\": [{\"type\":\"tool_search_tool_regex\",\"name\":\"tool_search\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, - { "name": "Anthropic: code_execution v2 (20250825)", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-7\",\n \"max_tokens\": 1024,\n \"messages\": [{\"role\":\"user\",\"content\":\"Compute 50! using Python\"}],\n \"tools\": [{\"type\":\"code_execution_20250825\",\"name\":\"code_execution\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, - { "name": "Anthropic: code_execution programmatic (20260120)", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-7\",\n \"max_tokens\": 1024,\n \"messages\": [{\"role\":\"user\",\"content\":\"Plot sin(x) and tell me the period\"}],\n \"tools\": [{\"type\":\"code_execution_20260120\",\"name\":\"code_execution\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, - { "name": "Anthropic: PDF input", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-haiku-4-5\",\n \"max_tokens\": 512,\n \"messages\": [{\"role\":\"user\",\"content\":[{\"type\":\"document\",\"source\":{\"type\":\"url\",\"url\":\"https://www.berkshirehathaway.com/letters/2024ltr.pdf\"}},{\"type\":\"text\",\"text\":\"Summarize\"}]}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, - { "name": "Anthropic: stop sequences", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-haiku-4-5\",\n \"max_tokens\": 800,\n \"messages\": [{\"role\":\"user\",\"content\":\"Count: one, two, three, four in lowercase\"}],\n \"stop_sequences\": [\"three\"]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, - { "name": "Anthropic: service_tier auto", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-haiku-4-5\",\n \"max_tokens\": 800,\n \"service_tier\": \"auto\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, - { "name": "Anthropic: output_config effort high", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-6\",\n \"max_tokens\": 1024,\n \"output_config\": {\"effort\": \"high\"},\n \"messages\": [{\"role\":\"user\",\"content\":\"Solve x^2 - 5x + 6 = 0\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, - { "name": "Anthropic: output_config format json_schema", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-haiku-4-5\",\n \"max_tokens\": 800,\n \"output_config\": {\"format\": {\"type\":\"json_schema\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}}}}},\n \"messages\": [{\"role\":\"user\",\"content\":\"Pick a city\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, - { "name": "Anthropic: tool defer_loading + advanced beta", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"advanced-tool-use-2025-09-15"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-7\",\n \"max_tokens\": 1024,\n \"tools\": [{\"name\":\"slow_tool\",\"input_schema\":{\"type\":\"object\"},\"defer_loading\":true},{\"name\":\"fast_tool\",\"input_schema\":{\"type\":\"object\"},\"defer_loading\":false}],\n \"messages\": [{\"role\":\"user\",\"content\":\"Call slow_tool\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, - { "name": "Anthropic: tool input_examples + beta", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"tool-examples-2025-10-29"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-haiku-4-5\",\n \"max_tokens\": 1024,\n \"tools\": [{\"name\":\"f\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"x\":{\"type\":\"number\"}}},\"input_examples\":[{\"input\":{\"x\":1},\"description\":\"basic\"}]}],\n \"messages\": [{\"role\":\"user\",\"content\":\"Call f\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, - { "name": "Anthropic: strict tool input", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-haiku-4-5\",\n \"max_tokens\": 512,\n \"tools\": [{\"name\":\"strict_fn\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"a\":{\"type\":\"string\"}},\"required\":[\"a\"],\"additionalProperties\":false},\"strict\":true}],\n \"messages\": [{\"role\":\"user\",\"content\":\"Call strict_fn\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, - { "name": "Anthropic: token counting", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-haiku-4-5\",\n \"messages\": [{\"role\":\"user\",\"content\":\"How many tokens?\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages/count_tokens","host":["{{baseUrl}}"],"path":["anthropic","v1","messages","count_tokens"]}}}, - { "name": "Anthropic: list models", "request": { "method": "GET", "header": [{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "url": {"raw":"{{baseUrl}}/anthropic/v1/models","host":["{{baseUrl}}"],"path":["anthropic","v1","models"]}}}, - { "name": "Anthropic: list batches", "request": { "method": "GET", "header": [{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "url": {"raw":"{{baseUrl}}/anthropic/v1/messages/batches","host":["{{baseUrl}}"],"path":["anthropic","v1","messages","batches"]}}}, - { "name": "Anthropic: list files", "request": { "method": "GET", "header": [{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"files-api-2025-04-14"}], "url": {"raw":"{{baseUrl}}/anthropic/v1/files","host":["{{baseUrl}}"],"path":["anthropic","v1","files"]}}} + "name": "[PREVIEW] Vertex: list files (GCS, /openai)", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/openai/v1/files?provider=vertex&storage_config[gcs][bucket]={{vertexGcsBucket}}&storage_config[gcs][prefix]={{vertexGcsPrefix}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "files" + ], + "query": [ + { + "key": "provider", + "value": "vertex" + }, + { + "key": "storage_config[gcs][bucket]", + "value": "{{vertexGcsBucket}}" + }, + { + "key": "storage_config[gcs][prefix]", + "value": "{{vertexGcsPrefix}}" + } + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var j = pm.response.json();", + "pm.test('Vertex list: uploaded file present', function () { pm.expect((j.data || []).map(function (f) { return f.id; })).to.include(pm.collectionVariables.get('vertexFileId')); });" + ] + } + } ] }, { - "name": "Anthropic Beta Headers", - "item": [ - { "name": "Beta: token-efficient-tools", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"token-efficient-tools-2025-02-19"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-haiku-4-5\",\n \"max_tokens\": 800,\n \"tools\": [{\"name\":\"f\",\"input_schema\":{\"type\":\"object\"}}],\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, - { "name": "Beta: fine-grained-tool-streaming", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"fine-grained-tool-streaming-2025-05-14"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-haiku-4-5\",\n \"max_tokens\": 800,\n \"stream\": true,\n \"tools\": [{\"name\":\"f\",\"input_schema\":{\"type\":\"object\"}}],\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, - { "name": "[PREVIEW] Beta: fast-mode (Opus 4.6)", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"fast-mode-2026-02-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-6\",\n \"max_tokens\": 800,\n \"speed\": \"fast\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, - { "name": "[PREVIEW] Beta: fast-mode (Opus 4.7)", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"fast-mode-2026-02-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-7\",\n \"max_tokens\": 800,\n \"speed\": \"fast\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, - { "name": "[PREVIEW] Beta: fast-mode (Opus 4.8)", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"fast-mode-2026-02-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-8\",\n \"max_tokens\": 800,\n \"speed\": \"fast\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, - { "name": "Beta: context-1m", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"context-1m-2025-09-15"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-6\",\n \"max_tokens\": 800,\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, - { "name": "Beta: interleaved-thinking", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"interleaved-thinking-2025-05-14"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-sonnet-4-6\",\n \"max_tokens\": 4096,\n \"thinking\": {\"type\":\"enabled\",\"budget_tokens\":2000},\n \"messages\": [{\"role\":\"user\",\"content\":\"Solve in steps\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, - { "name": "Beta: skills bundle", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"skills-2025-10-29"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-7\",\n \"max_tokens\": 1024,\n \"messages\": [{\"role\":\"user\",\"content\":\"Use skills\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, - { "name": "Beta: redact-thinking", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"redact-thinking-2025-09-15"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-sonnet-4-6\",\n \"max_tokens\": 1024,\n \"thinking\": {\"type\":\"enabled\",\"budget_tokens\":2000},\n \"messages\": [{\"role\":\"user\",\"content\":\"Reasoned answer\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, - { "name": "Beta: compaction", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"compact-2025-09-15"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-6\",\n \"max_tokens\": 800,\n \"messages\": [{\"role\":\"user\",\"content\":\"Long convo\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, - { "name": "Mid-conv system message (Opus 4.8) — system ends array", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-8\",\n \"max_tokens\": 512,\n \"system\": [{\"type\":\"text\",\"text\":\"You are a helpful assistant.\"}],\n \"messages\": [\n {\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]},\n {\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Hi!\"}]},\n {\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"How are you?\"}]},\n {\"role\":\"system\",\"content\":[{\"type\":\"text\",\"text\":\"Respond only in one word.\"}]}\n ]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, - { "name": "Mid-conv system message (Opus 4.8) — system mid-history (before assistant)", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-8\",\n \"max_tokens\": 512,\n \"system\": [{\"type\":\"text\",\"text\":\"You are a helpful assistant.\"}],\n \"messages\": [\n {\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]},\n {\"role\":\"system\",\"content\":[{\"type\":\"text\",\"text\":\"From now on be very concise.\"}]},\n {\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Hi!\"}]},\n {\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"How are you?\"}]}\n ]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}} + "name": "[PREVIEW] Vertex: retrieve file (GCS, /openai)", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/openai/v1/files/{{vertexFileId}}?provider=vertex", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "files", + "{{vertexFileId}}" + ], + "query": [ + { + "key": "provider", + "value": "vertex" + } + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var j = pm.response.json();", + "pm.test('Vertex retrieve: id matches uploaded', function () { pm.expect(j.id).to.eql(pm.collectionVariables.get('vertexFileId')); });" + ] + } + } ] }, { - "name": "Bedrock Backlog", - "item": [ - { "name": "Bedrock Converse: streaming", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"messages\": [{\"role\":\"user\",\"content\":[{\"text\":\"Count to 5\"}]}],\n \"inferenceConfig\": {\"maxTokens\": 256}\n}"}, "url": {"raw":"{{baseUrl}}/bedrock/model/{{bedrockModel}}/converse-stream","host":["{{baseUrl}}"],"path":["bedrock","model","{{bedrockModel}}","converse-stream"]}}}, - { "name": "Bedrock Converse: stop sequences", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"messages\": [{\"role\":\"user\",\"content\":[{\"text\":\"Count: one, two, three, four in lowercase\"}]}],\n \"inferenceConfig\": {\"maxTokens\": 256, \"stopSequences\": [\"three\"]}\n}"}, "url": {"raw":"{{baseUrl}}/bedrock/model/{{bedrockModel}}/converse","host":["{{baseUrl}}"],"path":["bedrock","model","{{bedrockModel}}","converse"]}}}, - { "name": "Bedrock Converse: tool choice forced", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"messages\": [{\"role\":\"user\",\"content\":[{\"text\":\"Pick a number\"}]}],\n \"toolConfig\": {\"tools\":[{\"toolSpec\":{\"name\":\"pick\",\"inputSchema\":{\"json\":{\"type\":\"object\",\"properties\":{\"n\":{\"type\":\"number\"}}}}}}],\"toolChoice\":{\"tool\":{\"name\":\"pick\"}}},\n \"inferenceConfig\": {\"maxTokens\":256}\n}"}, "url": {"raw":"{{baseUrl}}/bedrock/model/{{bedrockModel}}/converse","host":["{{baseUrl}}"],"path":["bedrock","model","{{bedrockModel}}","converse"]}}}, - { "name": "Bedrock Converse: performance config optimized", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"messages\": [{\"role\":\"user\",\"content\":[{\"text\":\"Hi\"}]}],\n \"performanceConfig\": {\"latency\":\"optimized\"},\n \"inferenceConfig\": {\"maxTokens\":256}\n}"}, "url": {"raw":"{{baseUrl}}/bedrock/model/us.amazon.nova-pro-v1:0/converse","host":["{{baseUrl}}"],"path":["bedrock","model","us.amazon.nova-pro-v1:0","converse"]}}}, - { "name": "Bedrock Converse: request metadata", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"messages\": [{\"role\":\"user\",\"content\":[{\"text\":\"Hi\"}]}],\n \"requestMetadata\": {\"session\":\"harness\"},\n \"inferenceConfig\": {\"maxTokens\":256}\n}"}, "url": {"raw":"{{baseUrl}}/bedrock/model/{{bedrockModel}}/converse","host":["{{baseUrl}}"],"path":["bedrock","model","{{bedrockModel}}","converse"]}}}, - { "name": "Bedrock InvokeModel: direct Anthropic shape", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"anthropic_version\": \"bedrock-2023-05-31\",\n \"max_tokens\": 800,\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}]\n}"}, "url": {"raw":"{{baseUrl}}/bedrock/model/{{bedrockModel}}/invoke","host":["{{baseUrl}}"],"path":["bedrock","model","{{bedrockModel}}","invoke"]}}} + "name": "[PREVIEW] Vertex: download file content (GCS, /openai)", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/openai/v1/files/{{vertexFileId}}/content?provider=vertex", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "files", + "{{vertexFileId}}", + "content" + ], + "query": [ + { + "key": "provider", + "value": "vertex" + } + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('Vertex content: non-empty body', function () { pm.expect((pm.response.text() || '').length).to.be.above(0); });" + ] + } + } ] }, { - "name": "Gemini Backlog", - "item": [ - { "name": "Gemini: tool config functionCallingConfig ANY", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-goog-api-key","value":"{{genaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"contents\": [{\"parts\":[{\"text\":\"Call get_weather\"}]}],\n \"tools\": [{\"functionDeclarations\":[{\"name\":\"get_weather\",\"parameters\":{\"type\":\"OBJECT\",\"properties\":{\"city\":{\"type\":\"STRING\"}}}}]}],\n \"toolConfig\": {\"functionCallingConfig\":{\"mode\":\"ANY\",\"allowedFunctionNames\":[\"get_weather\"]}}\n}"}, "url": {"raw":"{{baseUrl}}/genai/v1beta/models/{{genaiModel}}:generateContent","host":["{{baseUrl}}"],"path":["genai","v1beta","models","{{genaiModel}}:generateContent"]}}}, - { "name": "Gemini: stop sequences", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-goog-api-key","value":"{{genaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"contents\": [{\"parts\":[{\"text\":\"Count: one, two, three, four in lowercase\"}]}],\n \"generationConfig\": {\"stopSequences\":[\"three\"]}\n}"}, "url": {"raw":"{{baseUrl}}/genai/v1beta/models/{{genaiModel}}:generateContent","host":["{{baseUrl}}"],"path":["genai","v1beta","models","{{genaiModel}}:generateContent"]}}}, - { "name": "Gemini: temperature + topP + topK", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-goog-api-key","value":"{{genaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"contents\": [{\"parts\":[{\"text\":\"Pick a number\"}]}],\n \"generationConfig\": {\"temperature\":0.7,\"topP\":0.9,\"topK\":40}\n}"}, "url": {"raw":"{{baseUrl}}/genai/v1beta/models/{{genaiModel}}:generateContent","host":["{{baseUrl}}"],"path":["genai","v1beta","models","{{genaiModel}}:generateContent"]}}}, - { "name": "Gemini: response logprobs", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-goog-api-key","value":"{{genaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"contents\": [{\"parts\":[{\"text\":\"Hi\"}]}],\n \"generationConfig\": {\"responseLogprobs\":true,\"logprobs\":3}\n}"}, "url": {"raw":"{{baseUrl}}/genai/v1beta/models/{{genaiModel}}:generateContent","host":["{{baseUrl}}"],"path":["genai","v1beta","models","{{genaiModel}}:generateContent"]}}}, - { "name": "Gemini: presence + frequency penalty", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-goog-api-key","value":"{{genaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"contents\": [{\"parts\":[{\"text\":\"Tell a story\"}]}],\n \"generationConfig\": {\"presencePenalty\":0.5,\"frequencyPenalty\":0.5}\n}"}, "url": {"raw":"{{baseUrl}}/genai/v1beta/models/{{genaiModel}}:generateContent","host":["{{baseUrl}}"],"path":["genai","v1beta","models","{{genaiModel}}:generateContent"]}}}, - { "name": "[PREVIEW] Gemini: PDF input", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-goog-api-key","value":"{{genaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"contents\": [{\"parts\":[{\"text\":\"Summarize this PDF\"},{\"fileData\":{\"mimeType\":\"application/pdf\",\"fileUri\":\"https://storage.googleapis.com/generativeai-downloads/data/A17_FlightPlan.pdf\"}}]}]\n}"}, "url": {"raw":"{{baseUrl}}/genai/v1beta/models/{{genaiModel}}:generateContent","host":["{{baseUrl}}"],"path":["genai","v1beta","models","{{genaiModel}}:generateContent"]}}}, - { "name": "Gemini: YouTube URL input", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-goog-api-key","value":"{{genaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"contents\": [{\"parts\":[{\"text\":\"Summarize this video\"},{\"fileData\":{\"fileUri\":\"https://www.youtube.com/watch?v=jNQXAC9IVRw\"}}]}]\n}"}, "url": {"raw":"{{baseUrl}}/genai/v1beta/models/{{genaiModel}}:generateContent","host":["{{baseUrl}}"],"path":["genai","v1beta","models","{{genaiModel}}:generateContent"]}}}, - { "name": "Gemini: URL context tool", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-goog-api-key","value":"{{genaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"contents\": [{\"parts\":[{\"text\":\"Summarize https://anthropic.com/news\"}]}],\n \"tools\": [{\"urlContext\":{}}]\n}"}, "url": {"raw":"{{baseUrl}}/genai/v1beta/models/{{genaiModel}}:generateContent","host":["{{baseUrl}}"],"path":["genai","v1beta","models","{{genaiModel}}:generateContent"]}}}, - { "name": "Gemini: count tokens", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-goog-api-key","value":"{{genaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"contents\": [{\"parts\":[{\"text\":\"How many tokens?\"}]}]\n}"}, "url": {"raw":"{{baseUrl}}/genai/v1beta/models/{{genaiModel}}:countTokens","host":["{{baseUrl}}"],"path":["genai","v1beta","models","{{genaiModel}}:countTokens"]}}}, - { "name": "Gemini: list models", "request": { "method": "GET", "header": [{"key":"x-goog-api-key","value":"{{genaiKey}}"}], "url": {"raw":"{{baseUrl}}/genai/v1beta/models","host":["{{baseUrl}}"],"path":["genai","v1beta","models"]}}} + "name": "[PREVIEW] Vertex: delete file (GCS, /openai)", + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{baseUrl}}/openai/v1/files/{{vertexFileId}}?provider=vertex", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "files", + "{{vertexFileId}}" + ], + "query": [ + { + "key": "provider", + "value": "vertex" + } + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var j = pm.response.json();", + "pm.test('Vertex delete: deleted true', function () { pm.expect(j.deleted).to.eql(true); });" + ] + } + } ] }, { - "name": "Vertex Backlog", - "item": [ - { "name": "Vertex: anthropic_version in body", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"vertex/claude-opus-4-7\",\n \"anthropic_version\": \"vertex-2023-10-16\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi via Vertex\"}]\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, - { "name": "Vertex: streaming Anthropic", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"vertex/claude-opus-4-7\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Count 1-5\"}],\n \"stream\": true\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, - { "name": "[PREVIEW] Vertex Model Garden: Llama (publishers/meta/...)", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"vertex/meta/llama-4-maverick-17b-128e-instruct-maas\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}]\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, - { "name": "[PREVIEW] Vertex Model Garden: Mistral", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"vertex/publishers/mistralai/models/mistral-large-2411\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}]\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}} + "name": "[PREVIEW] Vertex: mint resumable session (native)", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "purpose", + "value": "user_data", + "type": "text" + }, + { + "key": "filename", + "value": "harness-video.bin", + "type": "text" + }, + { + "key": "content_type", + "value": "application/octet-stream", + "type": "text" + }, + { + "key": "gcs_bucket", + "value": "{{vertexGcsBucket}}", + "type": "text" + }, + { + "key": "gcs_prefix", + "value": "{{vertexGcsPrefix}}", + "type": "text" + } + ] + }, + "url": { + "raw": "{{baseUrl}}/v1/files?provider=vertex", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "files" + ], + "query": [ + { + "key": "provider", + "value": "vertex" + } + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var j = pm.response.json();", + "pm.test('Vertex resumable: status pending_upload', function () { pm.expect(j.status).to.eql('pending_upload'); });", + "pm.test('Vertex resumable: has upload_url', function () { pm.expect(j.upload_url).to.be.a('string').and.not.empty; });", + "pm.collectionVariables.set('vertexUploadUrl', j.upload_url);", + "pm.collectionVariables.set('vertexResumableId', encodeURIComponent(j.id));" + ] + } + } ] }, { - "name": "Azure Backlog", - "item": [ - { "name": "Azure: tools (function calling)", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"api-key","value":"{{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"messages\": [{\"role\":\"user\",\"content\":\"What's the weather?\"}],\n \"tools\": [{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}}}}}]\n}"}, "url": {"raw":"{{baseUrl}}/openai/openai/deployments/{{azureDeployment}}/chat/completions?api-version={{azureApiVersion}}","host":["{{baseUrl}}"],"path":["openai","openai","deployments","{{azureDeployment}}","chat","completions"],"query":[{"key":"api-version","value":"{{azureApiVersion}}"}]}}}, - { "name": "Azure: streaming", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"api-key","value":"{{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"messages\": [{\"role\":\"user\",\"content\":\"Count 1-5\"}],\n \"stream\": true\n}"}, "url": {"raw":"{{baseUrl}}/openai/openai/deployments/{{azureDeployment}}/chat/completions?api-version={{azureApiVersion}}","host":["{{baseUrl}}"],"path":["openai","openai","deployments","{{azureDeployment}}","chat","completions"],"query":[{"key":"api-version","value":"{{azureApiVersion}}"}]}}}, - { "name": "Azure: structured output json_schema", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"api-key","value":"{{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"messages\": [{\"role\":\"user\",\"content\":\"Extract city/country for Paris\"}],\n \"response_format\": {\"type\":\"json_schema\",\"json_schema\":{\"name\":\"city\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"},\"country\":{\"type\":\"string\"}},\"required\":[\"city\",\"country\"],\"additionalProperties\":false}}}\n}"}, "url": {"raw":"{{baseUrl}}/openai/openai/deployments/{{azureDeployment}}/chat/completions?api-version={{azureApiVersion}}","host":["{{baseUrl}}"],"path":["openai","openai","deployments","{{azureDeployment}}","chat","completions"],"query":[{"key":"api-version","value":"{{azureApiVersion}}"}]}}}, - { "name": "Azure: vision (image_url)", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"api-key","value":"{{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"messages\": [{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://storage.googleapis.com/generativeai-downloads/images/scones.jpg\"}}]}]\n}"}, "url": {"raw":"{{baseUrl}}/openai/openai/deployments/{{azureDeployment}}/chat/completions?api-version={{azureApiVersion}}","host":["{{baseUrl}}"],"path":["openai","openai","deployments","{{azureDeployment}}","chat","completions"],"query":[{"key":"api-version","value":"{{azureApiVersion}}"}]}}}, - { "name": "Azure: system + multi-turn", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"api-key","value":"{{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"messages\": [{\"role\":\"system\",\"content\":\"You are a pirate.\"},{\"role\":\"user\",\"content\":\"Hi\"},{\"role\":\"assistant\",\"content\":\"Arrr!\"},{\"role\":\"user\",\"content\":\"Tell a joke\"}]\n}"}, "url": {"raw":"{{baseUrl}}/openai/openai/deployments/{{azureDeployment}}/chat/completions?api-version={{azureApiVersion}}","host":["{{baseUrl}}"],"path":["openai","openai","deployments","{{azureDeployment}}","chat","completions"],"query":[{"key":"api-version","value":"{{azureApiVersion}}"}]}}}, - { "name": "Azure On Your Data: azure_search", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"api-key","value":"{{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"messages\": [{\"role\":\"user\",\"content\":\"What's in the docs?\"}],\n \"data_sources\": [{\"type\":\"azure_search\",\"parameters\":{\"endpoint\":\"https://placeholder.search.windows.net\",\"index_name\":\"placeholder\",\"authentication\":{\"type\":\"api_key\",\"key\":\"placeholder\"}}}]\n}"}, "url": {"raw":"{{baseUrl}}/openai/openai/deployments/{{azureDeployment}}/chat/completions?api-version={{azureApiVersion}}","host":["{{baseUrl}}"],"path":["openai","openai","deployments","{{azureDeployment}}","chat","completions"],"query":[{"key":"api-version","value":"{{azureApiVersion}}"}]}}} + "name": "[PREVIEW] Vertex: PUT bytes to GCS session (resumable, direct to GCS)", + "request": { + "method": "PUT", + "header": [], + "body": { + "mode": "file", + "file": { + "src": "tests/e2e/api/fixtures/sample.txt" + } + }, + "url": { + "raw": "{{vertexUploadUrl}}", + "host": [ + "{{vertexUploadUrl}}" + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('Vertex resumable: GCS accepted bytes (2xx)', function () { pm.expect(pm.response.code).to.be.within(200, 299); });" + ] + } + } ] }, { - "name": "Cross-Provider Backlog", - "item": [ - { "name": "Cross-cut: code execution Anthropic", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"anthropic/claude-opus-4-7\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Compute 50!\"}],\n \"tools\": [{\"type\":\"code_execution_20250522\",\"name\":\"code_execution\"}]\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, - { "name": "Cross-cut: code execution Gemini", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gemini/gemini-2.5-flash\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Compute 50!\"}],\n \"tools\": [{\"type\":\"code_execution\"}]\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, - { "name": "Cross-cut: extended thinking via cross-model", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"anthropic/claude-sonnet-4-6\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Plan a trip\"}],\n \"thinking\": {\"type\":\"enabled\",\"budget_tokens\":2000},\n \"max_tokens\": 4096\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, - { "name": "Cross-cut: prompt caching via cross-model", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"anthropic/claude-haiku-4-5\",\n \"messages\": [{\"role\":\"system\",\"content\":[{\"type\":\"text\",\"text\":\"Long ctx\",\"cache_control\":{\"type\":\"ephemeral\"}}]},{\"role\":\"user\",\"content\":\"Hi\"}]\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, - { "name": "Cross-cut: stop sequences (OpenAI)", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code < 400) { pm.test('Stop sequence: halted before stop token', function () { var j = pm.response.json(); var c = (j.choices && j.choices[0] && j.choices[0].message && j.choices[0].message.content) || ''; var fr = (j.choices && j.choices[0] && j.choices[0].finish_reason) || ''; pm.expect(c.toLowerCase(), 'stop token \"three\" leaked into content').to.not.include('three'); pm.expect(['stop','stop_sequence','length','content_filter'], 'unexpected finish_reason: ' + fr).to.include(fr); }); }"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"openai/gpt-4o-mini\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Count: one, two, three, four in lowercase\"}],\n \"stop\": [\"three\"]\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, - { "name": "Cross-cut: stop sequences (Anthropic)", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code < 400) { pm.test('Stop sequence: halted before stop token', function () { var j = pm.response.json(); var c = (j.choices && j.choices[0] && j.choices[0].message && j.choices[0].message.content) || ''; var fr = (j.choices && j.choices[0] && j.choices[0].finish_reason) || ''; pm.expect(c.toLowerCase(), 'stop token \"three\" leaked into content').to.not.include('three'); pm.expect(['stop','stop_sequence','length','content_filter'], 'unexpected finish_reason: ' + fr).to.include(fr); }); }"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"anthropic/claude-haiku-4-5\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Count: one, two, three, four in lowercase\"}],\n \"stop\": [\"three\"]\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, - { "name": "Cross-cut: stop sequences (Gemini)", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code < 400) { pm.test('Stop sequence: halted before stop token', function () { var j = pm.response.json(); var c = (j.choices && j.choices[0] && j.choices[0].message && j.choices[0].message.content) || ''; var fr = (j.choices && j.choices[0] && j.choices[0].finish_reason) || ''; pm.expect(c.toLowerCase(), 'stop token \"three\" leaked into content').to.not.include('three'); pm.expect(['stop','stop_sequence','length','content_filter'], 'unexpected finish_reason: ' + fr).to.include(fr); }); }"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gemini/gemini-2.5-flash\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Count: one, two, three, four in lowercase\"}],\n \"stop\": [\"three\"]\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, - { "name": "Cross-cut: tool_choice forced (OpenAI)", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"openai/gpt-4o-mini\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}],\n \"tools\": [{\"type\":\"function\",\"function\":{\"name\":\"f\",\"parameters\":{\"type\":\"object\"}}}],\n \"tool_choice\": \"required\"\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, - { "name": "Cross-cut: tool_choice forced (Bedrock)", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"bedrock/global.anthropic.claude-sonnet-4-6\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}],\n \"tools\": [{\"type\":\"function\",\"function\":{\"name\":\"f\",\"parameters\":{\"type\":\"object\"}}}],\n \"tool_choice\": \"required\"\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, + "name": "[PREVIEW] Vertex: delete resumable file (native, cleanup)", + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{baseUrl}}/v1/files/{{vertexResumableId}}?provider=vertex", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "files", + "{{vertexResumableId}}" + ], + "query": [ + { + "key": "provider", + "value": "vertex" + } + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var j = pm.response.json();", + "pm.test('Vertex resumable cleanup: deleted', function () { pm.expect(j.deleted).to.eql(true); });" + ] + } + } + ] + } + ] + }, + { + "name": "11c. Vertex Batches (/openai + native passthrough)", + "description": "Vertex batch prediction is GCS-backed. CRUD (create/list/retrieve/cancel) runs via the OpenAI drop-in (/openai/v1/batches, routed by provider=vertex); the base64 batch ids round-trip through every endpoint. The last two rows exercise the native genai surface raw passthrough: a verbatim Vertex BatchPredictionJob body POSTed to .../batchPredictionJobs is forwarded as-is and the native job resource is returned. All [PREVIEW]-tagged: needs a gateway Vertex provider key + a GCS bucket, plus vertexProject/vertexLocation for the native rows. Set vertexGcsBucket, vertexProject and run with --env-var include_preview=1.", + "item": [ + { + "name": "[PREVIEW] Vertex: upload batch input (GCS, /openai)", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "file", + "src": "tests/e2e/api/fixtures/sample.jsonl", + "type": "file" + }, + { + "key": "purpose", + "value": "batch", + "type": "text" + }, + { + "key": "provider", + "value": "vertex", + "type": "text" + }, + { + "key": "storage_config[gcs][bucket]", + "value": "{{vertexGcsBucket}}", + "type": "text" + }, + { + "key": "storage_config[gcs][prefix]", + "value": "{{vertexGcsPrefix}}", + "type": "text" + } + ] + }, + "url": { + "raw": "{{baseUrl}}/openai/v1/files", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "files" + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var j = pm.response.json();", + "pm.test('Vertex batch input: gcs backend', function () { pm.expect(j.storage_backend).to.eql('gcs'); });", + "pm.test('Vertex batch input: has id', function () { pm.expect(j.id).to.be.a('string').and.not.empty; });", + "pm.collectionVariables.set('vertexBatchInputFileId', j.id);", + "pm.collectionVariables.set('vertexBatchInputGcsUri', j.storage_uri);" + ] + } + } + ] + }, + { + "name": "[PREVIEW] Vertex: create batch (/openai)", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"input_file_id\": \"{{vertexBatchInputFileId}}\",\n \"endpoint\": \"/v1/chat/completions\",\n \"completion_window\": \"24h\",\n \"provider\": \"vertex\",\n \"model\": \"gemini-2.5-flash\",\n \"output_folder\": {\n \"url\": \"gs://{{vertexGcsBucket}}/{{vertexGcsPrefix}}batch-output\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{baseUrl}}/openai/v1/batches", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "batches" + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var j = pm.response.json();", + "pm.test('Vertex create batch: object batch', function () { pm.expect(j.object).to.eql('batch'); });", + "pm.test('Vertex create batch: has id', function () { pm.expect(j.id).to.be.a('string').and.not.empty; });", + "pm.test('Vertex create batch: input_file_id round-trips', function () { pm.expect(j.input_file_id).to.eql(pm.collectionVariables.get('vertexBatchInputFileId')); });", + "pm.collectionVariables.set('vertexBatchId', j.id);" + ] + } + } + ] + }, + { + "name": "[PREVIEW] Vertex: list batches (/openai)", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/openai/v1/batches?provider=vertex", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "batches" + ], + "query": [ + { + "key": "provider", + "value": "vertex" + } + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var j = pm.response.json();", + "pm.test('Vertex list batches: created batch present', function () { pm.expect((j.data || []).map(function (b) { return b.id; })).to.include(pm.collectionVariables.get('vertexBatchId')); });" + ] + } + } + ] + }, + { + "name": "[PREVIEW] Vertex: retrieve batch (/openai)", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/openai/v1/batches/{{vertexBatchId}}?provider=vertex", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "batches", + "{{vertexBatchId}}" + ], + "query": [ + { + "key": "provider", + "value": "vertex" + } + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var j = pm.response.json();", + "pm.test('Vertex retrieve batch: object batch', function () { pm.expect(j.object).to.eql('batch'); });", + "pm.test('Vertex retrieve batch: id matches create', function () { pm.expect(j.id).to.eql(pm.collectionVariables.get('vertexBatchId')); });" + ] + } + } + ] + }, + { + "name": "[PREVIEW] Vertex: cancel batch (/openai)", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"provider\": \"vertex\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{baseUrl}}/openai/v1/batches/{{vertexBatchId}}/cancel", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "batches", + "{{vertexBatchId}}", + "cancel" + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var j = pm.response.json();", + "pm.test('Vertex cancel batch: has id', function () { pm.expect(j.id).to.be.a('string').and.not.empty; });", + "pm.test('Vertex cancel batch: cancelling/cancelled', function () { pm.expect(['cancelling','cancelled']).to.include(j.status); });" + ] + } + } + ] + }, + { + "name": "[PREVIEW] Vertex: create batch RAW passthrough (native genai)", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"displayName\": \"bifrost-harness-passthrough\",\n \"model\": \"publishers/google/models/gemini-2.5-flash\",\n \"inputConfig\": {\n \"instancesFormat\": \"jsonl\",\n \"gcsSource\": {\n \"uris\": [\n \"{{vertexBatchInputGcsUri}}\"\n ]\n }\n },\n \"outputConfig\": {\n \"predictionsFormat\": \"jsonl\",\n \"gcsDestination\": {\n \"outputUriPrefix\": \"gs://{{vertexGcsBucket}}/{{vertexGcsPrefix}}batch-output\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{baseUrl}}/genai/v1/projects/{{vertexProject}}/locations/{{vertexLocation}}/batchPredictionJobs", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "genai", + "v1", + "projects", + "{{vertexProject}}", + "locations", + "{{vertexLocation}}", + "batchPredictionJobs" + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "// Sends a verbatim Vertex BatchPredictionJob body; Bifrost passes it through (UseRawRequestBody)", + "// and returns the native Vertex job resource. Proves the raw-passthrough path end to end.", + "var j = pm.response.json();", + "pm.test('Vertex RAW passthrough: name is a batchPredictionJobs resource', function () { pm.expect(j.name).to.be.a('string'); pm.expect(j.name).to.include('batchPredictionJobs'); });", + "pm.test('Vertex RAW passthrough: has JOB_STATE_ state', function () { pm.expect(j.state || '').to.match(/^JOB_STATE_/); });", + "var parts = (j.name || '').split('/'); pm.collectionVariables.set('vertexRawBatchId', parts[parts.length - 1]);" + ] + } + } + ] + }, + { + "name": "[PREVIEW] Vertex: delete batch RAW passthrough (native genai, cleanup)", + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{baseUrl}}/genai/v1/projects/{{vertexProject}}/locations/{{vertexLocation}}/batchPredictionJobs/{{vertexRawBatchId}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "genai", + "v1", + "projects", + "{{vertexProject}}", + "locations", + "{{vertexLocation}}", + "batchPredictionJobs", + "{{vertexRawBatchId}}" + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('Vertex RAW passthrough cleanup: 2xx', function () { pm.expect(pm.response.code).to.be.within(200, 299); });" + ] + } + } + ] + }, + { + "name": "[PREVIEW] Vertex: delete batch input file (/openai, cleanup)", + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{baseUrl}}/openai/v1/files/{{vertexBatchInputFileId}}?provider=vertex", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "files", + "{{vertexBatchInputFileId}}" + ], + "query": [ + { + "key": "provider", + "value": "vertex" + } + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var j = pm.response.json();", + "pm.test('Vertex batch input cleanup: deleted', function () { pm.expect(j.deleted).to.eql(true); });" + ] + } + } + ] + } + ] + }, + { + "name": "12. Backlog Coverage (auto-added missing cases)", + "description": "Comprehensive coverage of features sourced from each provider's docs. Organized by provider. Many entries will fail in environments without the corresponding model/feature provisioned - that's expected; they exist to surface gaps via the failure report.", + "item": [ + { + "name": "OpenAI Backlog", + "item": [ + { "name": "OpenAI: tool_choice specific function", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o-mini\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Pick a color\"}],\n \"tools\": [{\"type\":\"function\",\"function\":{\"name\":\"pick_color\",\"parameters\":{\"type\":\"object\",\"properties\":{\"hex\":{\"type\":\"string\"}}}}}],\n \"tool_choice\": {\"type\":\"function\",\"function\":{\"name\":\"pick_color\"}}\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/chat/completions","host":["{{baseUrl}}"],"path":["openai","v1","chat","completions"]}}}, + { "name": "OpenAI: parallel_tool_calls=false", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o-mini\",\n \"messages\": [{\"role\":\"user\",\"content\":\"What's the weather in NYC and SF?\"}],\n \"tools\": [{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}}}}}],\n \"parallel_tool_calls\": false\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/chat/completions","host":["{{baseUrl}}"],"path":["openai","v1","chat","completions"]}}}, + { "name": "OpenAI: response_format json_object", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o-mini\",\n \"messages\": [{\"role\":\"system\",\"content\":\"Output JSON only.\"},{\"role\":\"user\",\"content\":\"Tokyo population\"}],\n \"response_format\": {\"type\":\"json_object\"}\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/chat/completions","host":["{{baseUrl}}"],"path":["openai","v1","chat","completions"]}}}, + { "name": "OpenAI: logprobs + top_logprobs", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o-mini\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}],\n \"logprobs\": true,\n \"top_logprobs\": 5\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/chat/completions","host":["{{baseUrl}}"],"path":["openai","v1","chat","completions"]}}}, + { "name": "OpenAI: seed for deterministic output", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o-mini\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Pick a number\"}],\n \"seed\": 12345\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/chat/completions","host":["{{baseUrl}}"],"path":["openai","v1","chat","completions"]}}}, + { "name": "OpenAI: stop sequences", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o-mini\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Count: one, two, three, four, five\"}],\n \"stop\": [\"three\"]\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/chat/completions","host":["{{baseUrl}}"],"path":["openai","v1","chat","completions"]}}}, + { "name": "OpenAI: stream_options include_usage", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o-mini\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}],\n \"stream\": true,\n \"stream_options\": {\"include_usage\": true}\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/chat/completions","host":["{{baseUrl}}"],"path":["openai","v1","chat","completions"]}}}, + { "name": "OpenAI: predicted outputs", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o-mini\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Echo: hello world\"}],\n \"prediction\": {\"type\":\"content\",\"content\":\"hello world\"}\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/chat/completions","host":["{{baseUrl}}"],"path":["openai","v1","chat","completions"]}}}, + { "name": "OpenAI: service_tier auto", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o-mini\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}],\n \"service_tier\": \"auto\"\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/chat/completions","host":["{{baseUrl}}"],"path":["openai","v1","chat","completions"]}}}, + { "name": "OpenAI: store + metadata", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o-mini\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}],\n \"store\": true,\n \"metadata\": {\"harness\": \"backlog\"}\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/chat/completions","host":["{{baseUrl}}"],"path":["openai","v1","chat","completions"]}}}, + { "name": "OpenAI Responses: reasoning summary", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"o3-mini\",\n \"input\": \"What's 17*23?\",\n \"reasoning\": {\"summary\": \"auto\"}\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/responses","host":["{{baseUrl}}"],"path":["openai","v1","responses"]}}}, + { "name": "OpenAI Responses streaming: summary_index + obfuscation preserved", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code >= 400) { return; }","pm.test('summary_index and obfuscation survive stream', function () {"," var body = pm.response.text() || '';"," pm.expect(body, 'expected summary_index in SSE body').to.include('\"summary_index\"');"," pm.expect(body, 'expected obfuscation in SSE body').to.include('\"obfuscation\"');","});"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"o3-mini\",\n \"input\": \"What's 17*23?\",\n \"reasoning\": {\"summary\": \"auto\"},\n \"stream\": true,\n \"stream_options\": {\"include_obfuscation\": true}\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/responses","host":["{{baseUrl}}"],"path":["openai","v1","responses"]}}}, + { "name": "OpenAI Responses streaming: assistant phase preserved (gpt-5.3-codex)", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code >= 400) { return; }","pm.test('phase appears on assistant message items', function () {"," var body = pm.response.text() || '';"," var hasPhase = body.indexOf('\"phase\":\"final_answer\"') !== -1 || body.indexOf('\"phase\":\"commentary\"') !== -1;"," pm.expect(hasPhase, 'no phase field in SSE body. First 200 chars: ' + body.slice(0,200)).to.be.true;","});"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-5.3-codex\",\n \"input\": \"Solve 2+2 and explain your steps briefly.\",\n \"stream\": true\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/responses","host":["{{baseUrl}}"],"path":["openai","v1","responses"]}}}, + { "name": "OpenAI Responses: assistant phase input round-trip (gpt-5.3-codex)", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code >= 400) {"," pm.test('phase field accepted on input message', function () {"," var body = (pm.response.text() || '').toLowerCase();"," pm.expect(body, 'unexpected rejection of phase field: ' + body.slice(0,200)).to.not.include('unknown field \"phase\"');"," pm.expect(body).to.not.include('unexpected field \"phase\"');"," });"," return;","}","pm.test('response returned output items', function () {"," var body = pm.response.text() || '';"," pm.expect(body).to.include('\"output\"');","});"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-5.3-codex\",\n \"input\": [\n {\"role\":\"user\",\"content\":\"What's 2+2?\"},\n {\"role\":\"assistant\",\"phase\":\"final_answer\",\"content\":\"4\"},\n {\"role\":\"user\",\"content\":\"Now what's 3+3?\"}\n ]\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/responses","host":["{{baseUrl}}"],"path":["openai","v1","responses"]}}}, + { "name": "OpenAI Responses: background mode", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o\",\n \"input\": \"Hi\",\n \"background\": true\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/responses","host":["{{baseUrl}}"],"path":["openai","v1","responses"]}}}, + { "name": "OpenAI Responses: truncation auto", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o-mini\",\n \"input\": \"Hi\",\n \"truncation\": \"auto\"\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/responses","host":["{{baseUrl}}"],"path":["openai","v1","responses"]}}}, + { "name": "OpenAI Responses: include array", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o-mini\",\n \"input\": \"Hi\",\n \"include\": [\"message.input_image.image_url\"]\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/responses","host":["{{baseUrl}}"],"path":["openai","v1","responses"]}}}, + { "name": "OpenAI Responses: custom tool", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o\",\n \"input\": \"Send Slack message\",\n \"tools\": [{\"type\":\"function\",\"name\":\"send_slack\",\"parameters\":{\"type\":\"object\",\"properties\":{\"channel\":{\"type\":\"string\"},\"message\":{\"type\":\"string\"}}}}]\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/responses","host":["{{baseUrl}}"],"path":["openai","v1","responses"]}}}, + { "name": "OpenAI Responses: token counting", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"Authorization","value":"Bearer {{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gpt-4o-mini\",\n \"input\": \"Count tokens for me\"\n}"}, "url": {"raw":"{{baseUrl}}/openai/v1/responses/input_tokens","host":["{{baseUrl}}"],"path":["openai","v1","responses","input_tokens"]}}} + ] + }, + { + "name": "Anthropic Backlog", + "item": [ + { "name": "Anthropic: prompt caching 1h TTL", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"extended-cache-ttl-2025-04-11"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-haiku-4-5\",\n \"max_tokens\": 800,\n \"system\": [{\"type\":\"text\",\"text\":\"long context\",\"cache_control\":{\"type\":\"ephemeral\",\"ttl\":\"1h\"}}],\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Anthropic: web_fetch tool", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-7\",\n \"max_tokens\": 1024,\n \"messages\": [{\"role\":\"user\",\"content\":\"Fetch https://example.com and summarize\"}],\n \"tools\": [{\"type\":\"web_fetch_20250910\",\"name\":\"web_fetch\",\"max_uses\":2}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Anthropic: memory tool", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-7\",\n \"max_tokens\": 1024,\n \"messages\": [{\"role\":\"user\",\"content\":\"Remember my name is Akshay\"}],\n \"tools\": [{\"type\":\"memory_20250818\",\"name\":\"memory\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Anthropic: tool_search BM25", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-7\",\n \"max_tokens\": 1024,\n \"messages\": [{\"role\":\"user\",\"content\":\"List tools matching 'weather'\"}],\n \"tools\": [{\"type\":\"tool_search_tool_bm25\",\"name\":\"tool_search\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Anthropic: tool_search regex", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-7\",\n \"max_tokens\": 1024,\n \"messages\": [{\"role\":\"user\",\"content\":\"Find tools matching get_.*\"}],\n \"tools\": [{\"type\":\"tool_search_tool_regex\",\"name\":\"tool_search\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Anthropic: code_execution v2 (20250825)", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-7\",\n \"max_tokens\": 1024,\n \"messages\": [{\"role\":\"user\",\"content\":\"Compute 50! using Python\"}],\n \"tools\": [{\"type\":\"code_execution_20250825\",\"name\":\"code_execution\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Anthropic: code_execution programmatic (20260120)", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-7\",\n \"max_tokens\": 1024,\n \"messages\": [{\"role\":\"user\",\"content\":\"Plot sin(x) and tell me the period\"}],\n \"tools\": [{\"type\":\"code_execution_20260120\",\"name\":\"code_execution\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Anthropic: PDF input", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-haiku-4-5\",\n \"max_tokens\": 512,\n \"messages\": [{\"role\":\"user\",\"content\":[{\"type\":\"document\",\"source\":{\"type\":\"url\",\"url\":\"https://www.berkshirehathaway.com/letters/2024ltr.pdf\"}},{\"type\":\"text\",\"text\":\"Summarize\"}]}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Anthropic: stop sequences", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-haiku-4-5\",\n \"max_tokens\": 800,\n \"messages\": [{\"role\":\"user\",\"content\":\"Count: one, two, three, four in lowercase\"}],\n \"stop_sequences\": [\"three\"]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Anthropic: service_tier auto", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-haiku-4-5\",\n \"max_tokens\": 800,\n \"service_tier\": \"auto\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Anthropic: output_config effort high", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-6\",\n \"max_tokens\": 1024,\n \"output_config\": {\"effort\": \"high\"},\n \"messages\": [{\"role\":\"user\",\"content\":\"Solve x^2 - 5x + 6 = 0\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Anthropic: output_config format json_schema", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-haiku-4-5\",\n \"max_tokens\": 800,\n \"output_config\": {\"format\": {\"type\":\"json_schema\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}}}}},\n \"messages\": [{\"role\":\"user\",\"content\":\"Pick a city\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Anthropic: tool defer_loading + advanced beta", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"advanced-tool-use-2025-09-15"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-7\",\n \"max_tokens\": 1024,\n \"tools\": [{\"name\":\"slow_tool\",\"input_schema\":{\"type\":\"object\"},\"defer_loading\":true},{\"name\":\"fast_tool\",\"input_schema\":{\"type\":\"object\"},\"defer_loading\":false}],\n \"messages\": [{\"role\":\"user\",\"content\":\"Call slow_tool\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Anthropic: tool input_examples + beta", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"tool-examples-2025-10-29"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-haiku-4-5\",\n \"max_tokens\": 1024,\n \"tools\": [{\"name\":\"f\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"x\":{\"type\":\"number\"}}},\"input_examples\":[{\"input\":{\"x\":1},\"description\":\"basic\"}]}],\n \"messages\": [{\"role\":\"user\",\"content\":\"Call f\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Anthropic: strict tool input", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-haiku-4-5\",\n \"max_tokens\": 512,\n \"tools\": [{\"name\":\"strict_fn\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"a\":{\"type\":\"string\"}},\"required\":[\"a\"],\"additionalProperties\":false},\"strict\":true}],\n \"messages\": [{\"role\":\"user\",\"content\":\"Call strict_fn\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Anthropic: token counting", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-haiku-4-5\",\n \"messages\": [{\"role\":\"user\",\"content\":\"How many tokens?\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages/count_tokens","host":["{{baseUrl}}"],"path":["anthropic","v1","messages","count_tokens"]}}}, + { "name": "Anthropic: list models", "request": { "method": "GET", "header": [{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "url": {"raw":"{{baseUrl}}/anthropic/v1/models","host":["{{baseUrl}}"],"path":["anthropic","v1","models"]}}}, + { "name": "Anthropic: list batches", "request": { "method": "GET", "header": [{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "url": {"raw":"{{baseUrl}}/anthropic/v1/messages/batches","host":["{{baseUrl}}"],"path":["anthropic","v1","messages","batches"]}}}, + { "name": "Anthropic: list files", "request": { "method": "GET", "header": [{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"files-api-2025-04-14"}], "url": {"raw":"{{baseUrl}}/anthropic/v1/files","host":["{{baseUrl}}"],"path":["anthropic","v1","files"]}}} + ] + }, + { + "name": "Anthropic Beta Headers", + "item": [ + { "name": "Beta: token-efficient-tools", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"token-efficient-tools-2025-02-19"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-haiku-4-5\",\n \"max_tokens\": 800,\n \"tools\": [{\"name\":\"f\",\"input_schema\":{\"type\":\"object\"}}],\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Beta: fine-grained-tool-streaming", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"fine-grained-tool-streaming-2025-05-14"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-haiku-4-5\",\n \"max_tokens\": 800,\n \"stream\": true,\n \"tools\": [{\"name\":\"f\",\"input_schema\":{\"type\":\"object\"}}],\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "[PREVIEW] Beta: fast-mode (Opus 4.6)", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"fast-mode-2026-02-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-6\",\n \"max_tokens\": 800,\n \"speed\": \"fast\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "[PREVIEW] Beta: fast-mode (Opus 4.7)", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"fast-mode-2026-02-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-7\",\n \"max_tokens\": 800,\n \"speed\": \"fast\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "[PREVIEW] Beta: fast-mode (Opus 4.8)", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"fast-mode-2026-02-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-8\",\n \"max_tokens\": 800,\n \"speed\": \"fast\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Beta: context-1m", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"context-1m-2025-09-15"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-6\",\n \"max_tokens\": 800,\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Beta: interleaved-thinking", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"interleaved-thinking-2025-05-14"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-sonnet-4-6\",\n \"max_tokens\": 4096,\n \"thinking\": {\"type\":\"enabled\",\"budget_tokens\":2000},\n \"messages\": [{\"role\":\"user\",\"content\":\"Solve in steps\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Beta: skills bundle", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"skills-2025-10-29"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-7\",\n \"max_tokens\": 1024,\n \"messages\": [{\"role\":\"user\",\"content\":\"Use skills\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Beta: redact-thinking", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"redact-thinking-2025-09-15"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-sonnet-4-6\",\n \"max_tokens\": 1024,\n \"thinking\": {\"type\":\"enabled\",\"budget_tokens\":2000},\n \"messages\": [{\"role\":\"user\",\"content\":\"Reasoned answer\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Beta: compaction", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"compact-2025-09-15"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-6\",\n \"max_tokens\": 800,\n \"messages\": [{\"role\":\"user\",\"content\":\"Long convo\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Mid-conv system message (Opus 4.8) — system ends array", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-8\",\n \"max_tokens\": 512,\n \"system\": [{\"type\":\"text\",\"text\":\"You are a helpful assistant.\"}],\n \"messages\": [\n {\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]},\n {\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Hi!\"}]},\n {\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"How are you?\"}]},\n {\"role\":\"system\",\"content\":[{\"type\":\"text\",\"text\":\"Respond only in one word.\"}]}\n ]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Mid-conv system message (Opus 4.8) — system mid-history (before assistant)", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-8\",\n \"max_tokens\": 512,\n \"system\": [{\"type\":\"text\",\"text\":\"You are a helpful assistant.\"}],\n \"messages\": [\n {\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]},\n {\"role\":\"system\",\"content\":[{\"type\":\"text\",\"text\":\"From now on be very concise.\"}]},\n {\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Hi!\"}]},\n {\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"How are you?\"}]}\n ]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}} + ] + }, + { + "name": "Bedrock Backlog", + "item": [ + { "name": "Bedrock Converse: streaming", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"messages\": [{\"role\":\"user\",\"content\":[{\"text\":\"Count to 5\"}]}],\n \"inferenceConfig\": {\"maxTokens\": 256}\n}"}, "url": {"raw":"{{baseUrl}}/bedrock/model/{{bedrockModel}}/converse-stream","host":["{{baseUrl}}"],"path":["bedrock","model","{{bedrockModel}}","converse-stream"]}}}, + { "name": "Bedrock Converse: stop sequences", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"messages\": [{\"role\":\"user\",\"content\":[{\"text\":\"Count: one, two, three, four in lowercase\"}]}],\n \"inferenceConfig\": {\"maxTokens\": 256, \"stopSequences\": [\"three\"]}\n}"}, "url": {"raw":"{{baseUrl}}/bedrock/model/{{bedrockModel}}/converse","host":["{{baseUrl}}"],"path":["bedrock","model","{{bedrockModel}}","converse"]}}}, + { "name": "Bedrock Converse: tool choice forced", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"messages\": [{\"role\":\"user\",\"content\":[{\"text\":\"Pick a number\"}]}],\n \"toolConfig\": {\"tools\":[{\"toolSpec\":{\"name\":\"pick\",\"inputSchema\":{\"json\":{\"type\":\"object\",\"properties\":{\"n\":{\"type\":\"number\"}}}}}}],\"toolChoice\":{\"tool\":{\"name\":\"pick\"}}},\n \"inferenceConfig\": {\"maxTokens\":256}\n}"}, "url": {"raw":"{{baseUrl}}/bedrock/model/{{bedrockModel}}/converse","host":["{{baseUrl}}"],"path":["bedrock","model","{{bedrockModel}}","converse"]}}}, + { "name": "Bedrock Converse: performance config optimized", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"messages\": [{\"role\":\"user\",\"content\":[{\"text\":\"Hi\"}]}],\n \"performanceConfig\": {\"latency\":\"optimized\"},\n \"inferenceConfig\": {\"maxTokens\":256}\n}"}, "url": {"raw":"{{baseUrl}}/bedrock/model/us.amazon.nova-pro-v1:0/converse","host":["{{baseUrl}}"],"path":["bedrock","model","us.amazon.nova-pro-v1:0","converse"]}}}, + { "name": "Bedrock Converse: request metadata", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"messages\": [{\"role\":\"user\",\"content\":[{\"text\":\"Hi\"}]}],\n \"requestMetadata\": {\"session\":\"harness\"},\n \"inferenceConfig\": {\"maxTokens\":256}\n}"}, "url": {"raw":"{{baseUrl}}/bedrock/model/{{bedrockModel}}/converse","host":["{{baseUrl}}"],"path":["bedrock","model","{{bedrockModel}}","converse"]}}}, + { "name": "Bedrock InvokeModel: direct Anthropic shape", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"anthropic_version\": \"bedrock-2023-05-31\",\n \"max_tokens\": 800,\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}]\n}"}, "url": {"raw":"{{baseUrl}}/bedrock/model/{{bedrockModel}}/invoke","host":["{{baseUrl}}"],"path":["bedrock","model","{{bedrockModel}}","invoke"]}}} + ] + }, + { + "name": "Gemini Backlog", + "item": [ + { "name": "Gemini: tool config functionCallingConfig ANY", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-goog-api-key","value":"{{genaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"contents\": [{\"parts\":[{\"text\":\"Call get_weather\"}]}],\n \"tools\": [{\"functionDeclarations\":[{\"name\":\"get_weather\",\"parameters\":{\"type\":\"OBJECT\",\"properties\":{\"city\":{\"type\":\"STRING\"}}}}]}],\n \"toolConfig\": {\"functionCallingConfig\":{\"mode\":\"ANY\",\"allowedFunctionNames\":[\"get_weather\"]}}\n}"}, "url": {"raw":"{{baseUrl}}/genai/v1beta/models/{{genaiModel}}:generateContent","host":["{{baseUrl}}"],"path":["genai","v1beta","models","{{genaiModel}}:generateContent"]}}}, + { "name": "Gemini: stop sequences", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-goog-api-key","value":"{{genaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"contents\": [{\"parts\":[{\"text\":\"Count: one, two, three, four in lowercase\"}]}],\n \"generationConfig\": {\"stopSequences\":[\"three\"]}\n}"}, "url": {"raw":"{{baseUrl}}/genai/v1beta/models/{{genaiModel}}:generateContent","host":["{{baseUrl}}"],"path":["genai","v1beta","models","{{genaiModel}}:generateContent"]}}}, + { "name": "Gemini: temperature + topP + topK", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-goog-api-key","value":"{{genaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"contents\": [{\"parts\":[{\"text\":\"Pick a number\"}]}],\n \"generationConfig\": {\"temperature\":0.7,\"topP\":0.9,\"topK\":40}\n}"}, "url": {"raw":"{{baseUrl}}/genai/v1beta/models/{{genaiModel}}:generateContent","host":["{{baseUrl}}"],"path":["genai","v1beta","models","{{genaiModel}}:generateContent"]}}}, + { "name": "Gemini: response logprobs", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-goog-api-key","value":"{{genaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"contents\": [{\"parts\":[{\"text\":\"Hi\"}]}],\n \"generationConfig\": {\"responseLogprobs\":true,\"logprobs\":3}\n}"}, "url": {"raw":"{{baseUrl}}/genai/v1beta/models/{{genaiModel}}:generateContent","host":["{{baseUrl}}"],"path":["genai","v1beta","models","{{genaiModel}}:generateContent"]}}}, + { "name": "Gemini: presence + frequency penalty", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-goog-api-key","value":"{{genaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"contents\": [{\"parts\":[{\"text\":\"Tell a story\"}]}],\n \"generationConfig\": {\"presencePenalty\":0.5,\"frequencyPenalty\":0.5}\n}"}, "url": {"raw":"{{baseUrl}}/genai/v1beta/models/{{genaiModel}}:generateContent","host":["{{baseUrl}}"],"path":["genai","v1beta","models","{{genaiModel}}:generateContent"]}}}, + { "name": "[PREVIEW] Gemini: PDF input", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-goog-api-key","value":"{{genaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"contents\": [{\"parts\":[{\"text\":\"Summarize this PDF\"},{\"fileData\":{\"mimeType\":\"application/pdf\",\"fileUri\":\"https://storage.googleapis.com/generativeai-downloads/data/A17_FlightPlan.pdf\"}}]}]\n}"}, "url": {"raw":"{{baseUrl}}/genai/v1beta/models/{{genaiModel}}:generateContent","host":["{{baseUrl}}"],"path":["genai","v1beta","models","{{genaiModel}}:generateContent"]}}}, + { "name": "Gemini: YouTube URL input", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-goog-api-key","value":"{{genaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"contents\": [{\"parts\":[{\"text\":\"Summarize this video\"},{\"fileData\":{\"fileUri\":\"https://www.youtube.com/watch?v=jNQXAC9IVRw\"}}]}]\n}"}, "url": {"raw":"{{baseUrl}}/genai/v1beta/models/{{genaiModel}}:generateContent","host":["{{baseUrl}}"],"path":["genai","v1beta","models","{{genaiModel}}:generateContent"]}}}, + { "name": "Gemini: URL context tool", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-goog-api-key","value":"{{genaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"contents\": [{\"parts\":[{\"text\":\"Summarize https://anthropic.com/news\"}]}],\n \"tools\": [{\"urlContext\":{}}]\n}"}, "url": {"raw":"{{baseUrl}}/genai/v1beta/models/{{genaiModel}}:generateContent","host":["{{baseUrl}}"],"path":["genai","v1beta","models","{{genaiModel}}:generateContent"]}}}, + { "name": "Gemini: count tokens", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-goog-api-key","value":"{{genaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"contents\": [{\"parts\":[{\"text\":\"How many tokens?\"}]}]\n}"}, "url": {"raw":"{{baseUrl}}/genai/v1beta/models/{{genaiModel}}:countTokens","host":["{{baseUrl}}"],"path":["genai","v1beta","models","{{genaiModel}}:countTokens"]}}}, + { "name": "Gemini: list models", "request": { "method": "GET", "header": [{"key":"x-goog-api-key","value":"{{genaiKey}}"}], "url": {"raw":"{{baseUrl}}/genai/v1beta/models","host":["{{baseUrl}}"],"path":["genai","v1beta","models"]}}} + ] + }, + { + "name": "Vertex Backlog", + "item": [ + { "name": "Vertex: anthropic_version in body", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"vertex/claude-opus-4-7\",\n \"anthropic_version\": \"vertex-2023-10-16\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi via Vertex\"}]\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, + { "name": "Vertex: streaming Anthropic", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"vertex/claude-opus-4-7\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Count 1-5\"}],\n \"stream\": true\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, + { "name": "[PREVIEW] Vertex Model Garden: Llama (publishers/meta/...)", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"vertex/meta/llama-4-maverick-17b-128e-instruct-maas\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}]\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, + { "name": "[PREVIEW] Vertex Model Garden: Mistral", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"vertex/publishers/mistralai/models/mistral-large-2411\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}]\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}} + ] + }, + { + "name": "Azure Backlog", + "item": [ + { "name": "Azure: tools (function calling)", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"api-key","value":"{{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"messages\": [{\"role\":\"user\",\"content\":\"What's the weather?\"}],\n \"tools\": [{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}}}}}]\n}"}, "url": {"raw":"{{baseUrl}}/openai/openai/deployments/{{azureDeployment}}/chat/completions?api-version={{azureApiVersion}}","host":["{{baseUrl}}"],"path":["openai","openai","deployments","{{azureDeployment}}","chat","completions"],"query":[{"key":"api-version","value":"{{azureApiVersion}}"}]}}}, + { "name": "Azure: streaming", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"api-key","value":"{{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"messages\": [{\"role\":\"user\",\"content\":\"Count 1-5\"}],\n \"stream\": true\n}"}, "url": {"raw":"{{baseUrl}}/openai/openai/deployments/{{azureDeployment}}/chat/completions?api-version={{azureApiVersion}}","host":["{{baseUrl}}"],"path":["openai","openai","deployments","{{azureDeployment}}","chat","completions"],"query":[{"key":"api-version","value":"{{azureApiVersion}}"}]}}}, + { "name": "Azure: structured output json_schema", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"api-key","value":"{{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"messages\": [{\"role\":\"user\",\"content\":\"Extract city/country for Paris\"}],\n \"response_format\": {\"type\":\"json_schema\",\"json_schema\":{\"name\":\"city\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"},\"country\":{\"type\":\"string\"}},\"required\":[\"city\",\"country\"],\"additionalProperties\":false}}}\n}"}, "url": {"raw":"{{baseUrl}}/openai/openai/deployments/{{azureDeployment}}/chat/completions?api-version={{azureApiVersion}}","host":["{{baseUrl}}"],"path":["openai","openai","deployments","{{azureDeployment}}","chat","completions"],"query":[{"key":"api-version","value":"{{azureApiVersion}}"}]}}}, + { "name": "Azure: vision (image_url)", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"api-key","value":"{{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"messages\": [{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://storage.googleapis.com/generativeai-downloads/images/scones.jpg\"}}]}]\n}"}, "url": {"raw":"{{baseUrl}}/openai/openai/deployments/{{azureDeployment}}/chat/completions?api-version={{azureApiVersion}}","host":["{{baseUrl}}"],"path":["openai","openai","deployments","{{azureDeployment}}","chat","completions"],"query":[{"key":"api-version","value":"{{azureApiVersion}}"}]}}}, + { "name": "Azure: system + multi-turn", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"api-key","value":"{{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"messages\": [{\"role\":\"system\",\"content\":\"You are a pirate.\"},{\"role\":\"user\",\"content\":\"Hi\"},{\"role\":\"assistant\",\"content\":\"Arrr!\"},{\"role\":\"user\",\"content\":\"Tell a joke\"}]\n}"}, "url": {"raw":"{{baseUrl}}/openai/openai/deployments/{{azureDeployment}}/chat/completions?api-version={{azureApiVersion}}","host":["{{baseUrl}}"],"path":["openai","openai","deployments","{{azureDeployment}}","chat","completions"],"query":[{"key":"api-version","value":"{{azureApiVersion}}"}]}}}, + { "name": "Azure On Your Data: azure_search", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"api-key","value":"{{openaiKey}}"}], "body": {"mode":"raw","raw":"{\n \"messages\": [{\"role\":\"user\",\"content\":\"What's in the docs?\"}],\n \"data_sources\": [{\"type\":\"azure_search\",\"parameters\":{\"endpoint\":\"https://placeholder.search.windows.net\",\"index_name\":\"placeholder\",\"authentication\":{\"type\":\"api_key\",\"key\":\"placeholder\"}}}]\n}"}, "url": {"raw":"{{baseUrl}}/openai/openai/deployments/{{azureDeployment}}/chat/completions?api-version={{azureApiVersion}}","host":["{{baseUrl}}"],"path":["openai","openai","deployments","{{azureDeployment}}","chat","completions"],"query":[{"key":"api-version","value":"{{azureApiVersion}}"}]}}} + ] + }, + { + "name": "Cross-Provider Backlog", + "item": [ + { "name": "Cross-cut: code execution Anthropic", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"anthropic/claude-opus-4-7\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Compute 50!\"}],\n \"tools\": [{\"type\":\"code_execution_20250522\",\"name\":\"code_execution\"}]\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, + { "name": "Cross-cut: code execution Gemini", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gemini/gemini-2.5-flash\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Compute 50!\"}],\n \"tools\": [{\"type\":\"code_execution\"}]\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, + { "name": "Cross-cut: extended thinking via cross-model", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"anthropic/claude-sonnet-4-6\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Plan a trip\"}],\n \"thinking\": {\"type\":\"enabled\",\"budget_tokens\":2000},\n \"max_tokens\": 4096\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, + { "name": "Cross-cut: prompt caching via cross-model", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"anthropic/claude-haiku-4-5\",\n \"messages\": [{\"role\":\"system\",\"content\":[{\"type\":\"text\",\"text\":\"Long ctx\",\"cache_control\":{\"type\":\"ephemeral\"}}]},{\"role\":\"user\",\"content\":\"Hi\"}]\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, + { "name": "Cross-cut: stop sequences (OpenAI)", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code < 400) { pm.test('Stop sequence: halted before stop token', function () { var j = pm.response.json(); var c = (j.choices && j.choices[0] && j.choices[0].message && j.choices[0].message.content) || ''; var fr = (j.choices && j.choices[0] && j.choices[0].finish_reason) || ''; pm.expect(c.toLowerCase(), 'stop token \"three\" leaked into content').to.not.include('three'); pm.expect(['stop','stop_sequence','length','content_filter'], 'unexpected finish_reason: ' + fr).to.include(fr); }); }"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"openai/gpt-4o-mini\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Count: one, two, three, four in lowercase\"}],\n \"stop\": [\"three\"]\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, + { "name": "Cross-cut: stop sequences (Anthropic)", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code < 400) { pm.test('Stop sequence: halted before stop token', function () { var j = pm.response.json(); var c = (j.choices && j.choices[0] && j.choices[0].message && j.choices[0].message.content) || ''; var fr = (j.choices && j.choices[0] && j.choices[0].finish_reason) || ''; pm.expect(c.toLowerCase(), 'stop token \"three\" leaked into content').to.not.include('three'); pm.expect(['stop','stop_sequence','length','content_filter'], 'unexpected finish_reason: ' + fr).to.include(fr); }); }"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"anthropic/claude-haiku-4-5\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Count: one, two, three, four in lowercase\"}],\n \"stop\": [\"three\"]\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, + { "name": "Cross-cut: stop sequences (Gemini)", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code < 400) { pm.test('Stop sequence: halted before stop token', function () { var j = pm.response.json(); var c = (j.choices && j.choices[0] && j.choices[0].message && j.choices[0].message.content) || ''; var fr = (j.choices && j.choices[0] && j.choices[0].finish_reason) || ''; pm.expect(c.toLowerCase(), 'stop token \"three\" leaked into content').to.not.include('three'); pm.expect(['stop','stop_sequence','length','content_filter'], 'unexpected finish_reason: ' + fr).to.include(fr); }); }"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"gemini/gemini-2.5-flash\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Count: one, two, three, four in lowercase\"}],\n \"stop\": [\"three\"]\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, + { "name": "Cross-cut: tool_choice forced (OpenAI)", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"openai/gpt-4o-mini\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}],\n \"tools\": [{\"type\":\"function\",\"function\":{\"name\":\"f\",\"parameters\":{\"type\":\"object\"}}}],\n \"tool_choice\": \"required\"\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, + { "name": "Cross-cut: tool_choice forced (Bedrock)", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"bedrock/global.anthropic.claude-sonnet-4-6\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}],\n \"tools\": [{\"type\":\"function\",\"function\":{\"name\":\"f\",\"parameters\":{\"type\":\"object\"}}}],\n \"tool_choice\": \"required\"\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, { "name": "Cross-cut: structured output Anthropic", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code < 400) { pm.test('Structured output: schema-compliant JSON (city)', function () { var j = pm.response.json(); var c = ''; if (j.choices && j.choices[0] && j.choices[0].message) { c = j.choices[0].message.content || ''; } if (!c && Array.isArray(j.content)) { var tb = j.content.find(function (b) { return b.type === 'text' && b.text; }); c = tb ? tb.text : ''; } pm.expect(c, 'content was empty').to.be.a('string').and.not.empty; var p; try { p = JSON.parse(c); } catch (e) { pm.expect.fail('content not JSON: ' + e.message + ' (got: ' + c.slice(0,120) + ')'); return; } pm.expect(p).to.have.property('city').that.is.a('string'); }); }"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"anthropic/claude-haiku-4-5\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Pick a city\"}],\n \"response_format\": {\"type\":\"json_schema\",\"json_schema\":{\"name\":\"city\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, { "name": "Cross-cut: structured output Bedrock", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code < 400) { pm.test('Structured output: schema-compliant JSON (city)', function () { var j = pm.response.json(); var c = ''; if (j.choices && j.choices[0] && j.choices[0].message) { c = j.choices[0].message.content || ''; } if (!c && Array.isArray(j.content)) { var tb = j.content.find(function (b) { return b.type === 'text' && b.text; }); c = tb ? tb.text : ''; } pm.expect(c, 'content was empty').to.be.a('string').and.not.empty; var p; try { p = JSON.parse(c); } catch (e) { pm.expect.fail('content not JSON: ' + e.message + ' (got: ' + c.slice(0,120) + ')'); return; } pm.expect(p).to.have.property('city').that.is.a('string'); }); }"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"bedrock/global.anthropic.claude-sonnet-4-6\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Pick a city\"}],\n \"response_format\": {\"type\":\"json_schema\",\"json_schema\":{\"name\":\"city\",\"strict\":true,\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}}}}}\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, { "name": "Cross-cut: vision Bedrock", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"bedrock/global.anthropic.claude-sonnet-4-6\",\n \"messages\": [{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"https://storage.googleapis.com/generativeai-downloads/images/scones.jpg\"}}]}]\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, @@ -2585,9 +3314,342 @@ { "name": "Cross-cut: anthropic/claude-opus-4-7 mid-conv system (fallback: merged to top-level)", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code < 400) { pm.test('Response content non-empty', function () { var j = pm.response.json(); var c = (j.choices && j.choices[0] && j.choices[0].message && j.choices[0].message.content) || ''; pm.expect(c).to.be.a('string').and.not.empty; }); }"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"anthropic/claude-opus-4-7\",\n \"max_tokens\": 512,\n \"messages\": [\n {\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},\n {\"role\":\"user\",\"content\":\"Hello\"},\n {\"role\":\"assistant\",\"content\":\"Hi!\"},\n {\"role\":\"user\",\"content\":\"How are you?\"},\n {\"role\":\"system\",\"content\":\"Respond only in one word.\"}\n ]\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, { "name": "Cross-cut: anthropic/claude-opus-4-8 mid-conv system drop-in /anthropic (ends array)", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code < 400) { pm.test('Anthropic drop-in: content block present', function () { var j = pm.response.json(); var hasContent = Array.isArray(j.content) && j.content.some(function(b) { return b.type === 'text' && b.text; }); pm.expect(hasContent, 'expected text content block').to.be.true; }); }"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-opus-4-8\",\n \"max_tokens\": 512,\n \"system\": [{\"type\":\"text\",\"text\":\"You are a helpful assistant.\"}],\n \"messages\": [\n {\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]},\n {\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Hi!\"}]},\n {\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"How are you?\"}]},\n {\"role\":\"system\",\"content\":[{\"type\":\"text\",\"text\":\"Respond only in one word.\"}]}\n ]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}} ] + }, + { + "name": "Cross-Cut Round 30: Fable 5 / Mythos Feature Gating", + "description": "Verifies the Fable 5 / Mythos (claude-fable-5, claude-mythos-5) request-surface gating added to the Anthropic converters.\nFable/Mythos behave like Opus 4.7+ (adaptive-only thinking; temperature/top_p/top_k removed) AND additionally reject thinking:{type:\"disabled\"} (adaptive is always on).\nGating covered: adaptive thinking, budget_tokens→adaptive, sampling-param strip, disabled→omitted (typed path only), effort high/xhigh/max, structured outputs, task budgets, computer-use new-gen tools, web_search dynamic filtering, mid-conversation system messages, and fast-mode NOT supported (speed stripped).\nScripts are guarded on code<400 so the suite tolerates accounts without Fable access (same convention as the [PREVIEW] items).", + "item": [ + { "name": "Native Fable 5: adaptive thinking", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code < 400) { pm.test('Anthropic drop-in: content block present', function () { var j = pm.response.json(); var hasContent = Array.isArray(j.content) && j.content.some(function(b) { return b.type === 'text' && b.text; }); pm.expect(hasContent, 'expected text content block').to.be.true; }); }"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-fable-5\",\n \"max_tokens\": 4096,\n \"thinking\": { \"type\": \"adaptive\" },\n \"messages\": [{\"role\":\"user\",\"content\":\"Solve in steps\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Native Fable 5: output_config effort high", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code < 400) { pm.test('Anthropic drop-in: content block present', function () { var j = pm.response.json(); var hasContent = Array.isArray(j.content) && j.content.some(function(b) { return b.type === 'text' && b.text; }); pm.expect(hasContent, 'expected text content block').to.be.true; }); }"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-fable-5\",\n \"max_tokens\": 1024,\n \"output_config\": {\"effort\": \"high\"},\n \"messages\": [{\"role\":\"user\",\"content\":\"Solve x^2 - 5x + 6 = 0\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Native Fable 5: output_config effort xhigh", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code < 400) { pm.test('Anthropic drop-in: content block present', function () { var j = pm.response.json(); var hasContent = Array.isArray(j.content) && j.content.some(function(b) { return b.type === 'text' && b.text; }); pm.expect(hasContent, 'expected text content block').to.be.true; }); }"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-fable-5\",\n \"max_tokens\": 1024,\n \"output_config\": {\"effort\": \"xhigh\"},\n \"messages\": [{\"role\":\"user\",\"content\":\"Solve x^2 - 5x + 6 = 0\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Native Fable 5: output_config effort max", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code < 400) { pm.test('Anthropic drop-in: content block present', function () { var j = pm.response.json(); var hasContent = Array.isArray(j.content) && j.content.some(function(b) { return b.type === 'text' && b.text; }); pm.expect(hasContent, 'expected text content block').to.be.true; }); }"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-fable-5\",\n \"max_tokens\": 1024,\n \"output_config\": {\"effort\": \"max\"},\n \"messages\": [{\"role\":\"user\",\"content\":\"Solve x^2 - 5x + 6 = 0\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Native Fable 5: output_config format json_schema (structured outputs)", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code < 400) { pm.test('Anthropic drop-in: content block present', function () { var j = pm.response.json(); var hasContent = Array.isArray(j.content) && j.content.some(function(b) { return b.type === 'text' && b.text; }); pm.expect(hasContent, 'expected text content block').to.be.true; }); }"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-fable-5\",\n \"max_tokens\": 800,\n \"output_config\": {\"format\": {\"type\":\"json_schema\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}}}}},\n \"messages\": [{\"role\":\"user\",\"content\":\"Pick a city\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Native Fable 5: speed:fast stripped (fast mode unsupported, no beta header)", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code < 400) { pm.test('Anthropic drop-in: content block present', function () { var j = pm.response.json(); var hasContent = Array.isArray(j.content) && j.content.some(function(b) { return b.type === 'text' && b.text; }); pm.expect(hasContent, 'expected text content block').to.be.true; }); }"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-fable-5\",\n \"max_tokens\": 800,\n \"speed\": \"fast\",\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Native Fable 5: task_budget (beta task-budgets-2026-03-13)", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code < 400) { pm.test('Anthropic drop-in: content block present', function () { var j = pm.response.json(); var hasContent = Array.isArray(j.content) && j.content.some(function(b) { return b.type === 'text' && b.text; }); pm.expect(hasContent, 'expected text content block').to.be.true; }); }"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"task-budgets-2026-03-13"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-fable-5\",\n \"max_tokens\": 4096,\n \"output_config\": {\"task_budget\": {\"type\":\"tokens\",\"total\":20000}},\n \"messages\": [{\"role\":\"user\",\"content\":\"Plan and solve a multi-step task.\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Native Fable 5: computer use new-gen tools (computer-use-2025-11-24)", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"},{"key":"anthropic-beta","value":"computer-use-2025-11-24"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-fable-5\",\n \"max_tokens\": 4096,\n \"tools\": [\n { \"type\": \"computer_20251124\", \"name\": \"computer\", \"display_width_px\": 1024, \"display_height_px\": 768, \"display_number\": 1 },\n { \"type\": \"bash_20250124\", \"name\": \"bash\" },\n { \"type\": \"text_editor_20250728\", \"name\": \"str_replace_based_edit_tool\" }\n ],\n \"messages\": [{\"role\":\"user\",\"content\":\"Take a screenshot of the desktop.\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Native Fable 5: web_search dynamic filtering (web_search_20260209)", "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-fable-5\",\n \"max_tokens\": 1024,\n \"messages\": [{\"role\":\"user\",\"content\":\"Find recent AI papers.\"}],\n \"tools\": [{\"type\":\"web_search_20260209\",\"name\":\"web_search\",\"max_uses\":3}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Native Fable 5: mid-conv system message (ends array)", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code < 400) { pm.test('Anthropic drop-in: content block present', function () { var j = pm.response.json(); var hasContent = Array.isArray(j.content) && j.content.some(function(b) { return b.type === 'text' && b.text; }); pm.expect(hasContent, 'expected text content block').to.be.true; }); }"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-fable-5\",\n \"max_tokens\": 512,\n \"system\": [{\"type\":\"text\",\"text\":\"You are a helpful assistant.\"}],\n \"messages\": [\n {\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]},\n {\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Hi!\"}]},\n {\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"How are you?\"}]},\n {\"role\":\"system\",\"content\":[{\"type\":\"text\",\"text\":\"Respond only in one word.\"}]}\n ]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Native Fable 5: mid-conv system message (before assistant)", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code < 400) { pm.test('Anthropic drop-in: content block present', function () { var j = pm.response.json(); var hasContent = Array.isArray(j.content) && j.content.some(function(b) { return b.type === 'text' && b.text; }); pm.expect(hasContent, 'expected text content block').to.be.true; }); }"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-fable-5\",\n \"max_tokens\": 512,\n \"system\": [{\"type\":\"text\",\"text\":\"You are a helpful assistant.\"}],\n \"messages\": [\n {\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Hello\"}]},\n {\"role\":\"system\",\"content\":[{\"type\":\"text\",\"text\":\"From now on be very concise.\"}]},\n {\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Hi!\"}]},\n {\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"How are you?\"}]}\n ]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Native Mythos 5: adaptive thinking (family parity)", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code < 400) { pm.test('Anthropic drop-in: content block present', function () { var j = pm.response.json(); var hasContent = Array.isArray(j.content) && j.content.some(function(b) { return b.type === 'text' && b.text; }); pm.expect(hasContent, 'expected text content block').to.be.true; }); }"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"},{"key":"x-api-key","value":"{{anthropicKey}}"},{"key":"anthropic-version","value":"2023-06-01"}], "body": {"mode":"raw","raw":"{\n \"model\": \"claude-mythos-5\",\n \"max_tokens\": 4096,\n \"thinking\": { \"type\": \"adaptive\" },\n \"messages\": [{\"role\":\"user\",\"content\":\"Solve in steps\"}]\n}"}, "url": {"raw":"{{baseUrl}}/anthropic/v1/messages","host":["{{baseUrl}}"],"path":["anthropic","v1","messages"]}}}, + { "name": "Cross-cut: anthropic/claude-fable-5 adaptive thinking", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code < 400) { pm.test('Response content non-empty', function () { var j = pm.response.json(); var c = (j.choices && j.choices[0] && j.choices[0].message && j.choices[0].message.content) || ''; pm.expect(c).to.be.a('string').and.not.empty; }); }"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"anthropic/claude-fable-5\",\n \"max_tokens\": 4096,\n \"thinking\": {\"type\":\"adaptive\"},\n \"messages\": [{\"role\":\"user\",\"content\":\"Solve in steps\"}]\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, + { "name": "Cross-cut: anthropic/claude-fable-5 enabled thinking → adaptive (budget_tokens removed)", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code < 400) { pm.test('Response content non-empty', function () { var j = pm.response.json(); var c = (j.choices && j.choices[0] && j.choices[0].message && j.choices[0].message.content) || ''; pm.expect(c).to.be.a('string').and.not.empty; }); }"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"anthropic/claude-fable-5\",\n \"max_tokens\": 4096,\n \"thinking\": {\"type\":\"enabled\",\"budget_tokens\":2000},\n \"messages\": [{\"role\":\"user\",\"content\":\"Solve in steps\"}]\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, + { "name": "Cross-cut: anthropic/claude-fable-5 disabled thinking → omitted (no 400)", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code < 400) { pm.test('Response content non-empty', function () { var j = pm.response.json(); var c = (j.choices && j.choices[0] && j.choices[0].message && j.choices[0].message.content) || ''; pm.expect(c).to.be.a('string').and.not.empty; }); }"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"anthropic/claude-fable-5\",\n \"max_tokens\": 1024,\n \"thinking\": {\"type\":\"disabled\"},\n \"messages\": [{\"role\":\"user\",\"content\":\"Hello\"}]\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, + { "name": "Cross-cut: anthropic/claude-fable-5 sampling params stripped (temperature/top_p/top_k)", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code < 400) { pm.test('Response content non-empty', function () { var j = pm.response.json(); var c = (j.choices && j.choices[0] && j.choices[0].message && j.choices[0].message.content) || ''; pm.expect(c).to.be.a('string').and.not.empty; }); }"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"anthropic/claude-fable-5\",\n \"max_tokens\": 1024,\n \"temperature\": 0.7,\n \"top_p\": 0.9,\n \"top_k\": 5,\n \"messages\": [{\"role\":\"user\",\"content\":\"Hello\"}]\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}}, + { "name": "Cross-cut: anthropic/claude-fable-5 mid-conv system", "event": [{"listen":"test","script":{"type":"text/javascript","exec":["if (pm.response.code < 400) { pm.test('Response content non-empty', function () { var j = pm.response.json(); var c = (j.choices && j.choices[0] && j.choices[0].message && j.choices[0].message.content) || ''; pm.expect(c).to.be.a('string').and.not.empty; }); }"]}}], "request": { "method": "POST", "header": [{"key":"Content-Type","value":"application/json"}], "body": {"mode":"raw","raw":"{\n \"model\": \"anthropic/claude-fable-5\",\n \"max_tokens\": 512,\n \"messages\": [\n {\"role\":\"system\",\"content\":\"You are a helpful assistant.\"},\n {\"role\":\"user\",\"content\":\"Hello\"},\n {\"role\":\"assistant\",\"content\":\"Hi!\"},\n {\"role\":\"user\",\"content\":\"How are you?\"},\n {\"role\":\"system\",\"content\":\"Respond only in one word.\"}\n ]\n}"}, "url": {"raw":"{{baseUrl}}/v1/chat/completions","host":["{{baseUrl}}"],"path":["v1","chat","completions"]}}} + ] } ] + }, + { + "name": "13. OpenRouter Prompt Caching (cache_control)", + "description": "Sends OpenAI-format chat requests through the OpenRouter provider with Anthropic-style cache_control breakpoints on content blocks and tools. OpenRouter forwards cache_control to Claude/Gemini (explicit caching) and harmlessly ignores it for implicit-caching models (OpenAI/DeepSeek) \u2014 these must NOT 400. Bifrost preserves cache_control only when the provider is OpenRouter (stripped for native OpenAI).", + "item": [ + { + "name": "OpenRouter Anthropic Claude: cache_control on system block", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "if (pm.response.code < 400) { pm.test('cache_control forwarded: response ok + usage present', function () { var j = pm.response.json(); pm.expect(j.usage || {}).to.be.an('object'); }); } pm.test('cache_control not rejected (no 400)', function () { pm.expect(pm.response.code).to.not.equal(400); });" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\"model\": \"openrouter/anthropic/claude-sonnet-4\", \"max_tokens\": 64, \"messages\": [{\"role\": \"system\", \"content\": [{\"type\": \"text\", \"text\": \"You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant.\", \"cache_control\": {\"type\": \"ephemeral\"}}]}, {\"role\": \"user\", \"content\": \"Reply with one word.\"}]}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "OpenRouter Anthropic Claude: cache_control on tool", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "if (pm.response.code < 400) { pm.test('cache_control forwarded: response ok + usage present', function () { var j = pm.response.json(); pm.expect(j.usage || {}).to.be.an('object'); }); } pm.test('cache_control not rejected (no 400)', function () { pm.expect(pm.response.code).to.not.equal(400); });" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\"model\": \"openrouter/anthropic/claude-sonnet-4\", \"max_tokens\": 64, \"messages\": [{\"role\": \"system\", \"content\": [{\"type\": \"text\", \"text\": \"You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant.\", \"cache_control\": {\"type\": \"ephemeral\"}}]}, {\"role\": \"user\", \"content\": \"Reply with one word.\"}], \"tools\": [{\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Get weather for a city\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"]}}, \"cache_control\": {\"type\": \"ephemeral\"}}]}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "OpenRouter Anthropic Claude: cache_control 1h ttl", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "if (pm.response.code < 400) { pm.test('cache_control forwarded: response ok + usage present', function () { var j = pm.response.json(); pm.expect(j.usage || {}).to.be.an('object'); }); } pm.test('cache_control not rejected (no 400)', function () { pm.expect(pm.response.code).to.not.equal(400); });" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\"model\": \"openrouter/anthropic/claude-sonnet-4\", \"max_tokens\": 64, \"messages\": [{\"role\": \"system\", \"content\": [{\"type\": \"text\", \"text\": \"You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant.\", \"cache_control\": {\"type\": \"ephemeral\", \"ttl\": \"1h\"}}]}, {\"role\": \"user\", \"content\": \"Reply with one word.\"}]}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "OpenRouter Google Gemini Flash: cache_control on system block", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "if (pm.response.code < 400) { pm.test('cache_control forwarded: response ok + usage present', function () { var j = pm.response.json(); pm.expect(j.usage || {}).to.be.an('object'); }); } pm.test('cache_control not rejected (no 400)', function () { pm.expect(pm.response.code).to.not.equal(400); });" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\"model\": \"openrouter/google/gemini-2.5-flash\", \"max_tokens\": 64, \"messages\": [{\"role\": \"system\", \"content\": [{\"type\": \"text\", \"text\": \"You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant.\", \"cache_control\": {\"type\": \"ephemeral\"}}]}, {\"role\": \"user\", \"content\": \"Reply with one word.\"}]}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "OpenRouter Google Gemini Pro: cache_control on system block", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "if (pm.response.code < 400) { pm.test('cache_control forwarded: response ok + usage present', function () { var j = pm.response.json(); pm.expect(j.usage || {}).to.be.an('object'); }); } pm.test('cache_control not rejected (no 400)', function () { pm.expect(pm.response.code).to.not.equal(400); });" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\"model\": \"openrouter/google/gemini-2.5-pro\", \"max_tokens\": 64, \"messages\": [{\"role\": \"system\", \"content\": [{\"type\": \"text\", \"text\": \"You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant.\", \"cache_control\": {\"type\": \"ephemeral\"}}]}, {\"role\": \"user\", \"content\": \"Reply with one word.\"}]}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "OpenRouter OpenAI GPT-4o: cache_control ignored (no 400)", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "if (pm.response.code < 400) { pm.test('cache_control forwarded: response ok + usage present', function () { var j = pm.response.json(); pm.expect(j.usage || {}).to.be.an('object'); }); } pm.test('cache_control not rejected (no 400)', function () { pm.expect(pm.response.code).to.not.equal(400); });" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\"model\": \"openrouter/openai/gpt-4o\", \"max_tokens\": 64, \"messages\": [{\"role\": \"system\", \"content\": [{\"type\": \"text\", \"text\": \"You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant.\", \"cache_control\": {\"type\": \"ephemeral\"}}]}, {\"role\": \"user\", \"content\": \"Reply with one word.\"}]}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "OpenRouter DeepSeek: cache_control ignored (no 400)", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "if (pm.response.code < 400) { pm.test('cache_control forwarded: response ok + usage present', function () { var j = pm.response.json(); pm.expect(j.usage || {}).to.be.an('object'); }); } pm.test('cache_control not rejected (no 400)', function () { pm.expect(pm.response.code).to.not.equal(400); });" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\"model\": \"openrouter/deepseek/deepseek-chat\", \"max_tokens\": 64, \"messages\": [{\"role\": \"system\", \"content\": [{\"type\": \"text\", \"text\": \"You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant.\", \"cache_control\": {\"type\": \"ephemeral\"}}]}, {\"role\": \"user\", \"content\": \"Reply with one word.\"}]}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "OpenRouter Qwen: cache_control on system block", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "if (pm.response.code < 400) { pm.test('cache_control forwarded: response ok + usage present', function () { var j = pm.response.json(); pm.expect(j.usage || {}).to.be.an('object'); }); } pm.test('cache_control not rejected (no 400)', function () { pm.expect(pm.response.code).to.not.equal(400); });" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\"model\": \"openrouter/qwen/qwen-2.5-72b-instruct\", \"max_tokens\": 64, \"messages\": [{\"role\": \"system\", \"content\": [{\"type\": \"text\", \"text\": \"You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant.\", \"cache_control\": {\"type\": \"ephemeral\"}}]}, {\"role\": \"user\", \"content\": \"Reply with one word.\"}]}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + } + ] } ] } diff --git a/tests/e2e/api/runners/build-model-catalog-wiring.mjs b/tests/e2e/api/runners/build-model-catalog-wiring.mjs new file mode 100644 index 0000000000..e9efae66ae --- /dev/null +++ b/tests/e2e/api/runners/build-model-catalog-wiring.mjs @@ -0,0 +1,507 @@ +#!/usr/bin/env node +// Generate the model-catalog wiring Postman collection. +// +// Each scenario stands up an isolated, run-namespaced custom provider backed by +// a real upstream (OpenAI, Anthropic, or Gemini), drives provider/key mutations +// through the management API, and asserts the catalog read endpoints reflect +// each mutation. The live-model cache is populated asynchronously by the key +// hooks, so every post-mutation read polls with exponential backoff. Output is +// machine-generated — edit this script and re-run it; do not hand-edit the JSON. +// +// node build-model-catalog-wiring.mjs [--out path.json] + +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + url, + events, + request, + item, + folderPrerequest, + pollPrerequest, + pollTest, + mutationTest, + cleanupTest, + buildCollection, + writeCollection, + resolveOutPath, +} from "./lib/collection-builder.mjs"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const DEFAULT_OUT = join(HERE, "..", "collections", "bifrost-model-catalog-wiring.postman_collection.json"); + +// Each entry generates the full six-scenario suite against a custom provider +// whose base_provider_type is `type`. Key values use Bifrost's `env.` +// resolution: the credential is read from the Bifrost process env at request +// time, so the collection carries no secret and the runner injects nothing. +// base_url is omitted — the base provider type's default applies. +const PROVIDERS = [ + { type: "openai", keyEnv: "OPENAI_API_KEY", modelA: "gpt-4o", modelB: "gpt-4o-mini", inferenceModel: "gpt-4o-mini" }, + { type: "anthropic", keyEnv: "ANTHROPIC_API_KEY", modelA: "claude-sonnet-4-5", modelB: "claude-haiku-4-5", inferenceModel: "claude-haiku-4-5" }, + { type: "gemini", keyEnv: "GEMINI_API_KEY", modelA: "gemini-2.5-flash", modelB: "gemini-2.5-flash-lite", inferenceModel: "gemini-2.5-flash-lite" }, +]; + +// --------------------------------------------------------------------------- // +// Scenario spec helpers +// --------------------------------------------------------------------------- // + +// A key carried through add/update. Updates resend the full intended state +// (value, models, enabled, aliases) because a key PUT is a full-field replace +// and upstreams reject an empty credential value. +function key({ id, models = [], blacklisted = [], enabled = true, aliases = {} }) { + return { id, models, blacklisted, enabled, aliases }; +} + +// Key names and ids must be unique across all providers, so namespace by +// scenario (which embeds the provider type) + key id + run id. Two same-named +// keys (e.g. an empty name) collide otherwise. +function keyBody(k, sid, name, keyEnv) { + const out = { + id: keyId(sid, k.id), + name, + value: `env.${keyEnv}`, + models: [...k.models], + enabled: k.enabled, + }; + if (k.blacklisted.length) out.blacklisted_models = [...k.blacklisted]; + if (Object.keys(k.aliases).length) out.aliases = { ...k.aliases }; + return out; +} + +const keyId = (sid, kid) => `${sid}-${kid}-{{run_id}}`; +// Updates resend this same name: a PUT with no name clears it to "", and empty +// names collide with each other across providers (names are globally unique). +const keyName = (sid, kid) => `catwiring-mc-${sid}-${kid}-{{run_id}}`; +const providerSeg = (sid) => `catwiring-${sid}-{{run_id}}`; +const jsProviderName = (sid) => `'catwiring-${sid}-' + pm.variables.get('run_id')`; + +// --------------------------------------------------------------------------- // +// Assertion line builders +// --------------------------------------------------------------------------- // + +function listModelsAssertLines(sid, { subset = [], superset = [], absent = [], absentVars = [], empty = false, nonEmpty = false }) { + return [ + `var providerName = ${jsProviderName(sid)};`, + `var expectSubset = ${JSON.stringify(subset)};`, + `var expectSuperset = ${JSON.stringify(superset)};`, + `var expectAbsent = ${JSON.stringify(absent)};`, + `var expectAbsentVars = ${JSON.stringify(absentVars)};`, + `var expectEmpty = ${empty ? "true" : "false"};`, + `var expectNonEmpty = ${nonEmpty ? "true" : "false"};`, + "if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + "var body = pm.response.json();", + "var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + "if (expectEmpty && names.length !== 0) { throw new Error('expected no models for ' + providerName + ' but got ' + JSON.stringify(names)); }", + "if (expectNonEmpty && names.length === 0) { throw new Error('expected a non-empty catalog for ' + providerName); }", + "expectSubset.concat(expectSuperset).forEach(function (m) {", + " if (names.indexOf(m) < 0) { throw new Error('expected model ' + m + ' in ' + JSON.stringify(names)); }", + "});", + "expectAbsentVars.forEach(function (v) {", + " var captured = pm.variables.get(v);", + " if (!captured) { throw new Error('captured variable ' + v + ' is empty'); }", + " expectAbsent = expectAbsent.concat([captured]);", + "});", + "expectAbsent.forEach(function (m) {", + " if (names.indexOf(m) >= 0) { throw new Error('expected model ' + m + ' absent but found in ' + JSON.stringify(names)); }", + "});", + ]; +} + +// Capture the first live model whose name starts with `prefix` into a +// collection variable, so later steps can mutate and assert against whatever +// the upstream actually reported (some upstreams only list dated ids, e.g. +// claude-sonnet-4-5-20250929, so static names can't be relied on here). +function captureModelAssertLines(sid, prefix, varName) { + return [ + `var providerName = ${jsProviderName(sid)};`, + `var prefix = ${JSON.stringify(prefix)};`, + "if (pm.response.code !== 200) { throw new Error('list models status ' + pm.response.code); }", + "var body = pm.response.json();", + "var names = (body.models || []).filter(function (m) { return m.provider === providerName; })", + " .map(function (m) { return m.name; });", + "var hit = null;", + "for (var i = 0; i < names.length; i++) {", + " if (names[i].indexOf(prefix) === 0) { hit = names[i]; break; }", + "}", + "if (!hit) { throw new Error('no live model with prefix ' + prefix + ' in ' + JSON.stringify(names.slice(0, 20))); }", + `pm.collectionVariables.set(${JSON.stringify(varName)}, hit);`, + ]; +} + +function providersAssertLines(sid) { + return [ + `var providerName = ${jsProviderName(sid)};`, + "if (pm.response.code !== 200) { throw new Error('list providers status ' + pm.response.code); }", + "var body = pm.response.json();", + "var names = (body.providers || []).map(function (p) { return p.name; });", + "if (names.indexOf(providerName) < 0) { throw new Error('expected provider ' + providerName + ' in providers list'); }", + ]; +} + +function baseModelsAssertLines(target) { + return [ + `var target = ${JSON.stringify(target)};`, + "if (pm.response.code !== 200) { throw new Error('list base models status ' + pm.response.code); }", + "var body = pm.response.json();", + "var names = body.models || [];", + "if (names.indexOf(target) < 0) { throw new Error('expected base model ' + target + ' in ' + JSON.stringify(names)); }", + ]; +} + +function inferenceAssertLines(resolvedModel) { + return [ + "if (pm.response.code !== 200) { throw new Error('inference status ' + pm.response.code + ' body ' + pm.response.text()); }", + "var body = pm.response.json();", + "if (!body.choices || body.choices.length === 0) { throw new Error('inference returned no choices'); }", + "var used = body.model || '';", + `if (used.indexOf(${JSON.stringify(resolvedModel)}) < 0) { throw new Error('expected resolved model ${resolvedModel} in response.model=' + used); }`, + ]; +} + +// --------------------------------------------------------------------------- // +// Step expansion +// --------------------------------------------------------------------------- // + +function expandScenario(sc) { + const sid = sc.id; + const provider = sc.provider; + const seg = providerSeg(sid); + const cleanupName = `cleanup: delete provider [${sid}]`; + const items = []; + let counter = 0; + let ordinal = 0; + const nextId = (tag) => `catwiring-${sid}-${String(++counter).padStart(2, "0")}-${tag}`; + const uniq = (label) => `${String(++ordinal).padStart(2, "0")}. ${label} [${sid}]`; + + const addKey = (k) => { + const name = uniq("add key " + k.id); + return item( + nextId("add-key"), + name, + request("POST", url(["api", "providers", seg, "keys"]), keyBody(k, sid, keyName(sid, k.id), provider.keyEnv)), + events(null, mutationTest(name, [200, 201], cleanupName)) + ); + }; + + for (const step of sc.steps) { + switch (step.type) { + case "addProvider": { + const customProviderConfig = { base_provider_type: provider.type, is_key_less: !!step.keyless }; + if (step.allowedRequests) customProviderConfig.allowed_requests = step.allowedRequests; + const body = { provider: seg, custom_provider_config: customProviderConfig }; + if (step.baseUrl) body.network_config = { base_url: step.baseUrl }; + const name = uniq("add provider"); + items.push(item(nextId("add-provider"), name, request("POST", url(["api", "providers"]), body), + events(null, mutationTest(name, [200, 201], cleanupName)))); + for (const k of step.keys || []) items.push(addKey(k)); + break; + } + case "addKey": + items.push(addKey(step.key)); + break; + case "updateKey": { + const name = uniq("update key " + step.key.id); + items.push(item(nextId("update-key"), name, + request("PUT", url(["api", "providers", seg, "keys", keyId(sid, step.key.id)]), keyBody(step.key, sid, keyName(sid, step.key.id), provider.keyEnv)), + events(null, mutationTest(name, [200], cleanupName)))); + break; + } + case "deleteKey": { + const name = uniq("delete key " + step.id); + items.push(item(nextId("delete-key"), name, + request("DELETE", url(["api", "providers", seg, "keys", keyId(sid, step.id)]), null), + events(null, mutationTest(name, [200, 204], cleanupName)))); + break; + } + case "deleteProvider": { + const name = uniq("delete provider"); + items.push(item(nextId("delete-provider"), name, + request("DELETE", url(["api", "providers", seg]), null), + events(null, mutationTest(name, [200, 204], cleanupName)))); + break; + } + case "assertModels": { + const name = uniq(step.label); + const query = [{ key: "provider", value: seg }, { key: "limit", value: "1000" }]; + items.push(item(nextId("assert-models"), name, request("GET", url(["api", "models"], query), null), + events(pollPrerequest(step.waitSeconds), pollTest(name, listModelsAssertLines(sid, step), cleanupName)))); + break; + } + case "captureModel": { + const name = uniq(step.label); + const query = [{ key: "provider", value: seg }, { key: "limit", value: "2000" }]; + items.push(item(nextId("capture-model"), name, request("GET", url(["api", "models"], query), null), + events(pollPrerequest(step.waitSeconds), pollTest(name, captureModelAssertLines(sid, step.prefix, step.varName), cleanupName)))); + break; + } + case "assertModelDetails": { + const name = uniq(step.label); + const query = [{ key: "provider", value: seg }, { key: "limit", value: "1000" }]; + items.push(item(nextId("assert-model-details"), name, request("GET", url(["api", "models", "details"], query), null), + events(pollPrerequest(step.waitSeconds), pollTest(name, listModelsAssertLines(sid, step), cleanupName)))); + break; + } + case "assertProviders": { + const name = uniq(step.label); + items.push(item(nextId("assert-providers"), name, request("GET", url(["api", "providers"]), null), + events(pollPrerequest(step.waitSeconds), pollTest(name, providersAssertLines(sid), cleanupName)))); + break; + } + case "assertBaseModels": { + const name = uniq(step.label); + const query = [{ key: "query", value: step.model }, { key: "limit", value: "1000" }]; + items.push(item(nextId("assert-base-models"), name, request("GET", url(["api", "models", "base"], query), null), + events(pollPrerequest(step.waitSeconds), pollTest(name, baseModelsAssertLines(step.model), cleanupName)))); + break; + } + case "assertInference": { + const name = uniq(step.label); + const body = { + model: `${seg}/${step.requestModel}`, + messages: [{ role: "user", content: "Reply with the single word: ok." }], + max_tokens: 5, + }; + items.push(item(nextId("assert-inference"), name, request("POST", url(["v1", "chat", "completions"]), body), + events(pollPrerequest(step.waitSeconds), pollTest(name, inferenceAssertLines(step.expectResolved), cleanupName)))); + break; + } + case "cleanup": + break; // teardown folder is always appended + default: + throw new Error("unknown step type: " + step.type); + } + } + + const cleanupItem = item( + `catwiring-${sid}-cleanup`, + cleanupName, + request("DELETE", url(["api", "providers", seg]), null), + events(null, cleanupTest(cleanupName)) + ); + + return { + name: sc.title, + description: sc.description, + item: [...items, { name: "Cleanup", item: [cleanupItem] }], + event: events(folderPrerequest([]), null), + }; +} + +// --------------------------------------------------------------------------- // +// Scenarios — six canonical wiring contracts, generated per provider +// --------------------------------------------------------------------------- // + +function scenariosFor(provider) { + const p = provider.type; + const MODEL_A = provider.modelA; + const MODEL_B = provider.modelB; + const INFERENCE_MODEL = provider.inferenceModel; + const ALIAS = `catwiring-alias-${p}-{{run_id}}`; + // Collection variable holding the live model captured for the blacklist + // re-gate scenario; unique per provider so suites never cross-read. + const BL_VAR = `bl_target_${p}`; + + return [ + { + id: `${p}-add-provider-and-key`, + title: `Add provider and key surfaces models (${p})`, + description: "A fresh provider plus one gated key surfaces that key's allowed model in the catalog.", + steps: [ + { type: "addProvider" }, + { type: "addKey", key: key({ id: "k1", models: [MODEL_A] }) }, + { type: "assertModels", subset: [MODEL_A], waitSeconds: 2, label: "key model appears in catalog" }, + { type: "cleanup" }, + ], + }, + { + id: `${p}-update-key-models`, + title: `Updating key model set re-gates the catalog (${p})`, + description: "Changing a key's allow-list invalidates the stale live entry and re-gates the surfaced models.", + steps: [ + { type: "addProvider", keys: [key({ id: "k1", models: [MODEL_A] })] }, + { type: "assertModels", subset: [MODEL_A], waitSeconds: 2, label: "initial model gated in" }, + { type: "updateKey", key: key({ id: "k1", models: [MODEL_B] }) }, + { type: "assertModels", subset: [MODEL_B], absent: [MODEL_A], waitSeconds: 2, label: "new model gated in, old gated out" }, + { type: "cleanup" }, + ], + }, + { + id: `${p}-disable-reenable-key`, + title: `Disabled key drops models, re-enable restores (${p})`, + description: "A disabled key is skipped during aggregation; re-enabling it brings its models back.", + steps: [ + { type: "addProvider", keys: [key({ id: "k1", models: [MODEL_A] })] }, + { type: "assertModels", subset: [MODEL_A], waitSeconds: 2, label: "model present while enabled" }, + { type: "updateKey", key: key({ id: "k1", models: [MODEL_A], enabled: false }) }, + { type: "assertModels", absent: [MODEL_A], waitSeconds: 2, label: "model gone while disabled" }, + { type: "updateKey", key: key({ id: "k1", models: [MODEL_A], enabled: true }) }, + { type: "assertModels", subset: [MODEL_A], waitSeconds: 2, label: "model returns when re-enabled" }, + { type: "cleanup" }, + ], + }, + { + id: `${p}-delete-key-sibling-survives`, + title: `Deleting one key leaves sibling models intact (${p})`, + description: "With two keys gating distinct models, deleting one removes only its models; the sibling's survive.", + steps: [ + { type: "addProvider", keys: [key({ id: "k1", models: [MODEL_A] }), key({ id: "k2", models: [MODEL_B] })] }, + { type: "assertModels", superset: [MODEL_A, MODEL_B], waitSeconds: 2, label: "both keys' models present" }, + { type: "deleteKey", id: "k1" }, + { type: "assertModels", subset: [MODEL_B], absent: [MODEL_A], waitSeconds: 2, label: "sibling model survives delete" }, + { type: "cleanup" }, + ], + }, + { + id: `${p}-delete-provider`, + title: `Deleting provider removes it from the catalog (${p})`, + description: "Removing the provider drops all of its models from the catalog read endpoints.", + steps: [ + { type: "addProvider", keys: [key({ id: "k1", models: [MODEL_A] })] }, + { type: "assertModels", subset: [MODEL_A], waitSeconds: 2, label: "model present before delete" }, + { type: "deleteProvider" }, + { type: "assertModels", empty: true, waitSeconds: 1, label: "no models after provider delete" }, + { type: "cleanup" }, + ], + }, + { + id: `${p}-alias-resolution`, + title: `Alias resolves to underlying model at inference (${p})`, + description: "A key alias routes an inference request to the underlying wire model rather than being rejected.", + steps: [ + { type: "addProvider", keys: [key({ id: "k1", models: [ALIAS, INFERENCE_MODEL], aliases: { [ALIAS]: INFERENCE_MODEL } })] }, + { type: "assertModels", subset: [INFERENCE_MODEL], waitSeconds: 2, label: "aliased key model present" }, + { type: "assertInference", requestModel: ALIAS, expectResolved: INFERENCE_MODEL, waitSeconds: 2, label: "alias resolves to underlying model" }, + { type: "cleanup" }, + ], + }, + { + id: `${p}-blacklist-regate`, + title: `Blacklisting a model re-gates a wildcard catalog (${p})`, + description: + "A wildcard key surfaces the upstream's live model list; adding one of those models to the key's " + + "blacklist drops it from the catalog while the rest of the list stays. The target model is captured " + + "from the live list at run time because some upstreams only report dated ids.", + steps: [ + { type: "addProvider", keys: [key({ id: "k1", models: ["*"] })] }, + { type: "captureModel", prefix: MODEL_A, varName: BL_VAR, waitSeconds: 2, label: "wildcard catalog serves live models" }, + { type: "updateKey", key: key({ id: "k1", models: ["*"], blacklisted: [`{{${BL_VAR}}}`] }) }, + { type: "assertModels", absentVars: [BL_VAR], nonEmpty: true, waitSeconds: 2, label: "blacklisted model gated out, catalog still populated" }, + { type: "cleanup" }, + ], + }, + { + id: `${p}-multi-key-union`, + title: `Catalog aggregates the union across enabled keys (${p})`, + description: "Two keys gating distinct models both contribute: the catalog lists the union of their allow-lists.", + steps: [ + { type: "addProvider", keys: [key({ id: "k1", models: [MODEL_A] }), key({ id: "k2", models: [MODEL_B] })] }, + { type: "assertModels", superset: [MODEL_A, MODEL_B], waitSeconds: 2, label: "catalog lists union of both keys' models" }, + { type: "cleanup" }, + ], + }, + { + id: `${p}-model-details-gating`, + title: `Model details endpoint respects the key gate (${p})`, + description: "/api/models/details lists only the models the provider's keys allow, like the plain list endpoint.", + steps: [ + { type: "addProvider", keys: [key({ id: "k1", models: [MODEL_A] })] }, + { type: "assertModelDetails", subset: [MODEL_A], absent: [MODEL_B], waitSeconds: 2, label: "details list gated to key models" }, + { type: "cleanup" }, + ], + }, + { + id: `${p}-base-models-stable`, + title: `Base model list is unaffected by key changes (${p})`, + description: + "/api/models/base reflects the datasheet's distinct base names; removing a model from a key's " + + "allow-list must not remove its base name from that list.", + steps: [ + { type: "addProvider", keys: [key({ id: "k1", models: [MODEL_A] })] }, + { type: "assertBaseModels", model: MODEL_A, waitSeconds: 1, label: "base name listed while key allows it" }, + { type: "updateKey", key: key({ id: "k1", models: [MODEL_B] }) }, + { type: "assertBaseModels", model: MODEL_A, waitSeconds: 1, label: "base name still listed after key change" }, + { type: "cleanup" }, + ], + }, + ].map((sc) => ({ ...sc, provider })); +} + +// Scenarios whose wiring is base-type independent run once instead of per +// provider; they all use the first provider entry as an arbitrary base type. +const GLOBAL_SCENARIOS = [ + // A keyless custom provider is gate-unrestricted at request time but + // contributes no rows to the catalog read: there is no live discovery + // without keys, and datasheet rows are keyed by standard provider names only. + { + id: "keyless-provider", + title: "Keyless provider is listed but contributes no catalog rows", + description: + "A custom provider created with is_key_less=true appears in the providers list; its catalog read " + + "succeeds and is empty (no live discovery without keys, no datasheet rows under the custom name).", + provider: PROVIDERS[0], + steps: [ + { type: "addProvider", keyless: true }, + { type: "assertProviders", waitSeconds: 1, label: "keyless provider appears in providers list" }, + { type: "assertModels", empty: true, waitSeconds: 1, label: "keyless provider has no catalog rows" }, + { type: "cleanup" }, + ], + }, + // Discovery failure must degrade, not blank: when the upstream's list-models + // endpoint is unreachable, the catalog still serves the models aggregated + // from the keys' explicit allow-lists. Port 9 (discard) is never listening, + // so the failure is deterministic and no request leaves the host. + { + id: "list-models-failing", + title: "Unreachable list-models upstream keeps key allow-list models", + description: + "A provider whose base_url points at a dead port cannot complete live model discovery; the catalog " + + "still surfaces the models explicitly allowed by its keys.", + provider: PROVIDERS[0], + steps: [ + { type: "addProvider", baseUrl: "http://127.0.0.1:9", keys: [key({ id: "k1", models: [PROVIDERS[0].modelA] })] }, + { type: "assertModels", subset: [PROVIDERS[0].modelA], waitSeconds: 2, label: "key allow-list survives discovery failure" }, + { type: "cleanup" }, + ], + }, + // With list_models excluded from allowed_requests, discovery never runs: a + // wildcard key has no live list to expand against and surfaces nothing, + // while an explicit allow-list still surfaces through the key aggregates. + // The wait before the empty read is deliberately longer than discovery + // normally takes, so a wrongly-attempted discovery would be caught. + { + id: "list-models-disabled", + title: "Disallowed list-models blocks discovery but not explicit allow-lists", + description: + "A provider whose allowed_requests excludes list_models performs no live discovery: a wildcard key " + + "surfaces no catalog rows, while a key with an explicit allow-list still surfaces its models.", + provider: PROVIDERS[0], + steps: [ + { + type: "addProvider", + allowedRequests: { chat_completion: true, chat_completion_stream: true }, + keys: [key({ id: "k1", models: ["*"] })], + }, + { type: "assertModels", empty: true, waitSeconds: 4, label: "wildcard key surfaces nothing without discovery" }, + { type: "addKey", key: key({ id: "k2", models: [PROVIDERS[0].modelB] }) }, + { type: "assertModels", subset: [PROVIDERS[0].modelB], waitSeconds: 2, label: "explicit allow-list still surfaces" }, + { type: "cleanup" }, + ], + }, +]; + +const SCENARIOS = [...PROVIDERS.flatMap(scenariosFor), ...GLOBAL_SCENARIOS]; + +const collection = buildCollection({ + id: "bifrost-model-catalog-wiring", + name: "Bifrost Model Catalog Wiring", + description: + "End-to-end wiring tests between the management API and the model catalog. Each scenario stands up " + + "an isolated, run-namespaced custom provider backed by a real upstream (OpenAI, Anthropic, or Gemini), " + + "mutates its providers/keys, and asserts the catalog read endpoints reflect each mutation. Reads that " + + "depend on the asynchronously populated live-model cache poll with exponential backoff. " + + "Machine-generated by runners/build-model-catalog-wiring.mjs — do not hand-edit.", + expandedScenarios: SCENARIOS.map(expandScenario), +}); + +writeCollection(resolveOutPath(DEFAULT_OUT), collection, SCENARIOS.map((s) => s.id)); diff --git a/tests/e2e/api/runners/build-routing-wiring.mjs b/tests/e2e/api/runners/build-routing-wiring.mjs new file mode 100644 index 0000000000..8bee258490 --- /dev/null +++ b/tests/e2e/api/runners/build-routing-wiring.mjs @@ -0,0 +1,1005 @@ +#!/usr/bin/env node +// Generate the governance × model-catalog routing Postman collection. +// +// Each scenario stands up an isolated custom provider (backed by real OpenAI), +// adds gated keys, optionally creates a virtual key (VK) with provider/model/key +// restrictions, then drives inference and asserts the route taken. Two gates +// compose, evaluated in order: +// 1. governance — VK provider allowlist, allowed/blacklisted models, key +// restriction. Rejects early (403/402/429). +// 2. catalog/key — per-key models/blacklist/aliases. Rejects at key selection +// with 400 "no keys found for provider:

and model: ". +// +// Verification: +// * sync — on success assert extra_fields.routing_info {provider, model, key, +// resolved_key_alias}; on rejection assert the error message (routing_info is +// empty on errors). +// * async — the stored log (polled), correlated by virtual_key_ids/providers: +// status, selected_key_name, virtual_key presence. routing_engine_logs is +// null for plain governance allow/deny, so it is NOT asserted here. +// +// VK key restriction is set via key_ids (["*"] = all keys); allow_all_keys/keys +// are ignored on input. Output is machine-generated — edit this script and re-run. +// +// node build-routing-wiring.mjs [--out path.json] + +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + url, + events, + request, + item, + folderPrerequest, + pollPrerequest, + pollTest, + mutationTest, + cleanupTest, + buildCollection, + writeCollection, + resolveOutPath, +} from "./lib/collection-builder.mjs"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const DEFAULT_OUT = join(HERE, "..", "collections", "bifrost-routing-wiring.postman_collection.json"); + +const NAME_PREFIX = "catwiring-rt-"; + +// Provider catalog. Key values use Bifrost's `env.` resolution (credential +// read from the Bifrost process env at request time — no secret in the collection, +// nothing injected by the runner). base_url is omitted; the base type's default +// applies. Models are current, cheap, and asserted only as substrings/membership. +const PROVIDERS = { + openai: { base: "openai", env: "OPENAI_API_KEY", model: "gpt-4o-mini", altModel: "gpt-4o" }, + anthropic: { base: "anthropic", env: "ANTHROPIC_API_KEY", model: "claude-haiku-4-5", altModel: "claude-sonnet-4-5" }, + gemini: { base: "gemini", env: "GEMINI_API_KEY", model: "gemini-2.5-flash", altModel: "gemini-2.5-flash-lite" }, + // Vertex is config-heavy and standard-only (no custom_provider_config). Its key + // carries a vertex_key_config (project/region/credentials via env.) instead of a + // value. region "global" serves the cross-region Claude models. + vertex: { + base: "vertex", + standardOnly: true, + model: "claude-sonnet-4-5", + keyConfig: { + vertex_key_config: { + project_id: "env.VERTEX_PROJECT_ID", + project_number: "env.VERTEX_PROJECT_NUMBER", + region: "global", + auth_credentials: "env.VERTEX_CREDENTIALS", + }, + }, + }, + // Azure: standard-only; key carries the api key (value) plus azure_key_config. + // Deployments are named after the model, so gpt-4o-mini routes directly. + azure: { + base: "azure", + standardOnly: true, + model: "gpt-4o-mini", + keyConfig: { value: "env.AZURE_API_KEY", azure_key_config: { endpoint: "env.AZURE_ENDPOINT" } }, + }, + // Bedrock: standard-only; bedrock_key_config (IAM creds + region). Claude needs + // the cross-region inference-profile id ("us.anthropic.…"). + bedrock: { + base: "bedrock", + standardOnly: true, + // Routed via a key alias so Bedrock exposes the common Claude name; the alias + // resolves to the cross-region inference-profile id Bedrock requires. + model: "claude-sonnet-4-5", + aliasFor: { "claude-sonnet-4-5": "us.anthropic.claude-sonnet-4-5-20250929-v1:0" }, + keyConfig: { + bedrock_key_config: { access_key: "env.AWS_ACCESS_KEY_ID", secret_key: "env.AWS_SECRET_ACCESS_KEY", region: "env.AWS_REGION" }, + }, + }, +}; +const DEFAULT_PROVIDER = "openai"; + +// Back-compat aliases for the openai-based scenarios. +const MODEL_A = PROVIDERS.openai.altModel; // gpt-4o +const MODEL_B = PROVIDERS.openai.model; // gpt-4o-mini + +// --------------------------------------------------------------------------- // +// Spec helpers +// --------------------------------------------------------------------------- // + +function key({ id, models = [], blacklisted = [], enabled = true, aliases = {}, weight, badKey = false } = {}) { + return { id, models, blacklisted, enabled, aliases, weight, badKey }; +} + +// Key names must be unique across all providers, so namespace by scenario + key +// id + run id. Assertions reference a key by its id and recompute this name. +// prov is the PROVIDERS entry: a value=env. key for simple providers, or a +// provider-specific key config (e.g. vertex_key_config) when prov.keyConfig is set. +function keyBody(k, name, prov) { + const out = { id: keyId(k.id), name, models: [...k.models], enabled: k.enabled }; + if (k.badKey) { + // A deliberately invalid credential so the provider attempt fails upstream + // (e.g. a 401), exercising the fallback path. The provider itself is real + // and capable of the model — only the credential is broken. + out.value = "sk-deadbeef-invalid-000"; + } else if (prov.keyConfig) Object.assign(out, prov.keyConfig); + else out.value = `env.${prov.env}`; + if (k.blacklisted.length) out.blacklisted_models = [...k.blacklisted]; + // prov.aliasFor lets a provider expose a common model name (e.g. Bedrock maps + // claude-sonnet-4-5 → its inference-profile id). Scenario aliases win on conflict. + const aliases = { ...(prov.aliasFor || {}), ...k.aliases }; + if (Object.keys(aliases).length) out.aliases = aliases; + if (k.weight != null) out.weight = k.weight; + return out; +} + +const keyNameSeg = (sid, kid) => `catwiring-rt-${sid}-${kid}-{{run_id}}`; +const jsKeyName = (sid, kid) => `'catwiring-rt-${sid}-${kid}-' + pm.variables.get('run_id')`; +// JS expression for a provider's run-scoped name (ref "self" = primary provider). +const jsSegFor = (sid, ref) => + !ref || ref === "self" + ? `'${NAME_PREFIX}${sid}-' + pm.variables.get('run_id')` + : `'${NAME_PREFIX}${sid}-${ref}-' + pm.variables.get('run_id')`; + +// A VK provider config. provider_ref 'self' targets this scenario's provider. +function vkProvider({ providerRef = "self", keyIds = ["*"], allowedModels = ["*"], blacklistedModels = [], weight = 1 } = {}) { + return { providerRef, keyIds, allowedModels, blacklistedModels, weight }; +} + +const keyId = (kid) => `${kid}-{{run_id}}`; +const providerSeg = (sid) => NAME_PREFIX + sid + "-{{run_id}}"; +const jsProviderName = (sid) => `'${NAME_PREFIX}${sid}-' + pm.variables.get('run_id')`; + +// --------------------------------------------------------------------------- // +// Assertion line builders +// --------------------------------------------------------------------------- // + +function routeAssertLines(sid, step, jsNameOf) { + if (step.expectStatus === 200) { + const lines = [ + "if (pm.response.code !== 200) { throw new Error('route status ' + pm.response.code + ' body ' + pm.response.text()); }", + "var body = pm.response.json();", + "if (!body.choices || body.choices.length === 0) { throw new Error('no choices'); }", + "var ri = (body.extra_fields || {}).routing_info || {};", + ]; + if (step.expectProviderOneOf) { + lines.push(`var allowedProviders = [${step.expectProviderOneOf.map((r) => jsNameOf(r)).join(", ")}];`); + lines.push("if (allowedProviders.indexOf(ri.provider) < 0) { throw new Error('routing_info.provider=' + ri.provider + ' not in ' + JSON.stringify(allowedProviders)); }"); + } else { + lines.push(`var providerName = ${jsNameOf(step.routeRef)};`); + lines.push("if (ri.provider !== providerName) { throw new Error('routing_info.provider=' + ri.provider + ' expected ' + providerName); }"); + } + if (step.expectKeyId != null) { + lines.push(`var expectedKey = ${jsKeyName(sid, step.expectKeyId)};`); + lines.push("if (ri.key !== expectedKey) { throw new Error('routing_info.key=' + ri.key + ' expected ' + expectedKey); }"); + } + if (step.expectResolvedModelId != null) { + lines.push( + `if (!ri.resolved_key_alias || ri.resolved_key_alias.model_id !== ${JSON.stringify(step.expectResolvedModelId)}) { throw new Error('resolved_key_alias=' + JSON.stringify(ri.resolved_key_alias)); }` + ); + } + if (step.expectIsFallback != null) { + lines.push(`if (Boolean(ri.is_fallback) !== ${JSON.stringify(step.expectIsFallback)}) { throw new Error('is_fallback=' + ri.is_fallback); }`); + } + if (step.expectPrimaryProviderRef != null) { + lines.push(`var expectedPrimary = ${jsNameOf(step.expectPrimaryProviderRef)};`); + lines.push("if (ri.primary_provider !== expectedPrimary) { throw new Error('primary_provider=' + ri.primary_provider + ' expected ' + expectedPrimary); }"); + } + if (step.expectPrimaryModel != null) { + lines.push(`if (ri.primary_model !== ${JSON.stringify(step.expectPrimaryModel)}) { throw new Error('primary_model=' + ri.primary_model); }`); + } + return lines; + } + const lines = [ + `if (pm.response.code !== ${step.expectStatus}) { throw new Error('expected status ${step.expectStatus} got ' + pm.response.code + ' body ' + pm.response.text()); }`, + "var body = pm.response.json();", + "var msg = (body.error && (body.error.message || body.error)) || body.message || '';", + ]; + if (step.expectErrorSubstr) { + lines.push(`if (String(msg).indexOf(${JSON.stringify(step.expectErrorSubstr)}) < 0) { throw new Error('error message=' + msg); }`); + } + return lines; +} + +function logAssertLines(sid, step, jsNameOf) { + const lines = [ + `var providerName = ${jsNameOf(step.providerRef)};`, + "if (pm.response.code !== 200) { throw new Error('logs status ' + pm.response.code); }", + "var logs = (pm.response.json() || {}).logs || [];", + `var model = ${JSON.stringify(step.model)};`, + "var row = null;", + "for (var i = 0; i < logs.length; i++) {", + " if (logs[i].provider === providerName && logs[i].model === model) { row = logs[i]; break; }", + "}", + "if (!row) { throw new Error('no log row for ' + providerName + '/' + model + ' in ' + logs.length + ' rows'); }", + `if (row.status !== ${JSON.stringify(step.expectStatus)}) { throw new Error('log status=' + row.status); }`, + ]; + if (step.expectSelectedKeyId != null) { + lines.push(`var expectedKey = ${jsKeyName(sid, step.expectSelectedKeyId)};`); + lines.push("if (row.selected_key_name !== expectedKey) { throw new Error('selected_key_name=' + row.selected_key_name + ' expected ' + expectedKey); }"); + } + if (step.expectVkPresent) { + lines.push("if (!row.virtual_key_id) { throw new Error('log row missing virtual_key_id'); }"); + } + return lines; +} + +// One sample of a distribution probe: assert 200 and record which key served into +// a scenario-scoped set. `reset` clears the set on the first sample. +function distSampleTest(testname, sid, cleanupName, reset) { + return [ + `var cleanupReq = ${JSON.stringify(cleanupName)};`, + ...(reset ? [`pm.collectionVariables.set('dist_${sid}', '');`] : []), + "if (pm.response.code === 200) {", + " var k = ((pm.response.json().extra_fields || {}).routing_info || {}).key || '';", + ` var cur = pm.collectionVariables.get('dist_${sid}') || '';`, + " var set = cur ? cur.split(',') : [];", + ` if (k && set.indexOf(k) < 0) { set.push(k); pm.collectionVariables.set('dist_${sid}', set.join(',')); }`, + "}", + `pm.test(${JSON.stringify(testname)}, function () {`, + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }", + ]; +} + +// One sample of a provider-distribution probe: assert 200 and record which +// provider served into a scenario-scoped set. +function distSampleProviderTest(testname, sid, cleanupName, reset) { + return [ + `var cleanupReq = ${JSON.stringify(cleanupName)};`, + ...(reset ? [`pm.collectionVariables.set('pdist_${sid}', '');`] : []), + "if (pm.response.code === 200) {", + " var p = ((pm.response.json().extra_fields || {}).routing_info || {}).provider || '';", + ` var cur = pm.collectionVariables.get('pdist_${sid}') || '';`, + " var set = cur ? cur.split(',') : [];", + ` if (p && set.indexOf(p) < 0) { set.push(p); pm.collectionVariables.set('pdist_${sid}', set.join(',')); }`, + "}", + `pm.test(${JSON.stringify(testname)}, function () {`, + " pm.expect(pm.response.code, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { pm.execution.setNextRequest(cleanupReq); }", + ]; +} + +// Assert every expected key was observed across the distribution samples. +// Assert the observed key set against the step's expectations: +// expectKeyIds — each of these keys must have served at least once +// expectOnly — every observed key must be one of these +// expectNever — none of these keys may have served +function distAssertLines(sid, step) { + const arr = (ids) => "[" + (ids || []).map((kid) => jsKeyName(sid, kid)).join(", ") + "]"; + const lines = [`var seen = (pm.collectionVariables.get('dist_${sid}') || '').split(',').filter(Boolean);`]; + if (step.expectKeyIds) { + lines.push(`${"var"} mustServe = ${arr(step.expectKeyIds)};`); + lines.push("mustServe.forEach(function (e) { if (seen.indexOf(e) < 0) throw new Error('key ' + e + ' never served; observed ' + JSON.stringify(seen)); });"); + } + if (step.expectOnly) { + lines.push(`var only = ${arr(step.expectOnly)};`); + lines.push("if (seen.length === 0) throw new Error('no key observed across samples');"); + lines.push("seen.forEach(function (e) { if (only.indexOf(e) < 0) throw new Error('key ' + e + ' served but not in expectOnly ' + JSON.stringify(only)); });"); + } + if (step.expectNever) { + lines.push(`var never = ${arr(step.expectNever)};`); + lines.push("seen.forEach(function (e) { if (never.indexOf(e) >= 0) throw new Error('key ' + e + ' served but was expectNever; observed ' + JSON.stringify(seen)); });"); + } + return lines; +} + +// Capture the created VK's value+id into scenario-scoped collection vars. +function captureVkTest(testname, sid, cleanupName) { + return [ + `var cleanupReq = ${JSON.stringify(cleanupName)};`, + "var ok = pm.response.code === 200 || pm.response.code === 201;", + `pm.test(${JSON.stringify(testname)}, function () {`, + " pm.expect([200, 201], 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (ok) {", + " var vk = (pm.response.json() || {}).virtual_key || {};", + ` pm.collectionVariables.set('vkval_${sid}', vk.value || '');`, + ` pm.collectionVariables.set('vkid_${sid}', vk.id || '');`, + "} else { pm.execution.setNextRequest(cleanupReq); }", + ]; +} + +// --------------------------------------------------------------------------- // +// Step expansion +// --------------------------------------------------------------------------- // + +function expandScenario(sc) { + const sid = sc.id; + const seg = providerSeg(sid); + const cleanupProvider = `cleanup: delete provider [${sid}]`; + const cleanupVk = `cleanup: delete vk [${sid}]`; + // A failing step jumps to the FIRST cleanup item so the whole teardown runs in + // order. When the scenario has a VK, that first item is the VK delete (the + // provider delete follows it); jumping straight to the provider delete would + // skip the VK and leak it. + const hasVkScenario = sc.steps.some((s) => s.type === "createVK"); + const cleanupTarget = hasVkScenario ? cleanupVk : cleanupProvider; + // A scenario may stand up more than one provider (e.g. governance LB across + // providers). ref "self" is the scenario's primary provider; any other ref + // gets its own run-scoped name. All created providers are torn down. + const segFor = (ref) => (!ref || ref === "self" ? seg : `${NAME_PREFIX}${sid}-${ref}-{{run_id}}`); + // Resolve each provider ref (kind-aware). A "custom" provider is run-namespaced + // (parallel-safe); a "standard" provider uses the fixed base-type name (e.g. + // "openai") — a GLOBAL singleton, so standard-provider scenarios are NOT + // run-id-isolated: they assume a clean instance and must not run in parallel + // shards touching the same standard provider. + const providerByRef = {}; + for (const s of sc.steps) { + if (s.type !== "addProvider") continue; + const ref = s.ref || "self"; + const pt = s.providerType || DEFAULT_PROVIDER; + const std = s.providerKind === "standard" || !!PROVIDERS[pt].standardOnly; + providerByRef[ref] = { + pt, + std, + env: PROVIDERS[pt].env, + base: PROVIDERS[pt].base, + name: std ? PROVIDERS[pt].base : segFor(ref), + jsName: std ? JSON.stringify(PROVIDERS[pt].base) : jsSegFor(sid, ref), + }; + } + const providerRefs = Object.keys(providerByRef).length ? Object.keys(providerByRef) : ["self"]; + const nameOf = (ref) => (providerByRef[ref || "self"] || {}).name || segFor(ref); + const jsNameOf = (ref) => (providerByRef[ref || "self"] || {}).jsName || jsSegFor(sid, ref); + const items = []; + let counter = 0; + let ordinal = 0; + let hasVk = false; + const nextId = (tag) => `rt-${sid}-${String(++counter).padStart(2, "0")}-${tag}`; + const uniq = (label) => `${String(++ordinal).padStart(2, "0")}. ${label} [${sid}]`; + + for (const step of sc.steps) { + switch (step.type) { + case "addProvider": { + const pseg = nameOf(step.ref); + const prov = PROVIDERS[step.providerType || DEFAULT_PROVIDER]; + const plabel = step.ref && step.ref !== "self" ? ` (${step.ref})` : ""; + // Standard providers (fixed base-type name) carry no custom_provider_config — + // the handler rejects a custom config on a standard provider name. + const isStandard = step.providerKind === "standard" || !!prov.standardOnly; + const body = { provider: pseg }; + if (!isStandard) { + body.custom_provider_config = { base_provider_type: prov.base, is_key_less: false }; + } + const name = uniq("add provider" + plabel); + items.push(item(nextId("add-provider"), name, request("POST", url(["api", "providers"]), body), + events(null, mutationTest(name, [200, 201], cleanupTarget)))); + for (const k of step.keys || []) { + const kname = uniq("add key " + k.id + plabel); + items.push(item(nextId("add-key"), kname, + request("POST", url(["api", "providers", pseg, "keys"]), keyBody(k, keyNameSeg(sid, k.id), prov)), + events(null, mutationTest(kname, [200, 201], cleanupTarget)))); + } + break; + } + case "createVK": { + hasVk = true; + const pcs = (step.providerConfigs || []).map((pc) => { + const cfg = { + provider: nameOf(pc.providerRef), + // "*" stays literal; a logical key id (e.g. "k1") is namespaced to the + // run-scoped id the key was created with. + key_ids: pc.keyIds.map((kid) => (kid === "*" ? "*" : keyId(kid))), + allowed_models: [...pc.allowedModels], + blacklisted_models: [...pc.blacklistedModels], + }; + // weight:null → omit the field entirely (a VK with no weighted configs + // is allow-list-only: governance gates providers but skips load balancing). + if (pc.weight !== null) cfg.weight = pc.weight ?? 1; + return cfg; + }); + const body = { name: `catwiring-rtvk-${sid}-{{run_id}}`, is_active: true, provider_configs: pcs }; + const name = uniq("create vk"); + items.push(item(nextId("create-vk"), name, request("POST", url(["api", "governance", "virtual-keys"]), body), + events(null, captureVkTest(name, sid, cleanupTarget)))); + break; + } + case "route": { + const name = uniq(step.label); + const headers = step.useVk === false ? [] : [{ key: "x-bf-vk", value: `{{vkval_${sid}}}` }]; + const body = { + // bareModel routes without a provider prefix so an upstream routing + // layer (governance LB) resolves the provider. routeRef targets a + // specific provider in a multi-provider scenario (default: self). + model: step.bareModel ? step.model : `${nameOf(step.routeRef)}/${step.model}`, + messages: [{ role: "user", content: "Reply with the single word: ok." }], + max_tokens: 5, + }; + // Request-level fallbacks. The /v1 OpenAI-compatible endpoint takes the + // string "provider/model" form (not the {provider,model} object form). + if (step.fallbacks) { + body.fallbacks = step.fallbacks.map((f) => `${nameOf(f.providerRef)}/${f.model}`); + } + items.push(item(nextId("route"), name, request("POST", url(["v1", "chat", "completions"]), body, headers), + events(pollPrerequest(step.waitSeconds), pollTest(name, routeAssertLines(sid, step, jsNameOf), cleanupTarget)))); + break; + } + case "assertLog": { + const name = uniq(step.label); + const query = step.byVk === false + ? [{ key: "providers", value: seg }, { key: "limit", value: "10" }] + : [{ key: "virtual_key_ids", value: `{{vkid_${sid}}}` }, { key: "limit", value: "10" }]; + items.push(item(nextId("assert-log"), name, request("GET", url(["api", "logs"], query), null), + events(pollPrerequest(step.waitSeconds), pollTest(name, logAssertLines(sid, step, jsNameOf), cleanupTarget)))); + break; + } + case "keyDistribution": { + const n = step.n || 8; + for (let i = 0; i < n; i++) { + const sname = uniq(`sample ${i + 1}/${n} (${step.model})`); + const body = { + model: `${seg}/${step.model}`, + messages: [{ role: "user", content: "Reply with the single word: ok." }], + max_tokens: 5, + }; + items.push(item(nextId("dist-sample"), sname, request("POST", url(["v1", "chat", "completions"]), body), + events(null, distSampleTest(sname, sid, cleanupTarget, i === 0)))); + } + const aname = uniq(step.label); + const assertExec = [ + `var cleanupReq = ${JSON.stringify(cleanupTarget)};`, + "var ok = true, errMsg = '';", + "try {", + ...distAssertLines(sid, step).map((l) => " " + l), + "} catch (e) { ok = false; errMsg = e.message; }", + `pm.test(${JSON.stringify(aname)}, function () { if (!ok) throw new Error(errMsg); });`, + "if (!ok) { pm.execution.setNextRequest(cleanupReq); }", + ]; + items.push(item(nextId("dist-assert"), aname, request("GET", url(["health"]), null), + events(null, assertExec))); + break; + } + case "providerDistribution": { + const n = step.n || 8; + for (let i = 0; i < n; i++) { + const sname = uniq(`sample ${i + 1}/${n} (${step.model})`); + const headers = step.useVk === false ? [] : [{ key: "x-bf-vk", value: `{{vkval_${sid}}}` }]; + const body = { + model: step.bareModel ? step.model : `${nameOf(step.routeRef)}/${step.model}`, + messages: [{ role: "user", content: "Reply with the single word: ok." }], + max_tokens: 5, + }; + items.push(item(nextId("pdist-sample"), sname, request("POST", url(["v1", "chat", "completions"]), body, headers), + events(null, distSampleProviderTest(sname, sid, cleanupTarget, i === 0)))); + } + const aname = uniq(step.label); + const only = (step.expectOnly || []).map((r) => jsNameOf(r)); + const never = (step.expectNever || []).map((r) => jsNameOf(r)); + const all = (step.expectAll || []).map((r) => jsNameOf(r)); + const exec = [ + `var cleanupReq = ${JSON.stringify(cleanupTarget)};`, + "var ok = true, errMsg = '';", + "try {", + ` var seen = (pm.collectionVariables.get('pdist_${sid}') || '').split(',').filter(Boolean);`, + ` var only = [${only.join(", ")}];`, + ` var never = [${never.join(", ")}];`, + ` var all = [${all.join(", ")}];`, + " if (seen.length === 0) throw new Error('no provider observed across samples');", + " seen.forEach(function (p) {", + " if (only.length && only.indexOf(p) < 0) throw new Error('provider ' + p + ' served but not in expectOnly ' + JSON.stringify(only));", + " if (never.indexOf(p) >= 0) throw new Error('provider ' + p + ' served but was expectNever; observed ' + JSON.stringify(seen));", + " });", + " all.forEach(function (p) { if (seen.indexOf(p) < 0) throw new Error('provider ' + p + ' expected to serve but did not; observed ' + JSON.stringify(seen)); });", + "} catch (e) { ok = false; errMsg = e.message; }", + `pm.test(${JSON.stringify(aname)}, function () { if (!ok) throw new Error(errMsg); });`, + "if (!ok) { pm.execution.setNextRequest(cleanupReq); }", + ]; + items.push(item(nextId("pdist-assert"), aname, request("GET", url(["health"]), null), events(null, exec))); + break; + } + case "assertRoutingTrail": { + // Step 1: poll the log list for the VK's most recent row and capture its id. + const capName = uniq("capture routing log id"); + const capQuery = [{ key: "virtual_key_ids", value: `{{vkid_${sid}}}` }, { key: "limit", value: "1" }]; + const capAssert = [ + "if (pm.response.code !== 200) { throw new Error('logs status ' + pm.response.code); }", + "var logs = (pm.response.json() || {}).logs || [];", + "if (!logs.length || !logs[0].id) { throw new Error('no log row yet for VK'); }", + `pm.collectionVariables.set('logid_${sid}', logs[0].id);`, + ]; + items.push(item(nextId("capture-log"), capName, request("GET", url(["api", "logs"], capQuery), null), + events(pollPrerequest(step.waitSeconds), pollTest(capName, capAssert, cleanupTarget)))); + // Step 2: fetch the log detail and assert the routing-engine decision trail. + const trailName = uniq(step.label); + const trailAssert = [ + "if (pm.response.code !== 200) { throw new Error('log detail status ' + pm.response.code); }", + "var row = pm.response.json() || {};", + "var trail = row.routing_engine_logs || '';", + `var expected = ${JSON.stringify(step.expectSubstrings || [])};`, + "expected.forEach(function (s) {", + " if (String(trail).indexOf(s) < 0) { throw new Error('routing_engine_logs missing ' + JSON.stringify(s) + '; got ' + JSON.stringify(trail)); }", + "});", + ]; + items.push(item(nextId("assert-trail"), trailName, + request("GET", url(["api", "logs", `{{logid_${sid}}}`]), null), + events(null, pollTest(trailName, trailAssert, cleanupTarget)))); + break; + } + case "cleanup": + break; + default: + throw new Error("unknown step type: " + step.type); + } + } + + const cleanupItems = []; + if (hasVk) { + cleanupItems.push(item(`rt-${sid}-cleanup-vk`, cleanupVk, + request("DELETE", url(["api", "governance", "virtual-keys", `{{vkid_${sid}}}`]), null), + events(null, cleanupTest(cleanupVk)))); + } + // Delete every provider the scenario created (the primary plus any extra refs). + // The primary keeps the canonical cleanup name so failing-step jumps still hit it. + const refsToClean = providerRefs.length ? providerRefs : ["self"]; + for (const ref of refsToClean) { + const cname = ref === "self" ? cleanupProvider : `cleanup: delete provider ${ref} [${sid}]`; + cleanupItems.push(item(`rt-${sid}-cleanup-provider-${ref}`, cname, + request("DELETE", url(["api", "providers", nameOf(ref)]), null), + events(null, cleanupTest(cname)))); + } + + return { + name: sc.title, + description: sc.description, + item: [...items, { name: "Cleanup", item: cleanupItems }], + event: events(folderPrerequest([]), null), + }; +} + +// --------------------------------------------------------------------------- // +// Scenarios — governance × model-catalog routing +// --------------------------------------------------------------------------- // + +const SCENARIOS = [ + { + id: "vk-allows-model", + title: "VK allows model, key allows — request routes", + description: "A VK whose allowed_models includes the request, over a key that allows it, routes successfully and records the key.", + steps: [ + { type: "addProvider", keys: [key({ id: "k1", models: ["*"] })] }, + { type: "createVK", providerConfigs: [vkProvider({ allowedModels: [MODEL_B] })] }, + { type: "route", model: MODEL_B, expectStatus: 200, expectKeyId: "k1", waitSeconds: 1, label: "allowed model routes (routing_info.key)" }, + { type: "assertLog", model: MODEL_B, expectStatus: "success", expectSelectedKeyId: "k1", expectVkPresent: true, waitSeconds: 2, label: "log records VK + selected key" }, + { type: "cleanup" }, + ], + }, + { + id: "vk-key-restriction", + title: "VK key restriction pins routing to the allowed key", + description: "A VK whose key_ids names only k1 routes through k1 even though k2 also serves the model; routing_info.key and the log's selected_key_name are k1.", + steps: [ + { type: "addProvider", keys: [key({ id: "k1", models: ["*"] }), key({ id: "k2", models: ["*"] })] }, + { type: "createVK", providerConfigs: [vkProvider({ keyIds: ["k1"], allowedModels: ["*"] })] }, + { type: "route", model: MODEL_B, expectStatus: 200, expectKeyId: "k1", waitSeconds: 1, label: "routes via the VK-permitted key (routing_info.key=k1)" }, + { type: "assertLog", model: MODEL_B, expectStatus: "success", expectSelectedKeyId: "k1", expectVkPresent: true, waitSeconds: 2, label: "log selected_key_name is k1" }, + { type: "cleanup" }, + ], + }, + { + id: "vk-disabled-key", + title: "VK pinned to a disabled key is unroutable even with an enabled sibling", + description: "The VK restricts to k1, which is disabled; k2 is enabled but VK-excluded, so key selection finds no usable key and rejects with 400.", + steps: [ + { type: "addProvider", keys: [key({ id: "k1", models: ["*"], enabled: false }), key({ id: "k2", models: ["*"] })] }, + { type: "createVK", providerConfigs: [vkProvider({ keyIds: ["k1"], allowedModels: ["*"] })] }, + { type: "route", model: MODEL_B, expectStatus: 400, expectErrorSubstr: "no keys found", waitSeconds: 1, label: "disabled+VK-restricted key yields no usable key (400)" }, + { type: "cleanup" }, + ], + }, + { + id: "multi-key-distribution", + title: "Both keys serve under weighted key selection", + description: "Two enabled, equal-weight keys on one provider (no VK). Over a batch of requests, core's weighted key selection routes through both keys.", + steps: [ + { type: "addProvider", keys: [key({ id: "k1", models: ["*"], weight: 1 }), key({ id: "k2", models: ["*"], weight: 1 })] }, + { type: "keyDistribution", model: MODEL_B, n: 8, expectKeyIds: ["k1", "k2"], label: "both keys served across 8 samples" }, + { type: "cleanup" }, + ], + }, + { + id: "weightless-vk-allowlist-via-logs", + title: "A weightless VK is an allow-list (no LB), confirmed by the routing log trail", + description: "Two providers on a VK with NO weights: A's key serves gpt-4o-mini, B's serves only gpt-4o. Routing the bare gpt-4o-mini, governance filters by capability (excludes B) and — having no weighted configs — skips load balancing, routing to A. The routing_engine_logs record the allow-list decisions.", + steps: [ + { type: "addProvider", ref: "self", providerType: "openai", keys: [key({ id: "ka", models: [MODEL_B] })] }, + { type: "addProvider", ref: "b", providerType: "openai", keys: [key({ id: "kb", models: [MODEL_A] })] }, + { type: "createVK", providerConfigs: [vkProvider({ providerRef: "self", weight: null, allowedModels: ["*"] }), vkProvider({ providerRef: "b", weight: null, allowedModels: ["*"] })] }, + { type: "route", model: MODEL_B, bareModel: true, expectStatus: 200, expectProviderOneOf: ["self"], waitSeconds: 3, label: "bare model routes to the only capable provider (allow-list, no LB)" }, + { type: "assertRoutingTrail", expectSubstrings: ["not in allowed models list", "No weighted configs", "skipping load balancing"], waitSeconds: 2, label: "log trail shows allow-list filtering and LB skipped" }, + { type: "cleanup" }, + ], + }, + { + id: "governance-lb-distributes", + title: "Governance load-balances a bare model across VK providers", + description: "A VK with two weighted providers, routing a bare (un-prefixed) model, has governance pick one of them; the log detail's routing_engine_logs records the load-balancing decision.", + steps: [ + { type: "addProvider", ref: "self", keys: [key({ id: "ka", models: ["*"], weight: 1 })] }, + { type: "addProvider", ref: "b", keys: [key({ id: "kb", models: ["*"], weight: 1 })] }, + { type: "createVK", providerConfigs: [vkProvider({ providerRef: "self", allowedModels: ["*"] }), vkProvider({ providerRef: "b", allowedModels: ["*"] })] }, + { type: "route", model: MODEL_B, bareModel: true, expectStatus: 200, expectProviderOneOf: ["self", "b"], waitSeconds: 2, label: "bare model routes via a governance-selected provider" }, + { type: "assertRoutingTrail", expectSubstrings: ["Load balancing model", "Selected provider"], waitSeconds: 2, label: "log detail records the LB decision trail" }, + { type: "cleanup" }, + ], + }, + { + id: "reverse-alias-gate", + title: "Routing the resolved model directly (not the alias) is rejected", + description: "A key gates the alias name, not the resolved id. Routing the alias resolves and succeeds; routing the resolved model id directly fails the gate — alias targets are not auto-added to the key's Models.", + steps: [ + { type: "addProvider", keys: [key({ id: "k1", models: ["catwiring-alias-{{run_id}}"], aliases: { "catwiring-alias-{{run_id}}": MODEL_B } })] }, + { type: "route", model: "catwiring-alias-{{run_id}}", useVk: false, expectStatus: 200, expectResolvedModelId: MODEL_B, waitSeconds: 1, label: "alias routes (resolves to the model id)" }, + { type: "route", model: MODEL_B, useVk: false, expectStatus: 400, expectErrorSubstr: "no keys found that support model", waitSeconds: 0, label: "resolved id routed directly → 400 (not in Models)" }, + { type: "cleanup" }, + ], + }, + { + id: "alias-collision-last-wins", + title: "An alias defined on two keys resolves to the last key", + description: "Two keys define the same alias to different models. The alias resolves to the last-defined key's target and is served by that key.", + steps: [ + { type: "addProvider", keys: [ + key({ id: "k1", models: ["catwiring-dup-{{run_id}}"], aliases: { "catwiring-dup-{{run_id}}": MODEL_A } }), + key({ id: "k2", models: ["catwiring-dup-{{run_id}}"], aliases: { "catwiring-dup-{{run_id}}": MODEL_B } }), + ] }, + { type: "route", model: "catwiring-dup-{{run_id}}", useVk: false, expectStatus: 200, expectKeyId: "k2", expectResolvedModelId: MODEL_B, waitSeconds: 1, label: "alias resolves to the last key (k2 → gpt-4o-mini)" }, + { type: "cleanup" }, + ], + }, + { + id: "alias-case-insensitive-fallback", + title: "A request resolves to an alias whose name differs only in case", + description: "A key defines a mixed-case alias and gates the mixed-case name. Routing the lowercased form finds no exact-case alias but resolves through a case-insensitive fallback to the same target.", + steps: [ + { type: "addProvider", keys: [key({ id: "k1", models: ["CatWiring-CI-{{run_id}}"], aliases: { "CatWiring-CI-{{run_id}}": MODEL_B } })] }, + { type: "route", model: "catwiring-ci-{{run_id}}", useVk: false, expectStatus: 200, expectKeyId: "k1", expectResolvedModelId: MODEL_B, waitSeconds: 1, label: "lowercased request resolves via case-insensitive fallback" }, + { type: "cleanup" }, + ], + }, + { + id: "alias-routing-no-vk", + title: "Alias resolves at routing with no governance", + description: "Pure model-catalog case (no VK): a key alias routes an inference request to the underlying model; routing_info.resolved_key_alias records the resolution.", + steps: [ + { type: "addProvider", keys: [key({ id: "k1", models: ["catwiring-alias-{{run_id}}"], aliases: { "catwiring-alias-{{run_id}}": MODEL_B } })] }, + { type: "route", model: "catwiring-alias-{{run_id}}", useVk: false, expectStatus: 200, expectResolvedModelId: MODEL_B, waitSeconds: 1, label: "alias routes to underlying model (no VK)" }, + { type: "cleanup" }, + ], + }, + { + id: "blacklist-gate-no-vk", + title: "Key blacklist gates routing with no governance", + description: "Pure model-catalog case (no VK): a key that allows all models but blacklists one rejects that model at key selection.", + steps: [ + { type: "addProvider", keys: [key({ id: "k1", models: ["*"], blacklisted: [MODEL_A] })] }, + { type: "route", model: MODEL_A, useVk: false, expectStatus: 400, expectErrorSubstr: "no keys found that support model", waitSeconds: 0, label: "blacklisted model rejected by key gate (400)" }, + { type: "cleanup" }, + ], + }, + { + id: "disabled-key-gate-no-vk", + title: "Disabled key makes a model unroutable with no governance", + description: "Pure model-catalog case (no VK): the only key for a model is disabled, so key selection finds nothing.", + steps: [ + { type: "addProvider", keys: [key({ id: "k1", models: ["*"], enabled: false })] }, + { type: "route", model: MODEL_B, useVk: false, expectStatus: 400, expectErrorSubstr: "no keys found", waitSeconds: 0, label: "disabled key yields no route (400)" }, + { type: "cleanup" }, + ], + }, + { + id: "route-azure", + title: "Azure provider routes a deployment", + description: "A standard azure provider (value + azure_key_config via env.) routes a model whose deployment matches the name. Serial-only (global standard provider).", + steps: [ + { type: "addProvider", providerType: "azure", keys: [key({ id: "kaz", models: ["*"] })] }, + { type: "route", model: PROVIDERS.azure.model, useVk: false, expectStatus: 200, expectKeyId: "kaz", waitSeconds: 1, label: "azure routes its deployment (200)" }, + { type: "cleanup" }, + ], + }, + { + id: "route-bedrock", + title: "Bedrock routes Claude via a key alias to its inference profile", + description: "A standard bedrock provider whose key aliases the common name claude-sonnet-4-5 to the cross-region inference-profile id. Routing the friendly name resolves to the wire id. Serial-only (global standard provider).", + steps: [ + { type: "addProvider", providerType: "bedrock", keys: [key({ id: "kbr", models: ["*"] })] }, + { type: "route", model: PROVIDERS.bedrock.model, useVk: false, expectStatus: 200, expectKeyId: "kbr", expectResolvedModelId: "us.anthropic.claude-sonnet-4-5-20250929-v1:0", waitSeconds: 1, label: "bedrock alias resolves to inference profile (200)" }, + { type: "cleanup" }, + ], + }, + { + id: "cross-provider-same-model-openai-azure", + title: "Governance LBs one openai model across OpenAI and Azure", + description: "OpenAI (custom) and Azure (standard) both serve gpt-4o-mini. A VK over both, routing the bare model, has governance distribute across them. Serial-only (Azure is a global standard provider).", + steps: [ + { type: "addProvider", ref: "self", providerType: "openai", keys: [key({ id: "ko", models: ["*"], weight: 1 })] }, + { type: "addProvider", ref: "az", providerType: "azure", keys: [key({ id: "kaz", models: ["*"], weight: 1 })] }, + { type: "createVK", providerConfigs: [vkProvider({ providerRef: "self", allowedModels: ["*"] }), vkProvider({ providerRef: "az", allowedModels: ["*"] })] }, + { type: "route", model: "gpt-4o-mini", bareModel: true, expectStatus: 200, expectProviderOneOf: ["self", "az"], waitSeconds: 3, label: "bare model routes via openai or azure (200)" }, + { type: "assertRoutingTrail", expectSubstrings: ["Load balancing model gpt-4o-mini", "Selected provider"], waitSeconds: 2, label: "log detail records openai/azure LB trail" }, + { type: "cleanup" }, + ], + }, + { + id: "route-anthropic", + title: "Anthropic provider routes its model", + description: "A custom provider backed by anthropic (key via env.) routes a claude model.", + steps: [ + { type: "addProvider", providerType: "anthropic", keys: [key({ id: "ka", models: ["*"] })] }, + { type: "route", model: PROVIDERS.anthropic.model, useVk: false, expectStatus: 200, expectKeyId: "ka", waitSeconds: 1, label: "anthropic provider routes claude model (200)" }, + { type: "cleanup" }, + ], + }, + { + id: "route-gemini", + title: "Gemini provider routes its model", + description: "A custom provider backed by gemini (key via env.) routes a gemini model.", + steps: [ + { type: "addProvider", providerType: "gemini", keys: [key({ id: "kg", models: ["*"] })] }, + { type: "route", model: PROVIDERS.gemini.model, useVk: false, expectStatus: 200, expectKeyId: "kg", waitSeconds: 1, label: "gemini provider routes gemini model (200)" }, + { type: "cleanup" }, + ], + }, + { + id: "vk-explicit-allowed-key-cant-serve", + title: "VK explicit allowed-model the key can't serve → 400 (not 403)", + description: "Unlike the wildcard case (which 403s via the catalog-aware check), an EXPLICIT allowed_models entry is string-matched by governance and passes; key selection then fails → 400 no keys. Pins the explicit-vs-wildcard split: explicit lists are string-matched by governance, wildcards go through the catalog-aware check.", + steps: [ + { type: "addProvider", keys: [key({ id: "k1", models: [MODEL_B] })] }, + { type: "createVK", providerConfigs: [vkProvider({ allowedModels: [MODEL_A] })] }, + { type: "route", model: MODEL_A, expectStatus: 400, expectErrorSubstr: "no keys found that support model", waitSeconds: 0, label: "explicit allowed model the key can't serve → 400" }, + { type: "cleanup" }, + ], + }, + { + id: "key-gate-beats-key-weight", + title: "Within a provider, the capable key wins over a higher-weight incapable key", + description: "Pure model-catalog/core (no VK): k1 weight 99 allows only gpt-4o; k2 weight 1 allows gpt-4o-mini. Routing gpt-4o-mini always uses k2 — key capability filters before weighting.", + steps: [ + { type: "addProvider", keys: [key({ id: "k1", models: [MODEL_A], weight: 99 }), key({ id: "k2", models: [MODEL_B], weight: 1 })] }, + { type: "keyDistribution", model: MODEL_B, n: 10, expectOnly: ["k2"], expectNever: ["k1"], label: "all requests use the capable 1%-weight key (k2)" }, + { type: "cleanup" }, + ], + }, + { + id: "key-blacklist-intersection", + title: "A model blacklisted on one key still routes via a sibling key", + description: "Pure model-catalog/core (no VK): k1 allows all but blacklists gpt-4o-mini; k2 allows all. The model is blocked only on k1, so the provider still serves it via k2 (blacklist is per-key, not provider-wide unless all keys block).", + steps: [ + { type: "addProvider", keys: [key({ id: "k1", models: ["*"], blacklisted: [MODEL_B] }), key({ id: "k2", models: ["*"] })] }, + { type: "keyDistribution", model: MODEL_B, n: 10, expectOnly: ["k2"], expectNever: ["k1"], label: "model routes via the non-blacklisting sibling (k2)" }, + { type: "cleanup" }, + ], + }, + { + id: "lb-partial-exclusion-3-providers", + title: "LB excludes only the incapable provider; the rest still split", + description: "Three providers on a VK: A can't serve the model (key gate), B and C can. Routing the bare model, A is excluded and B/C both still serve over the batch.", + steps: [ + { type: "addProvider", ref: "self", providerType: "openai", keys: [key({ id: "ka", models: [MODEL_A], weight: 1 })] }, + { type: "addProvider", ref: "b", providerType: "openai", keys: [key({ id: "kb", models: ["*"], weight: 1 })] }, + { type: "addProvider", ref: "c", providerType: "openai", keys: [key({ id: "kc", models: ["*"], weight: 1 })] }, + { type: "createVK", providerConfigs: [vkProvider({ providerRef: "self", weight: 1, allowedModels: ["*"] }), vkProvider({ providerRef: "b", weight: 1, allowedModels: ["*"] }), vkProvider({ providerRef: "c", weight: 1, allowedModels: ["*"] })] }, + { type: "providerDistribution", model: MODEL_B, bareModel: true, n: 12, expectOnly: ["b", "c"], expectAll: ["b", "c"], expectNever: ["self"], label: "incapable provider excluded; B and C both serve" }, + { type: "cleanup" }, + ], + }, + { + id: "lb-skips-model-gated-provider", + title: "LB skips a 99%-weight provider that can't serve the model (key gate)", + description: "A is weighted 99% but its key only allows gpt-4o; B is weighted 1% and allows all. Routing the bare gpt-4o-mini, capability filtering excludes A entirely, so every request lands on B regardless of weight.", + steps: [ + { type: "addProvider", ref: "self", providerType: "openai", keys: [key({ id: "ka", models: ["gpt-4o"], weight: 1 })] }, + { type: "addProvider", ref: "b", providerType: "openai", keys: [key({ id: "kb", models: ["*"], weight: 1 })] }, + { type: "createVK", providerConfigs: [vkProvider({ providerRef: "self", weight: 99, allowedModels: ["*"] }), vkProvider({ providerRef: "b", weight: 1, allowedModels: ["*"] })] }, + { type: "providerDistribution", model: "gpt-4o-mini", bareModel: true, n: 10, expectOnly: ["b"], expectNever: ["self"], label: "all requests go to the 1% provider (99% provider can't serve the model)" }, + { type: "cleanup" }, + ], + }, + { + id: "lb-skips-blacklisted-provider", + title: "LB skips a 99%-weight provider that blacklists the model", + description: "A (99%) allows all models but blacklists gpt-4o-mini on its key; B (1%) allows all. The blacklist removes A from the candidates, so every request lands on B.", + steps: [ + { type: "addProvider", ref: "self", providerType: "openai", keys: [key({ id: "ka", models: ["*"], blacklisted: ["gpt-4o-mini"], weight: 1 })] }, + { type: "addProvider", ref: "b", providerType: "openai", keys: [key({ id: "kb", models: ["*"], weight: 1 })] }, + { type: "createVK", providerConfigs: [vkProvider({ providerRef: "self", weight: 99, allowedModels: ["*"] }), vkProvider({ providerRef: "b", weight: 1, allowedModels: ["*"] })] }, + { type: "providerDistribution", model: "gpt-4o-mini", bareModel: true, n: 10, expectOnly: ["b"], expectNever: ["self"], label: "all requests go to the 1% provider (99% provider blacklists the model)" }, + { type: "cleanup" }, + ], + }, + { + id: "lb-skips-disabled-key-provider", + title: "LB skips a 99%-weight provider whose only key is disabled", + description: "A (99%) has its only key disabled; B (1%) is enabled. With no usable key, A is excluded, so every request lands on B.", + steps: [ + { type: "addProvider", ref: "self", providerType: "openai", keys: [key({ id: "ka", models: ["*"], enabled: false, weight: 1 })] }, + { type: "addProvider", ref: "b", providerType: "openai", keys: [key({ id: "kb", models: ["*"], weight: 1 })] }, + { type: "createVK", providerConfigs: [vkProvider({ providerRef: "self", weight: 99, allowedModels: ["*"] }), vkProvider({ providerRef: "b", weight: 1, allowedModels: ["*"] })] }, + { type: "providerDistribution", model: "gpt-4o-mini", bareModel: true, n: 10, expectOnly: ["b"], expectNever: ["self"], label: "all requests go to the 1% provider (99% provider key disabled)" }, + { type: "cleanup" }, + ], + }, + { + id: "cross-provider-allowlist", + title: "VK provider allowlist blocks a different real provider", + description: "A VK that lists only the openai provider routes openai but rejects an explicit request to the anthropic provider (pruned from the routing allowlist).", + steps: [ + { type: "addProvider", ref: "self", providerType: "openai", keys: [key({ id: "ko", models: ["*"] })] }, + { type: "addProvider", ref: "b", providerType: "anthropic", keys: [key({ id: "ka", models: ["*"] })] }, + { type: "createVK", providerConfigs: [vkProvider({ providerRef: "self", allowedModels: ["*"] })] }, + { type: "route", routeRef: "self", model: PROVIDERS.openai.model, expectStatus: 200, expectKeyId: "ko", waitSeconds: 2, label: "VK-allowed provider routes (200)" }, + { type: "route", routeRef: "b", model: PROVIDERS.anthropic.model, expectStatus: 400, expectErrorSubstr: "is not permitted for this request", waitSeconds: 0, label: "VK-disallowed provider blocked (400)" }, + { type: "cleanup" }, + ], + }, + { + id: "cross-provider-same-model-lb", + title: "Governance LBs one Claude model across Anthropic, Vertex, and Bedrock", + description: "Anthropic (custom, native), Vertex (standard, native) and Bedrock (standard, via a key alias to its inference-profile id) all serve claude-sonnet-4-5. A VK over all three, routing the bare model, has governance distribute across the heterogeneous providers; the log detail records the LB trail. Serial-only (Vertex/Bedrock are global standard providers).", + steps: [ + { type: "addProvider", ref: "self", providerType: "anthropic", keys: [key({ id: "kan", models: ["*"], weight: 1 })] }, + { type: "addProvider", ref: "vtx", providerType: "vertex", keys: [key({ id: "kvx", models: ["*"], weight: 1 })] }, + { type: "addProvider", ref: "bdr", providerType: "bedrock", keys: [key({ id: "kbd", models: ["*"], weight: 1 })] }, + { type: "createVK", providerConfigs: [vkProvider({ providerRef: "self", allowedModels: ["*"] }), vkProvider({ providerRef: "vtx", allowedModels: ["*"] }), vkProvider({ providerRef: "bdr", allowedModels: ["*"] })] }, + { type: "route", model: "claude-sonnet-4-5", bareModel: true, expectStatus: 200, expectProviderOneOf: ["self", "vtx", "bdr"], waitSeconds: 3, label: "bare claude model routes via anthropic, vertex, or bedrock (200)" }, + { type: "assertRoutingTrail", expectSubstrings: ["Load balancing model claude-sonnet-4-5", "Selected provider"], waitSeconds: 2, label: "log detail records cross-provider LB trail" }, + { type: "cleanup" }, + ], + }, + { + id: "fallback-cross-provider", + title: "A request fails over to a healthy provider via a request-level fallback", + description: "The primary provider's key is invalid, so its attempt fails upstream; a request-level fallback to a second provider serving the same model succeeds. routing_info marks the fallback and records the original primary provider/model. The /v1 endpoint takes fallbacks in the string \"provider/model\" form.", + steps: [ + { type: "addProvider", ref: "self", keys: [key({ id: "kbad", models: [MODEL_B], badKey: true })] }, + { type: "addProvider", ref: "b", keys: [key({ id: "kgood", models: [MODEL_B] })] }, + { type: "route", model: MODEL_B, routeRef: "self", useVk: false, fallbacks: [{ providerRef: "b", model: MODEL_B }], expectStatus: 200, expectProviderOneOf: ["b"], expectIsFallback: true, expectPrimaryProviderRef: "self", expectPrimaryModel: MODEL_B, waitSeconds: 1, label: "primary fails; request fallback serves (200, is_fallback)" }, + { type: "cleanup" }, + ], + }, + { + id: "fallback-chain-first-healthy-wins", + title: "Fallbacks are tried in order until one succeeds", + description: "The primary and the first fallback both have invalid keys; the second fallback succeeds. routing_info reports the surviving provider and still names the original primary.", + steps: [ + { type: "addProvider", ref: "self", keys: [key({ id: "kbad1", models: [MODEL_B], badKey: true })] }, + { type: "addProvider", ref: "b", keys: [key({ id: "kbad2", models: [MODEL_B], badKey: true })] }, + { type: "addProvider", ref: "c", keys: [key({ id: "kgood", models: [MODEL_B] })] }, + { type: "route", model: MODEL_B, routeRef: "self", useVk: false, fallbacks: [{ providerRef: "b", model: MODEL_B }, { providerRef: "c", model: MODEL_B }], expectStatus: 200, expectProviderOneOf: ["c"], expectIsFallback: true, expectPrimaryProviderRef: "self", expectPrimaryModel: MODEL_B, waitSeconds: 1, label: "chain falls through to the only healthy provider (200)" }, + { type: "cleanup" }, + ], + }, + { + id: "fallback-pruned-by-vk-allowlist", + title: "A fallback to a provider off the VK allowlist is pruned, not tried", + description: "Under a VK that permits only the primary provider, a request-level fallback to a healthy off-allowlist provider is pruned before the attempt loop. The primary's invalid key fails with a 401 and the pruned provider — which has a valid key and would otherwise return 200 — never rescues it, proving it was dropped.", + steps: [ + { type: "addProvider", ref: "self", keys: [key({ id: "kbad", models: [MODEL_B], badKey: true })] }, + { type: "addProvider", ref: "b", keys: [key({ id: "kgood", models: [MODEL_B] })] }, + { type: "createVK", providerConfigs: [vkProvider({ providerRef: "self", allowedModels: ["*"] })] }, + { type: "route", model: MODEL_B, routeRef: "self", fallbacks: [{ providerRef: "b", model: MODEL_B }], expectStatus: 401, expectErrorSubstr: "Incorrect API key", waitSeconds: 1, label: "off-allowlist fallback pruned; request fails on the primary (401)" }, + { type: "cleanup" }, + ], + }, + { + id: "vk-auto-attached-fallback", + title: "A VK with weighted providers auto-attaches the others as fallbacks", + description: "A VK weights two providers that both serve the model; the dominant-weight provider's key is invalid. With no request-level fallbacks, governance auto-attaches the remaining weighted config as a fallback, so every request still lands on the healthy low-weight provider — whether it was the load-balanced primary or the fallback.", + steps: [ + { type: "addProvider", ref: "self", keys: [key({ id: "kbad", models: [MODEL_B], badKey: true })] }, + { type: "addProvider", ref: "b", keys: [key({ id: "kgood", models: [MODEL_B] })] }, + { type: "createVK", providerConfigs: [vkProvider({ providerRef: "self", weight: 100, allowedModels: ["*"] }), vkProvider({ providerRef: "b", weight: 1, allowedModels: ["*"] })] }, + { type: "providerDistribution", model: MODEL_B, bareModel: true, n: 6, expectOnly: ["b"], label: "every request lands on the healthy provider via the auto-attached fallback" }, + { type: "cleanup" }, + ], + }, + { + id: "standard-openai-route", + title: "Standard (non-custom) openai provider routes", + description: "A standard openai provider created via the API routes its model. The catalog is datasheet-backed (no live-cache wait). NOTE: standard providers are global singletons — this scenario is NOT run-id-isolated; run serially against a clean instance, not in parallel shards.", + steps: [ + { type: "addProvider", providerKind: "standard", providerType: "openai", keys: [key({ id: "ko", models: ["gpt-4o-mini"] })] }, + { type: "route", model: "gpt-4o-mini", useVk: false, expectStatus: 200, expectKeyId: "ko", waitSeconds: 0, label: "standard openai routes its model (200)" }, + { type: "cleanup" }, + ], + }, + { + id: "standard-openai-vk-gate", + title: "Governance gates a standard provider", + description: "A VK over a standard openai provider routes an allowed model and prunes a disallowed one. Serial-only (global provider).", + steps: [ + { type: "addProvider", providerKind: "standard", providerType: "openai", keys: [key({ id: "ko", models: ["*"] })] }, + { type: "createVK", providerConfigs: [vkProvider({ providerRef: "self", allowedModels: ["gpt-4o-mini"] })] }, + { type: "route", model: "gpt-4o-mini", expectStatus: 200, expectKeyId: "ko", waitSeconds: 2, label: "VK-allowed model routes on standard provider (200)" }, + { type: "assertLog", model: "gpt-4o-mini", expectStatus: "success", expectSelectedKeyId: "ko", expectVkPresent: true, waitSeconds: 2, label: "log records standard-provider route" }, + { type: "route", model: "gpt-4o", expectStatus: 400, expectErrorSubstr: "is not permitted for this request", waitSeconds: 0, label: "VK-disallowed model pruned (400)" }, + { type: "cleanup" }, + ], + }, + { + id: "vk-model-whitelist", + title: "VK model whitelist blocks an unlisted model", + description: "A model outside the VK's allowed_models prunes the (single) provider from the routing allowlist, so core rejects with a 'provider not permitted' 400 (intended).", + steps: [ + { type: "addProvider", keys: [key({ id: "k1", models: ["*"] })] }, + { type: "createVK", providerConfigs: [vkProvider({ allowedModels: [MODEL_B] })] }, + { type: "route", model: MODEL_A, expectStatus: 400, expectErrorSubstr: "is not permitted for this request", waitSeconds: 0, label: "unlisted model rejected via empty allowlist (400)" }, + { type: "cleanup" }, + ], + }, + { + id: "vk-blacklist", + title: "VK blacklist blocks a model", + description: "A model in the VK's blacklisted_models prunes the (single) provider from the routing allowlist, so core rejects with a 'provider not permitted' 400 (intended).", + steps: [ + { type: "addProvider", keys: [key({ id: "k1", models: ["*"] })] }, + { type: "createVK", providerConfigs: [vkProvider({ allowedModels: ["*"], blacklistedModels: [MODEL_A] })] }, + { type: "route", model: MODEL_A, expectStatus: 400, expectErrorSubstr: "is not permitted for this request", waitSeconds: 0, label: "blacklisted model rejected via empty allowlist (400)" }, + { type: "cleanup" }, + ], + }, + { + id: "vk-wildcard-bounded-by-key-gate", + title: "VK wildcard is still bounded by the key gate", + description: "Governance's model check is catalog-aware, so a VK allowed_models=[\"*\"] does not widen past what the key actually gates; a model outside the key's allow-list is blocked by governance.", + steps: [ + { type: "addProvider", keys: [key({ id: "k1", models: [MODEL_B] })] }, + { type: "createVK", providerConfigs: [vkProvider({ allowedModels: ["*"] })] }, + { type: "route", model: MODEL_A, expectStatus: 403, expectErrorSubstr: "is not allowed for this virtual key", waitSeconds: 0, label: "wildcard VK still blocks a model the key does not gate (403)" }, + { type: "cleanup" }, + ], + }, + { + id: "catalog-gate-blocks-no-vk", + title: "Catalog/key gate blocks a model with no VK (400)", + description: "Without a virtual key, governance does not run; a model the key does not allow is rejected by core key selection.", + steps: [ + { type: "addProvider", keys: [key({ id: "k1", models: [MODEL_B] })] }, + { type: "route", model: MODEL_A, useVk: false, expectStatus: 400, expectErrorSubstr: "no keys found that support model", waitSeconds: 0, label: "key gate rejects unlisted model (400)" }, + { type: "cleanup" }, + ], + }, + { + id: "alias-whitelisted-by-name", + title: "Alias whitelisted by name resolves", + description: "When the VK whitelists the alias name, the request routes and resolves to the underlying model id.", + steps: [ + { type: "addProvider", keys: [key({ id: "k1", models: ["catwiring-alias-{{run_id}}"], aliases: { "catwiring-alias-{{run_id}}": MODEL_B } })] }, + { type: "createVK", providerConfigs: [vkProvider({ allowedModels: ["catwiring-alias-{{run_id}}"] })] }, + { type: "route", model: "catwiring-alias-{{run_id}}", expectStatus: 200, expectResolvedModelId: MODEL_B, waitSeconds: 1, label: "alias routes, resolves to model id" }, + { type: "cleanup" }, + ], + }, + { + id: "alias-vs-whitelist", + title: "Alias name not whitelisted is blocked", + description: "The VK whitelists the resolved model id, but the request uses the alias name; the alias isn't in allowed_models, so the provider is pruned from the routing allowlist and core rejects with a 'provider not permitted' 400 (intended).", + steps: [ + { type: "addProvider", keys: [key({ id: "k1", models: ["catwiring-alias-{{run_id}}"], aliases: { "catwiring-alias-{{run_id}}": MODEL_B } })] }, + { type: "createVK", providerConfigs: [vkProvider({ allowedModels: [MODEL_B] })] }, + { type: "route", model: "catwiring-alias-{{run_id}}", expectStatus: 400, expectErrorSubstr: "is not permitted for this request", waitSeconds: 0, label: "unlisted alias rejected via empty allowlist (400)" }, + { type: "cleanup" }, + ], + }, + { + id: "vk-empty-configs", + title: "VK with no provider configs blocks everything", + description: "An empty provider_configs is deny-by-default; the provider is not permitted.", + steps: [ + { type: "addProvider", keys: [key({ id: "k1", models: ["*"] })] }, + { type: "createVK", providerConfigs: [] }, + { type: "route", model: MODEL_B, expectStatus: 400, expectErrorSubstr: "is not permitted for this request", waitSeconds: 0, label: "deny-by-default blocks request (400 routing allowlist)" }, + { type: "cleanup" }, + ], + }, +]; + +const collection = buildCollection({ + id: "bifrost-routing-wiring", + name: "Bifrost Routing Wiring (Governance x Catalog)", + description: + "Governance × model-catalog routing wiring. Each scenario stands up an isolated, run-namespaced " + + "custom provider backed by real OpenAI, gates it with keys and a virtual key, then drives inference " + + "and asserts the route via extra_fields.routing_info (success), the error message (rejection), and " + + "the stored log. Machine-generated by runners/build-routing-wiring.mjs — do not hand-edit.", + expandedScenarios: SCENARIOS.map(expandScenario), +}); + +writeCollection(resolveOutPath(DEFAULT_OUT), collection, SCENARIOS.map((s) => s.id)); diff --git a/tests/e2e/api/runners/filter-collection.mjs b/tests/e2e/api/runners/filter-collection.mjs index d2036d3c33..a5132c04f2 100644 --- a/tests/e2e/api/runners/filter-collection.mjs +++ b/tests/e2e/api/runners/filter-collection.mjs @@ -52,6 +52,7 @@ const PROVIDER_KEYWORDS = { vertex: ["vertex", "/genai/v1beta/models/{{vertexModel}}"], azure: ["azure", "deployments"], passthrough: ["_passthrough"], + openrouter: ["openrouter"], }; // Haystack = item JSON + ancestor folder names. Folder names encode the harness @@ -102,6 +103,12 @@ const itemMatchesProvider = (item, ancestorNames) => { if (!PROVIDER) return true; const keywords = PROVIDER_KEYWORDS[PROVIDER] || [PROVIDER]; const haystack = buildHaystack(item, ancestorNames); + // OpenRouter rows (model "openrouter//") embed vendor substrings like + // gpt-/claude-/gemini, so they'd otherwise be claimed by the openai/anthropic/gemini + // partitions too. Route them exclusively to the openrouter partition. + const isOpenRouter = haystack.includes("openrouter"); + if (PROVIDER === "openrouter") return isOpenRouter; + if (isOpenRouter) return false; return keywords.some((k) => haystack.includes(k)); }; diff --git a/tests/e2e/api/runners/individual/run-newman-model-catalog-wiring-tests.sh b/tests/e2e/api/runners/individual/run-newman-model-catalog-wiring-tests.sh new file mode 100755 index 0000000000..e52b29978a --- /dev/null +++ b/tests/e2e/api/runners/individual/run-newman-model-catalog-wiring-tests.sh @@ -0,0 +1,145 @@ +#!/bin/bash + +# Bifrost Model Catalog Wiring Test Runner +# Drives the model-catalog wiring collection against a running Bifrost instance. +# Each scenario stands up an isolated, run-namespaced custom provider backed by a +# real upstream, mutates its providers/keys, and asserts the catalog read +# endpoints reflect every mutation. + +set -e +set -o pipefail + +# Associative arrays (declare -A), SEED_ENV_PATH parsing, and add_env_var_if_set +# all require Bash 4.0+. macOS still ships Bash 3.2 as /bin/bash, which would +# fail partway through with a cryptic "declare: -A: invalid option". +if [ "${BASH_VERSINFO[0]:-0}" -lt 4 ]; then + echo "Error: this script requires Bash 4.0+ (uses 'declare -A seed_env_values' and SEED_ENV_PATH parsing)." >&2 + echo "Detected Bash version: ${BASH_VERSION:-unknown}" >&2 + echo "On macOS, install a newer bash (e.g. 'brew install bash') and run the script with that interpreter." >&2 + exit 1 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +API_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" +cd "$API_DIR" + +COLLECTION="collections/bifrost-model-catalog-wiring.postman_collection.json" +ENVIRONMENT="bifrost-v1.postman_environment.json" +REPORT_DIR="newman-reports/model-catalog-wiring" + +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +NC='\033[0m' + +echo -e "${GREEN}========================================${NC}" +echo -e "${GREEN}Bifrost Model Catalog Wiring Tests${NC}" +echo -e "${GREEN}========================================${NC}" +echo "" + +if ! command -v newman &> /dev/null; then + echo -e "${RED}Error: Newman is not installed${NC}" + echo "Install it with: npm install -g newman newman-reporter-htmlextra" + exit 1 +fi + +if [ ! -f "$COLLECTION" ]; then + echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}" + exit 1 +fi + +ENV_FLAG=() +if [ -f "$ENVIRONMENT" ]; then + ENV_FLAG=(-e "$ENVIRONMENT") +else + echo -e "${YELLOW}Warning: Environment file not found: $ENVIRONMENT (using collection variables only)${NC}" +fi + +# Seed env loading: prefer an explicit BIFROST_E2E_SEED_ENV, else the standard +# generated/seed.env that CI writes. +SEED_ENV_PATH="${BIFROST_E2E_SEED_ENV:-}" +if [ -z "$SEED_ENV_PATH" ] && [ -f "$API_DIR/generated/seed.env" ]; then + SEED_ENV_PATH="$API_DIR/generated/seed.env" +fi + +declare -A seed_env_values +if [ -n "$SEED_ENV_PATH" ] && [ -f "$SEED_ENV_PATH" ]; then + # Parse the seed env as data, never source it — values may contain command + # substitution that sourcing would execute. + while IFS= read -r line || [ -n "$line" ]; do + [[ "$line" =~ ^[[:space:]]*# ]] && continue + [[ -z "${line//[[:space:]]/}" ]] && continue + [[ "$line" != *=* ]] && continue + key="${line%%=*}" + value="${line#*=}" + [[ ! "$key" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] && continue + # Unwrap outer single quotes written by the seed writer and undo its + # '\''-escape; also tolerate plain double-quoted values. + if [[ "$value" == \'*\' ]]; then + value="${value:1:${#value}-2}" + value="${value//\'\"\'\"\'/\'}" + elif [[ "$value" == \"*\" ]]; then + value="${value:1:${#value}-2}" + fi + seed_env_values["$key"]="$value" + done < "$SEED_ENV_PATH" +fi + +mkdir -p "$REPORT_DIR" + +cmd=(newman run "$COLLECTION" "${ENV_FLAG[@]}" -r cli,htmlextra) + +# Forward a seed-env value if present, falling back to the process environment +# so local runs work with credentials exported in the shell. +add_env_var_if_set() { + local name="$1" + local value="${seed_env_values[$name]:-}" + if [ -z "$value" ]; then + value="${!name:-}" + fi + if [ -n "$value" ]; then + cmd+=(--env-var "$name=$value") + fi +} + +# Forward the run-id prefix only. Provider credentials are NOT injected here: +# the collection's keys use Bifrost's `env.` resolution, so each key reads +# its credential from the Bifrost process env at request time (the Bifrost server +# must have the provider env vars, e.g. OPENAI_API_KEY/ANTHROPIC_API_KEY/...). +for v in e2e_seed_prefix; do + add_env_var_if_set "$v" +done + +# The live-model cache populates with one upstream round-trip's latency, and the +# in-collection poll loop can busy-wait up to ~4s per attempt. Give scripts and +# requests generous ceilings. +cmd+=(--timeout-script 120000 --timeout 900000) + +# In CI keep going past failures so every scenario's cleanup folder runs. +ci_normalized="$(printf '%s' "${CI:-}" | tr '[:upper:]' '[:lower:]')" +if [ "$ci_normalized" = "1" ] || [ "$ci_normalized" = "true" ]; then + cmd+=(--reporter-cli-no-failures false) +fi + +echo -e "Collection: ${YELLOW}$COLLECTION${NC}" +echo -e "Reports: ${YELLOW}$REPORT_DIR${NC}" +if [ -n "$SEED_ENV_PATH" ]; then + echo -e "Seed env: ${YELLOW}$SEED_ENV_PATH${NC}" +fi +echo "" +echo -e "${GREEN}Running tests...${NC}" +echo "" + +set +e +"${cmd[@]}" --reporter-htmlextra-export "$REPORT_DIR/report.html" --reporter-htmlextra-title "Bifrost Model Catalog Wiring" +EXIT_CODE=$? +set -e + +echo "" +if [ $EXIT_CODE -eq 0 ]; then + echo -e "${GREEN}✓ All tests passed!${NC}" +else + echo -e "${RED}✗ Some tests failed${NC}" +fi +echo -e "Report: ${YELLOW}$REPORT_DIR/report.html${NC}" +exit $EXIT_CODE diff --git a/tests/e2e/api/runners/individual/run-newman-routing-wiring-tests.sh b/tests/e2e/api/runners/individual/run-newman-routing-wiring-tests.sh new file mode 100755 index 0000000000..79050f1649 --- /dev/null +++ b/tests/e2e/api/runners/individual/run-newman-routing-wiring-tests.sh @@ -0,0 +1,145 @@ +#!/bin/bash + +# Bifrost Routing Wiring Test Runner +# Drives the model-catalog wiring collection against a running Bifrost instance. +# Each scenario stands up an isolated, run-namespaced custom provider backed by a +# real upstream, mutates its providers/keys, and asserts the catalog read +# endpoints reflect every mutation. + +set -e +set -o pipefail + +# Associative arrays (declare -A), SEED_ENV_PATH parsing, and add_env_var_if_set +# all require Bash 4.0+. macOS still ships Bash 3.2 as /bin/bash, which would +# fail partway through with a cryptic "declare: -A: invalid option". +if [ "${BASH_VERSINFO[0]:-0}" -lt 4 ]; then + echo "Error: this script requires Bash 4.0+ (uses 'declare -A seed_env_values' and SEED_ENV_PATH parsing)." >&2 + echo "Detected Bash version: ${BASH_VERSION:-unknown}" >&2 + echo "On macOS, install a newer bash (e.g. 'brew install bash') and run the script with that interpreter." >&2 + exit 1 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +API_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" +cd "$API_DIR" + +COLLECTION="collections/bifrost-routing-wiring.postman_collection.json" +ENVIRONMENT="bifrost-v1.postman_environment.json" +REPORT_DIR="newman-reports/routing-wiring" + +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +NC='\033[0m' + +echo -e "${GREEN}========================================${NC}" +echo -e "${GREEN}Bifrost Routing Wiring Tests${NC}" +echo -e "${GREEN}========================================${NC}" +echo "" + +if ! command -v newman &> /dev/null; then + echo -e "${RED}Error: Newman is not installed${NC}" + echo "Install it with: npm install -g newman newman-reporter-htmlextra" + exit 1 +fi + +if [ ! -f "$COLLECTION" ]; then + echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}" + exit 1 +fi + +ENV_FLAG=() +if [ -f "$ENVIRONMENT" ]; then + ENV_FLAG=(-e "$ENVIRONMENT") +else + echo -e "${YELLOW}Warning: Environment file not found: $ENVIRONMENT (using collection variables only)${NC}" +fi + +# Seed env loading: prefer an explicit BIFROST_E2E_SEED_ENV, else the standard +# generated/seed.env that CI writes. +SEED_ENV_PATH="${BIFROST_E2E_SEED_ENV:-}" +if [ -z "$SEED_ENV_PATH" ] && [ -f "$API_DIR/generated/seed.env" ]; then + SEED_ENV_PATH="$API_DIR/generated/seed.env" +fi + +declare -A seed_env_values +if [ -n "$SEED_ENV_PATH" ] && [ -f "$SEED_ENV_PATH" ]; then + # Parse the seed env as data, never source it — values may contain command + # substitution that sourcing would execute. + while IFS= read -r line || [ -n "$line" ]; do + [[ "$line" =~ ^[[:space:]]*# ]] && continue + [[ -z "${line//[[:space:]]/}" ]] && continue + [[ "$line" != *=* ]] && continue + key="${line%%=*}" + value="${line#*=}" + [[ ! "$key" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] && continue + # Unwrap outer single quotes written by the seed writer and undo its + # '\''-escape; also tolerate plain double-quoted values. + if [[ "$value" == \'*\' ]]; then + value="${value:1:${#value}-2}" + value="${value//\'\"\'\"\'/\'}" + elif [[ "$value" == \"*\" ]]; then + value="${value:1:${#value}-2}" + fi + seed_env_values["$key"]="$value" + done < "$SEED_ENV_PATH" +fi + +mkdir -p "$REPORT_DIR" + +cmd=(newman run "$COLLECTION" "${ENV_FLAG[@]}" -r cli,htmlextra) + +# Forward a seed-env value if present, falling back to the process environment +# so local runs work with credentials exported in the shell. +add_env_var_if_set() { + local name="$1" + local value="${seed_env_values[$name]:-}" + if [ -z "$value" ]; then + value="${!name:-}" + fi + if [ -n "$value" ]; then + cmd+=(--env-var "$name=$value") + fi +} + +# Forward the run-id prefix only. Provider credentials are NOT injected here: +# the collection's keys use Bifrost's `env.` resolution, so each key reads +# its credential from the Bifrost process env at request time (the Bifrost server +# must have the provider env vars, e.g. OPENAI_API_KEY/ANTHROPIC_API_KEY/...). +for v in e2e_seed_prefix; do + add_env_var_if_set "$v" +done + +# The live-model cache populates with one upstream round-trip's latency, and the +# in-collection poll loop can busy-wait up to ~4s per attempt. Give scripts and +# requests generous ceilings. +cmd+=(--timeout-script 120000 --timeout 900000) + +# In CI keep going past failures so every scenario's cleanup folder runs. +ci_normalized="$(printf '%s' "${CI:-}" | tr '[:upper:]' '[:lower:]')" +if [ "$ci_normalized" = "1" ] || [ "$ci_normalized" = "true" ]; then + cmd+=(--reporter-cli-no-failures false) +fi + +echo -e "Collection: ${YELLOW}$COLLECTION${NC}" +echo -e "Reports: ${YELLOW}$REPORT_DIR${NC}" +if [ -n "$SEED_ENV_PATH" ]; then + echo -e "Seed env: ${YELLOW}$SEED_ENV_PATH${NC}" +fi +echo "" +echo -e "${GREEN}Running tests...${NC}" +echo "" + +set +e +"${cmd[@]}" --reporter-htmlextra-export "$REPORT_DIR/report.html" --reporter-htmlextra-title "Bifrost Routing Wiring" +EXIT_CODE=$? +set -e + +echo "" +if [ $EXIT_CODE -eq 0 ]; then + echo -e "${GREEN}✓ All tests passed!${NC}" +else + echo -e "${RED}✗ Some tests failed${NC}" +fi +echo -e "Report: ${YELLOW}$REPORT_DIR/report.html${NC}" +exit $EXIT_CODE diff --git a/tests/e2e/api/runners/lib/collection-builder.mjs b/tests/e2e/api/runners/lib/collection-builder.mjs new file mode 100644 index 0000000000..952f93d5b5 --- /dev/null +++ b/tests/e2e/api/runners/lib/collection-builder.mjs @@ -0,0 +1,241 @@ +// Shared builders for the wiring/routing Postman collection generators. +// +// These emit the run-namespacing, exponential-backoff polling, cleanup-folder, +// and skip-on-missing-credential scaffolding that both the model-catalog and +// routing generators rely on. Everything here produces plain Postman v2.1 JSON; +// the generators add their scenario-specific request bodies and assertions. + +import { writeFileSync } from "node:fs"; + +export const MAX_POLL_ATTEMPTS = 8; + +// --------------------------------------------------------------------------- // +// Collection / folder / request-level event scripts (arrays of JS source lines) +// --------------------------------------------------------------------------- // + +// Build a single run id for the whole run so every created resource is +// namespaced and parallel runs never collide. Runs before every request, so it +// only sets the id once. +export function collectionPrerequest() { + return [ + "if (!pm.collectionVariables.get('run_id')) {", + " var seed = pm.variables.get('e2e_seed_prefix') || pm.environment.get('e2e_seed_prefix') || 'local';", + " var nonce = Date.now().toString(36) + '-' + Math.floor(Math.random() * 1000000).toString(36);", + " pm.collectionVariables.set('run_id', seed + '-' + nonce);", + " console.log('run_id = ' + pm.collectionVariables.get('run_id'));", + "}", + ]; +} + +// Skip the scenario when its required credentials are absent so a developer can +// run with whatever providers they have configured. +export function folderPrerequest(requiredVars) { + return [ + `var required = ${JSON.stringify(requiredVars)};`, + "for (var i = 0; i < required.length; i++) {", + " var v = required[i];", + " if (!pm.variables.get(v) && !pm.environment.get(v)) {", + " console.log('SKIP: missing ' + v);", + " pm.execution.skipRequest();", + " return;", + " }", + "}", + ]; +} + +// Prelude for a polling request: track the attempt counter and (on the first +// attempt) optionally settle for waitSeconds before the read. +export function pollPrerequest(waitSeconds) { + const waitMs = Math.round((waitSeconds || 0) * 1000); + const lines = [ + "var pollKey = '__poll_' + pm.info.requestName;", + "var attempt = parseInt(pm.collectionVariables.get(pollKey) || '0', 10);", + "pm.collectionVariables.set('__cur_poll_key', pollKey);", + "pm.collectionVariables.set('__cur_poll_attempt', String(attempt));", + ]; + if (waitMs > 0) { + lines.push( + `if (attempt === 0) { var __ws = Date.now(); while (Date.now() - __ws < ${waitMs}) {} }` + ); + } + return lines; +} + +// Wrap a throwing assertion in the exponential-backoff retry loop. Only the +// terminal attempt records a pm.test, so Newman's exit code reflects the real +// outcome while intermediate retries stay silent. On terminal failure it jumps +// to the scenario cleanup so a wedged scenario tears down instead of burning +// retries on every remaining step. +export function pollTest(testname, assertLines, cleanupName) { + return [ + `var maxAttempts = ${MAX_POLL_ATTEMPTS};`, + "var pollKey = pm.collectionVariables.get('__cur_poll_key');", + "var attempt = parseInt(pm.collectionVariables.get('__cur_poll_attempt') || '0', 10);", + `var cleanupReq = ${JSON.stringify(cleanupName)};`, + "function assertNow() {", + ...assertLines.map((l) => " " + l), + "}", + "var ok = true, errMsg = '';", + "try { assertNow(); } catch (e) { ok = false; errMsg = e.message; }", + "if (ok) {", + " pm.collectionVariables.set(pollKey, '0');", + ` pm.test(${JSON.stringify(testname)}, function () { pm.expect(true, 'assertion satisfied').to.be.true; });`, + "} else if (attempt < maxAttempts) {", + " pm.collectionVariables.set(pollKey, String(attempt + 1));", + " var sleepMs = Math.min(250 * Math.pow(2, attempt), 4000);", + " var start = Date.now(); while (Date.now() - start < sleepMs) {}", + " pm.execution.setNextRequest(pm.info.requestName);", + "} else {", + " pm.collectionVariables.set(pollKey, '0');", + ` pm.test(${JSON.stringify(testname)}, function () { throw new Error(errMsg); });`, + " pm.execution.setNextRequest(cleanupReq);", + "}", + ]; +} + +// A non-polling mutation assertion: status must be in `acceptable`, else jump +// to cleanup so downstream steps don't run against inconsistent state. +export function mutationTest(testname, acceptable, cleanupName) { + return [ + `var acceptable = ${JSON.stringify(acceptable)};`, + `var cleanupReq = ${JSON.stringify(cleanupName)};`, + `pm.test(${JSON.stringify(testname)}, function () {`, + " pm.expect(acceptable, 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code);", + "});", + "if (acceptable.indexOf(pm.response.code) < 0) { pm.execution.setNextRequest(cleanupReq); }", + ]; +} + +// Cleanup steps accept already-deleted (404) so they pass whether the scenario +// reached its own delete or jumped here mid-flight. +export function cleanupTest(testname) { + return [ + `pm.test(${JSON.stringify(testname)}, function () {`, + " pm.expect([200, 204, 404], 'cleanup status ' + pm.response.code).to.include(pm.response.code);", + "});", + ]; +} + +// A setup folder that deletes every provider before any scenario runs, so the +// suite starts from a clean slate. Standard (non-custom) providers are global +// singletons keyed by name — a leftover `openai`/`azure`/... from a prior or +// interrupted run makes a scenario's create fail with 409 "already exists". +// Clearing up front removes that whole class of cross-run collisions. Scenarios +// recreate every provider they need (keys resolve via Bifrost's env., not +// from any pre-existing provider), so nothing depends on what was there before. +// +// Implemented as a two-request loop: "list" reads all provider names into a +// queue and seeds the first target; "delete" removes the current target, then +// re-points the queue's next entry at itself until the queue drains. +export function clearProvidersFolder() { + const LIST = "setup: list providers to clear"; + const DEL = "setup: delete provider"; + const listTest = [ + "pm.test('setup: list providers', function () { pm.expect(pm.response.code, pm.response.text()).to.equal(200); });", + "if (pm.response.code !== 200) { return; }", + "var provs = ((pm.response.json() || {}).providers || []).map(function (p) { return p.name; }).filter(Boolean);", + "if (provs.length === 0) { pm.collectionVariables.set('__purge_target', ''); pm.collectionVariables.set('__purge_queue', ''); return; }", + "pm.collectionVariables.set('__purge_target', provs[0]);", + "pm.collectionVariables.set('__purge_queue', provs.slice(1).join(','));", + `pm.execution.setNextRequest(${JSON.stringify(DEL)});`, + ]; + const delPre = [ + // No target → nothing to clear (empty instance, or the queue just drained): + // skip this request and fall through to the first scenario. + "if (!pm.collectionVariables.get('__purge_target')) { pm.execution.skipRequest(); }", + ]; + const delTest = [ + "pm.test('setup: delete provider', function () { pm.expect([200, 204, 404], 'status ' + pm.response.code + ' body ' + pm.response.text()).to.include(pm.response.code); });", + "var queue = (pm.collectionVariables.get('__purge_queue') || '').split(',').filter(Boolean);", + "if (queue.length) {", + " pm.collectionVariables.set('__purge_target', queue[0]);", + " pm.collectionVariables.set('__purge_queue', queue.slice(1).join(','));", + ` pm.execution.setNextRequest(${JSON.stringify(DEL)});`, + "} else {", + " pm.collectionVariables.set('__purge_target', '');", + "}", + ]; + return { + id: "setup-clear-providers", + name: "Setup: clear providers", + item: [ + item("setup-list-providers", LIST, request("GET", url(["api", "providers"]), null), events(null, listTest)), + item("setup-delete-provider", DEL, request("DELETE", url(["api", "providers", "{{__purge_target}}"]), null), events(delPre, delTest)), + ], + }; +} + +// --------------------------------------------------------------------------- // +// Postman item primitives +// --------------------------------------------------------------------------- // + +export function url(pathSegments, query) { + let raw = "{{base_url}}/" + pathSegments.join("/"); + if (query && query.length) { + raw += "?" + query.map((q) => `${q.key}=${q.value}`).join("&"); + } + const out = { raw, host: ["{{base_url}}"], path: [...pathSegments] }; + if (query && query.length) out.query = query; + return out; +} + +export function events(prerequest, test) { + const out = []; + if (prerequest) out.push({ listen: "prerequest", script: { type: "text/javascript", exec: prerequest } }); + if (test) out.push({ listen: "test", script: { type: "text/javascript", exec: test } }); + return out; +} + +export function request(method, urlObj, body, headers) { + const req = { method, header: [...(headers || [])], url: urlObj }; + if (body !== undefined && body !== null) { + req.header.push({ key: "Content-Type", value: "application/json" }); + req.body = { mode: "raw", raw: JSON.stringify(body) }; + } + return req; +} + +export function item(id, name, req, evts) { + return { id, name, event: evts, request: req }; +} + +// --------------------------------------------------------------------------- // +// Assembly + output +// --------------------------------------------------------------------------- // + +export function buildCollection({ id, name, description, expandedScenarios, extraVariables }) { + return { + info: { + _postman_id: id, + name, + description, + schema: "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", + }, + variable: [ + { key: "base_url", value: "http://localhost:8080", type: "string" }, + { key: "run_id", value: "", type: "string" }, + { key: "__purge_target", value: "", type: "string" }, + { key: "__purge_queue", value: "", type: "string" }, + ...(extraVariables || []), + ], + event: events(collectionPrerequest(), null), + // The clear-providers setup folder always runs first so every run starts + // from a clean provider slate. + item: [clearProvidersFolder(), ...expandedScenarios], + }; +} + +// Resolve --out from argv, falling back to the generator's default path. +export function resolveOutPath(defaultPath) { + const argv = process.argv.slice(2); + const idx = argv.indexOf("--out"); + if (idx >= 0 && argv[idx + 1]) return argv[idx + 1]; + return defaultPath; +} + +export function writeCollection(outPath, collection, scenarioIds) { + writeFileSync(outPath, JSON.stringify(collection, null, 2) + "\n", "utf8"); + console.log("wrote " + outPath); + console.log("scenarios: " + scenarioIds.length); + for (const sid of scenarioIds) console.log(" - " + sid); +} diff --git a/tests/e2e/api/runners/run-newman-api-tests.sh b/tests/e2e/api/runners/run-newman-api-tests.sh index efe1c5fdae..9196bc136d 100755 --- a/tests/e2e/api/runners/run-newman-api-tests.sh +++ b/tests/e2e/api/runners/run-newman-api-tests.sh @@ -164,7 +164,7 @@ while [[ $# -gt 0 ]]; do echo " (default: ./config.json; also reads BIFROST_CONFIG_PATH env)" echo " --extra-collection

" echo " Merge an additional Postman collection into this API run." - echo " Intended for enterprise-only API e2e coverage from another repo." + echo " Intended for API e2e coverage maintained in another repo." echo " --seed-env

Load generated seed dotenv values and pass them to Newman." echo " --expected

Load generated DAC expected-manifest JSON for assertions." echo " --help Show this help message" diff --git a/tests/integrations/python/config.yml b/tests/integrations/python/config.yml index 1d1be6e3aa..b01310393a 100644 --- a/tests/integrations/python/config.yml +++ b/tests/integrations/python/config.yml @@ -165,6 +165,17 @@ providers: streaming: "gemini-2.5-flash" count_tokens: "claude-sonnet-4-5" video: "veo-3.1-generate-preview" + batch_create: "gemini-2.5-flash" + batch_inline: "gemini-2.5-flash" + batch_file_upload: "gemini-2.5-flash" + batch_list: "gemini-2.5-flash" + batch_retrieve: "gemini-2.5-flash" + batch_cancel: "gemini-2.5-flash" + file_upload: "gemini-2.5-flash" + file_list: "gemini-2.5-flash" + file_retrieve: "gemini-2.5-flash" + file_delete: "gemini-2.5-flash" + file_content: "gemini-2.5-flash" bedrock: chat: "global.anthropic.claude-sonnet-4-20250514-v1:0" vision: "global.anthropic.claude-sonnet-4-20250514-v1:0" @@ -465,18 +476,18 @@ provider_scenarios: langchain_structured_output: true pydantic_structured_output: false # PydanticAI structured output unreliable via Bifrost for Gemini pydanticai_streaming: false # PydanticAI GoogleModel streaming has asyncio issues - batch_file_upload: false # Gemini supports file upload via Files API + batch_file_upload: true # Vertex batches read a gs:// JSONL uploaded via the Files API batch_create: false - batch_list: false - batch_retrieve: false - batch_cancel: false - batch_inline: false # Gemini uses inline requests for batch (synchronous) - batch_s3: false # Gemini does not use S3 for batch - file_upload: false - file_list: false - file_retrieve: false - file_delete: false - file_content: false # Gemini doesn't support direct file download + batch_list: true # Vertex lists batchPredictionJobs for the key's project/region + batch_retrieve: true # Vertex retrieves a batchPredictionJob by id + batch_cancel: true # Vertex cancels a running batchPredictionJob + batch_inline: false # inline requests not wired through the OpenAI integration for Vertex + batch_s3: false # Vertex uses GCS, not S3 + file_upload: true # Vertex uploads files to a customer GCS bucket + file_list: true # Vertex lists files in the GCS bucket + file_retrieve: true # Vertex retrieves GCS object metadata + file_delete: true # Vertex deletes GCS objects + file_content: true # Vertex downloads GCS object content count_tokens: false bedrock: diff --git a/tests/integrations/python/tests/test_google.py b/tests/integrations/python/tests/test_google.py index ec3864b51d..d7fc219bda 100644 --- a/tests/integrations/python/tests/test_google.py +++ b/tests/integrations/python/tests/test_google.py @@ -109,17 +109,34 @@ get_api_key, get_provider_voice, get_provider_voices, + # Vertex batch GCS utilities + get_vertex_batch_dest_uri, + get_vertex_project, + get_vertex_location, + get_bifrost_base_url, + is_vertex_gcs_configured, + skip_if_no_vertex_gcs, + skip_if_no_vertex_native_batch, + stage_vertex_batch_input, skip_if_no_api_key, ) -from .utils.config_loader import get_model +from .utils.config_loader import get_config, get_model from .utils.parametrize import ( format_provider_model, get_cross_provider_params_for_scenario, ) -def get_provider_google_client(provider: str = "gemini", passthrough: bool = False): - """Create Google GenAI client with x-model-provider header for given provider""" +def get_provider_google_client( + provider: str = "gemini", + passthrough: bool = False, + extra_headers: Dict[str, str] | None = None, +): + """Create Google GenAI client with x-model-provider header for given provider. + + extra_headers: optional additional HTTP headers forwarded on every request + (e.g. to carry provider-specific routing/config the SDK doesn't model natively). + """ from .utils.config_loader import get_config, get_integration_url api_key = get_api_key(provider) @@ -135,8 +152,11 @@ def get_provider_google_client(provider: str = "gemini", passthrough: bool = Fal } # Add base URL support, timeout, and x-model-provider header through HttpOptions + headers = {"x-model-provider": provider} + if extra_headers: + headers.update(extra_headers) http_options_kwargs = { - "headers": {"x-model-provider": provider}, + "headers": headers, } if base_url: http_options_kwargs["base_url"] = base_url @@ -148,6 +168,58 @@ def get_provider_google_client(provider: str = "gemini", passthrough: bool = Fal return genai.Client(**client_kwargs) +def get_vertex_job_service_client(): + """Build a native Vertex AI JobServiceClient pointed at the Bifrost gateway. + + Vertex batch prediction is a Vertex-native (aiplatform) API — not the Gemini + Developer batches surface — so these tests use the aiplatform gapic + JobServiceClient with the regional batchPredictionJobs methods, routed through + Bifrost via the gateway base URL. Auth is anonymous because Bifrost injects the + real Vertex credentials from its key config; Bifrost detects Vertex routing from + the /projects/{p}/locations/{l}/... request path. + """ + from google.cloud import aiplatform + from google.api_core.client_options import ClientOptions + from google.auth.credentials import AnonymousCredentials + + # Route through Bifrost's genai integration (the Vertex batch routes are mounted + # under the /genai prefix alongside the other GenAI endpoints). + api_endpoint = get_bifrost_base_url().rstrip("/") + "/genai" + return aiplatform.gapic.JobServiceClient( + client_options=ClientOptions(api_endpoint=api_endpoint), + transport="rest", + credentials=AnonymousCredentials(), + ) + + +def build_vertex_batch_prediction_job( + display_name: str, + model: str, + gcs_source_uri: str, + gcs_destination_output_uri_prefix: str, +) -> Dict[str, Any]: + """Build a native Vertex BatchPredictionJob request body (jsonl GCS in/out). + + Mirrors the official aiplatform create_batch_prediction_job sample. Gemini + publisher models do not require dedicated_resources/machine_spec, so those are + omitted (they apply to custom-trained models). + """ + if "/" not in model: + model = "publishers/google/models/" + model + return { + "display_name": display_name, + "model": model, + "input_config": { + "instances_format": "jsonl", + "gcs_source": {"uris": [gcs_source_uri]}, + }, + "output_config": { + "predictions_format": "jsonl", + "gcs_destination": {"output_uri_prefix": gcs_destination_output_uri_prefix}, + }, + } + + @pytest.fixture def google_client(): """Configure Google GenAI client for testing with default gemini provider""" @@ -1660,6 +1732,211 @@ def test_28_gemini_3_pro_thought_signatures_multi_turn(self, test_config): print("\n✓ Gemini 3 Pro Preview thought signature handling test completed successfully!") + @skip_if_no_api_key("gemini") + @pytest.mark.parametrize("model,expect_image_understood", [ + ("gemini-3-flash-preview", True), + ]) + def test_30_multimodal_function_response_image(self, test_config, model, expect_image_understood): + """Test Case 30: Image returned inside a function response (functionResponse.parts). + + A tool returns a solid red image as multimodal content nested in the function response. + The image must survive Bifrost's Gemini<->Bifrost translation instead of being dropped. + Regression guard for multimodal function_call_output (text + image), and for the + model-version gating Bifrost applies: + + - gemini-3-flash-preview: Bifrost forwards the image as functionResponse.parts (no $ref; + the $ref form is rejected by the Gemini Developer API), so the model sees it -> "red". + - gemini-2.5-flash: multimodal function responses are unsupported (a hard 400 upstream), + so Bifrost drops the image and sends text only. The request must still SUCCEED; the + model just can't see the image. This proves gating prevents a regression. + + Notes: + - No real first turn is needed: we inject the `skip_thought_signature_validator` sentinel + as the function call's thought signature (Gemini 3 requires a signature on tool calls). + - We deliberately do NOT put a {"$ref": ...} in `response` (Developer API rejects it). + """ + from google.genai import types + import base64 + + client = get_provider_google_client(provider="gemini") + + read_image_tool = types.Tool( + function_declarations=[ + { + "name": "read_image", + "description": "Reads an image file from disk and returns it.", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string", "description": "Path to the image file"} + }, + "required": ["path"], + }, + } + ] + ) + + # BASE64_IMAGE is a 64x64 solid red PNG. FunctionResponseBlob.data takes raw bytes; + # the SDK base64-encodes it on the wire (what Bifrost receives). + image_bytes = base64.b64decode(BASE64_IMAGE) + + # Bypass Gemini 3's thought-signature validation for this fabricated multi-turn history. + # The SDK base64-encodes these bytes on the wire; Gemini decodes and matches the sentinel. + skip_sig = b"skip_thought_signature_validator" + + conversation = [ + types.Content( + role="user", + parts=[types.Part(text=( + "Call read_image to read 'photo.png', then tell me the single dominant " + "color of the image it returns. Reply with only the color word." + ))], + ), + types.Content( + role="model", + parts=[types.Part( + function_call=types.FunctionCall( + id="call_1", name="read_image", args={"path": "photo.png"} + ), + thought_signature=skip_sig, + )], + ), + types.Content( + role="user", + parts=[types.Part(function_response=types.FunctionResponse( + id="call_1", + name="read_image", + response={"output": "Here is the image."}, + parts=[types.FunctionResponsePart( + inline_data=types.FunctionResponseBlob( + mime_type="image/png", + display_name="photo.png", + data=image_bytes, + ) + )], + ))], + ), + ] + + response = client.models.generate_content( + model=model, + contents=conversation, + config=types.GenerateContentConfig(tools=[read_image_tool]), + ) + + assert response.candidates, "Response should have candidates" + text = (response.text or "").strip().lower() + print(f"\n[{model}] reply: {text!r}") + assert text, "Model should produce a text reply after the multimodal tool result" + + if expect_image_understood: + assert "red" in text, ( + f"[{model}] model did not identify the tool-returned image color (got: {text!r}). " + "The image was likely dropped during functionResponse translation." + ) + + @skip_if_no_api_key("gemini") + @pytest.mark.parametrize("model,expect_image_understood", [ + ("gemini-2.5-flash", False), + ("gemini-3-flash-preview", True), + ]) + def test_30b_multimodal_function_response_full_workflow(self, test_config, model, expect_image_understood): + """Test Case 30b: Full two-turn multimodal function-calling workflow (mirrors the + Gemini docs example) through Bifrost. + + Unlike test_30 (which fabricates the history), this drives a real round trip: + 1. Send a prompt that triggers the tool -> the model returns a genuine functionCall + (carrying its own thought signature, required by Gemini 3). + 2. Send the tool result back as a multimodal functionResponse: the response references + the image via {"image_ref": {"$ref": ""}} and the bytes live in parts, + exactly like the docs/Vertex example. + 3. The model produces a final answer describing the returned image. + + Bifrost adapts the $ref per provider: it keeps it for Vertex and strips it for the Gemini + Developer API (where the $ref form is an upstream 400 bug), so this works on both. + - gemini-3-flash-preview: image reaches the model -> final answer says "red". + - gemini-2.5-flash: multimodal tool output is unsupported, so Bifrost drops the image and + the request still succeeds (gating prevents a hard 400); we only assert a text reply. + """ + from google.genai import types + import base64 + + client = get_provider_google_client(provider="gemini") + + get_image_declaration = types.FunctionDeclaration( + name="get_image", + description="Retrieves the image file for a specific item.", + parameters={ + "type": "object", + "properties": { + "item_name": { + "type": "string", + "description": "The name or description of the item (e.g., 'red shirt').", + } + }, + "required": ["item_name"], + }, + ) + tool_config = types.Tool(function_declarations=[get_image_declaration]) + + # Turn 1: a prompt that should make the model call get_image + prompt = ( + "Use get_image to retrieve the shirt I ordered, then tell me its single dominant " + "color. Reply with only the color word." + ) + response_1 = client.models.generate_content( + model=format_provider_model("vertex", model), + contents=[prompt], + config=types.GenerateContentConfig(tools=[tool_config]), + ) + + if not getattr(response_1, "function_calls", None): + pytest.skip(f"[{model}] model did not call the tool on turn 1; cannot drive the workflow") + function_call = response_1.function_calls[0] + print(f"\n[{model}] turn 1 called: {function_call.name}({dict(function_call.args)})") + + # BASE64_IMAGE is a 64x64 solid red PNG; the tool "returns" it as multimodal content. + image_bytes = base64.b64decode(BASE64_IMAGE) + function_response_data = {"image_ref": {"$ref": "shirt.png"}} + function_response_multimodal_data = types.FunctionResponsePart( + inline_data=types.FunctionResponseBlob( + mime_type="image/png", + display_name="shirt.png", + data=image_bytes, + ) + ) + + # Turn 2: append the real model turn + the tool result, then ask for the final answer. + history = [ + types.Content(role="user", parts=[types.Part(text=prompt)]), + response_1.candidates[0].content, + types.Content( + role="tool", + parts=[types.Part.from_function_response( + name=function_call.name, + response=function_response_data, + parts=[function_response_multimodal_data], + )], + ), + ] + + response_2 = client.models.generate_content( + model=format_provider_model("vertex", model), + contents=history, + config=types.GenerateContentConfig(tools=[tool_config]), + ) + + assert response_2.candidates, "Turn 2 should have candidates" + text = (response_2.text or "").strip().lower() + print(f"[{model}] final reply: {text!r}") + assert text, "Model should produce a final text reply after the multimodal tool result" + + if expect_image_understood: + assert "red" in text, ( + f"[{model}] model did not identify the tool-returned image color (got: {text!r}). " + "The image was likely dropped during functionResponse translation." + ) + @skip_if_no_api_key("google") @pytest.mark.parametrize("provider,model", get_cross_provider_params_for_scenario("thinking")) def test_29_structured_output_with_thinking(self, google_client, test_config, provider, model): @@ -2841,6 +3118,187 @@ def test_39_batch_e2e_file_api(self, test_config, provider, model): except Exception as e: print(f"Cleanup warning: Failed to delete file: {e}") + # ========================================================================= + # VERTEX AI BATCH API TEST CASES (native aiplatform JobServiceClient) + # + # Vertex batch prediction is a Vertex-native (aiplatform) API, distinct from + # the Gemini Developer batches surface. These tests use the aiplatform gapic + # JobServiceClient with the regional batchPredictionJobs methods, routed + # through Bifrost. Inputs/outputs live in GCS (instances_format / + # predictions_format = jsonl). Requires VERTEX_PROJECT_ID + VERTEX_GCS_BUCKET + # (and ADC for staging the input object); otherwise the tests skip. + # ========================================================================= + + @staticmethod + def _vertex_parent(): + return f"projects/{get_vertex_project()}/locations/{get_vertex_location()}" + + @staticmethod + def _cleanup_vertex_job(client, job_name): + """Best-effort cancel + delete of a native Vertex batch prediction job.""" + if not job_name: + return + try: + client.cancel_batch_prediction_job(name=job_name) + except Exception as e: + print(f"Cleanup info: Could not cancel job: {e}") + try: + client.delete_batch_prediction_job(name=job_name) + except Exception as e: + print(f"Cleanup info: Could not delete job: {e}") + + def test_vertex_batch_create(self, test_config): + """Vertex Batch: create a batch prediction job (jsonl GCS in/out).""" + skip_if_no_vertex_native_batch() + client = get_vertex_job_service_client() + model = get_config().get_provider_model("vertex", "batch_create") + + gcs_source_uri = stage_vertex_batch_input( + create_google_batch_json_content(model=model, num_requests=2) + ) + body = build_vertex_batch_prediction_job( + display_name="bifrost-vertex-batch-create", + model=model, + gcs_source_uri=gcs_source_uri, + gcs_destination_output_uri_prefix=get_vertex_batch_dest_uri(), + ) + + job = None + try: + job = client.create_batch_prediction_job( + parent=self._vertex_parent(), batch_prediction_job=body + ) + assert job.name, "Created job should have a resource name" + print(f"Success: Created Vertex batch job {job.name}, state: {job.state.name}") + finally: + self._cleanup_vertex_job(client, job.name if job else None) + + @skip_if_no_api_key("vertex") + def test_vertex_batch_get(self, test_config): + """Vertex Batch: retrieve a batch prediction job by name.""" + skip_if_no_vertex_native_batch() + client = get_vertex_job_service_client() + model = get_config().get_provider_model("vertex", "batch_retrieve") + + gcs_source_uri = stage_vertex_batch_input( + create_google_batch_json_content(model=model, num_requests=1) + ) + body = build_vertex_batch_prediction_job( + display_name="bifrost-vertex-batch-get", + model=model, + gcs_source_uri=gcs_source_uri, + gcs_destination_output_uri_prefix=get_vertex_batch_dest_uri(), + ) + + job = None + try: + job = client.create_batch_prediction_job( + parent=self._vertex_parent(), batch_prediction_job=body + ) + retrieved = client.get_batch_prediction_job(name=job.name) + assert retrieved.name == job.name, ( + f"Retrieved job name should match: expected {job.name}, got {retrieved.name}" + ) + print(f"Success: Retrieved Vertex batch job {retrieved.name}, state: {retrieved.state.name}") + finally: + self._cleanup_vertex_job(client, job.name if job else None) + + @skip_if_no_api_key("vertex") + def test_vertex_batch_list(self, test_config): + """Vertex Batch: list batch prediction jobs for the project/location.""" + skip_if_no_vertex_native_batch() + client = get_vertex_job_service_client() + model = get_config().get_provider_model("vertex", "batch_create") + + gcs_source_uri = stage_vertex_batch_input( + create_google_batch_json_content(model=model, num_requests=1) + ) + body = build_vertex_batch_prediction_job( + display_name="bifrost-vertex-batch-list", + model=model, + gcs_source_uri=gcs_source_uri, + gcs_destination_output_uri_prefix=get_vertex_batch_dest_uri(), + ) + + job = None + try: + job = client.create_batch_prediction_job( + parent=self._vertex_parent(), batch_prediction_job=body + ) + found = any( + listed.name == job.name + for listed in client.list_batch_prediction_jobs(parent=self._vertex_parent()) + ) + assert found, f"Created job {job.name} should appear in the listing" + print(f"Success: Found created Vertex batch job {job.name} in listing") + finally: + self._cleanup_vertex_job(client, job.name if job else None) + + @skip_if_no_api_key("vertex") + def test_vertex_batch_cancel(self, test_config): + """Vertex Batch: cancel a running batch prediction job.""" + skip_if_no_vertex_native_batch() + client = get_vertex_job_service_client() + model = get_config().get_provider_model("vertex", "batch_cancel") + + gcs_source_uri = stage_vertex_batch_input( + create_google_batch_json_content(model=model, num_requests=2) + ) + body = build_vertex_batch_prediction_job( + display_name="bifrost-vertex-batch-cancel", + model=model, + gcs_source_uri=gcs_source_uri, + gcs_destination_output_uri_prefix=get_vertex_batch_dest_uri(), + ) + + job = None + try: + job = client.create_batch_prediction_job( + parent=self._vertex_parent(), batch_prediction_job=body + ) + client.cancel_batch_prediction_job(name=job.name) + + retrieved = client.get_batch_prediction_job(name=job.name) + assert retrieved.state.name in ("JOB_STATE_CANCELLING", "JOB_STATE_CANCELLED"), ( + f"Job state should be cancelling/cancelled, got {retrieved.state.name}" + ) + print(f"Success: Cancelled Vertex batch job {job.name}, state: {retrieved.state.name}") + finally: + # Already cancelled above; just delete. + if job: + try: + client.delete_batch_prediction_job(name=job.name) + except Exception as e: + print(f"Cleanup info: Could not delete job: {e}") + + @skip_if_no_api_key("vertex") + def test_vertex_batch_delete(self, test_config): + """Vertex Batch: delete a batch prediction job.""" + skip_if_no_vertex_native_batch() + client = get_vertex_job_service_client() + model = get_config().get_provider_model("vertex", "batch_create") + + gcs_source_uri = stage_vertex_batch_input( + create_google_batch_json_content(model=model, num_requests=1) + ) + body = build_vertex_batch_prediction_job( + display_name="bifrost-vertex-batch-delete", + model=model, + gcs_source_uri=gcs_source_uri, + gcs_destination_output_uri_prefix=get_vertex_batch_dest_uri(), + ) + + job = client.create_batch_prediction_job( + parent=self._vertex_parent(), batch_prediction_job=body + ) + # Cancel first so the job is deletable, then delete (returns an LRO). + try: + client.cancel_batch_prediction_job(name=job.name) + except Exception as e: + print(f"Info: Could not cancel before delete: {e}") + client.delete_batch_prediction_job(name=job.name) + print(f"Success: Deleted Vertex batch job {job.name}") + # ========================================================================= # INPUT TOKENS / TOKEN COUNTING TEST CASES # ========================================================================= diff --git a/tests/integrations/python/tests/test_openai.py b/tests/integrations/python/tests/test_openai.py index e7eef3b807..c645ed98a1 100644 --- a/tests/integrations/python/tests/test_openai.py +++ b/tests/integrations/python/tests/test_openai.py @@ -90,6 +90,7 @@ import json import os import time +import uuid from datetime import datetime, timedelta from typing import Any from urllib.parse import quote @@ -178,8 +179,11 @@ get_content_string, get_provider_voice, get_provider_voices, + get_vertex_batch_dest_uri, + get_vertex_gcs_config, mock_tool_response, skip_if_no_api_key, + skip_if_no_vertex_gcs, # Citation utilities assert_valid_openai_annotation, # WebSocket utilities @@ -269,6 +273,43 @@ def get_provider_openai_client(provider, vk_enabled=False): ) +def get_file_storage_config(provider): + """Resolve the storage_config sent to the Files API for a provider. + + Bedrock stores files in S3; Vertex stores them in a customer-owned GCS bucket. + Both are passed to the OpenAI SDK via storage_config (s3 / gcs) and the + returned file id round-trips through every endpoint. Skips the test when the + backing bucket is not configured. + + Returns the storage_config dict to pass via extra_body (upload) / extra_query (list). + """ + if provider == "vertex": + skip_if_no_vertex_gcs() + cfg = get_vertex_gcs_config() + # Use a unique sub-prefix per call so list() returns exactly the files this + # test uploaded. The bucket/prefix is shared with batch-staging objects, and + # GCS list is prefix + lexicographic with a page limit, so a shared prefix can + # page past a freshly uploaded object. Both upload and list in a given test + # reuse the same returned config, so the unique prefix stays consistent. + base = (cfg.get("prefix") or "").rstrip("/") + unique = f"file-api-tests/{uuid.uuid4()}" + prefix = f"{base}/{unique}" if base else unique + return {"gcs": {"bucket": cfg["bucket"], "prefix": prefix}} + + # Default: S3-backed (Bedrock) + settings = get_config().get_integration_settings("bedrock") + s3_bucket = settings.get("s3_bucket") + if not s3_bucket: + pytest.skip("S3 bucket not configured for file tests") + return { + "s3": { + "bucket": s3_bucket, + "region": settings.get("region", "us-west-2"), + "prefix": settings.get("output_s3_prefix", "bifrost-batch-output"), + } + } + + def _wait_for_video_terminal_status( client: OpenAI, video_id: str, @@ -2817,22 +2858,15 @@ def test_40_text_completion_streaming(self, openai_client, test_config): @pytest.mark.parametrize( "provider,model,vk_enabled", - get_cross_provider_params_with_vk_for_scenario("batch_file_upload"), + get_cross_provider_params_with_vk_for_scenario("file_upload"), ) def test_41_file_upload(self, test_config, provider, model, vk_enabled): - """Test Case 41: Upload a file for batch processing""" + """Test Case 41: Direct file upload (S3-backed for Bedrock, GCS-backed for Vertex)""" if provider == "_no_providers_" or model == "_no_model_": - pytest.skip("No providers configured for batch_file_upload scenario") - - # Get S3 settings from config (bedrock uses S3 for file storage) - config = get_config() - integration_settings = config.get_integration_settings("bedrock") - s3_bucket = integration_settings.get("s3_bucket") - s3_region = integration_settings.get("region", "us-west-2") - s3_prefix = integration_settings.get("output_s3_prefix", "bifrost-batch-output") + pytest.skip("No providers configured for file_upload scenario") - if not s3_bucket: - pytest.skip("S3 bucket not configured for file tests") + # Resolve provider-specific storage (S3 for Bedrock, GCS for Vertex) + storage_config = get_file_storage_config(provider) # Get provider-specific client client = get_provider_openai_client(provider, vk_enabled=vk_enabled) @@ -2846,13 +2880,7 @@ def test_41_file_upload(self, test_config, provider, model, vk_enabled): purpose="batch", extra_body={ "provider": provider, - "storage_config": { - "s3": { - "bucket": s3_bucket, - "region": s3_region, - "prefix": s3_prefix, - }, - }, + "storage_config": storage_config, }, ) @@ -2866,13 +2894,7 @@ def test_41_file_upload(self, test_config, provider, model, vk_enabled): list_response = client.files.list( extra_query={ "provider": provider, - "storage_config": { - "s3": { - "bucket": s3_bucket, - "region": s3_region, - "prefix": s3_prefix, - }, - }, + "storage_config": storage_config, } ) assert_valid_file_list_response(list_response, min_count=1) @@ -2900,16 +2922,10 @@ def test_42_file_list(self, test_config, provider, model, vk_enabled): """Test Case 42: List uploaded files""" if provider == "_no_providers_" or model == "_no_model_": - pytest.skip("No providers configured for batch_file_upload scenario") - - config = get_config() - integration_settings = config.get_integration_settings("bedrock") - s3_bucket = integration_settings.get("s3_bucket") - s3_region = integration_settings.get("region", "us-west-2") - s3_prefix = integration_settings.get("output_s3_prefix", "bifrost-batch-output") + pytest.skip("No providers configured for file_list scenario") - if not s3_bucket: - pytest.skip("S3 bucket not configured for file tests") + # Resolve provider-specific storage (S3 for Bedrock, GCS for Vertex) + storage_config = get_file_storage_config(provider) # First upload a file to ensure we have at least one jsonl_content = create_batch_jsonl_content(model=model, num_requests=1, provider=provider) @@ -2921,13 +2937,7 @@ def test_42_file_list(self, test_config, provider, model, vk_enabled): purpose="batch", extra_body={ "provider": provider, - "storage_config": { - "s3": { - "bucket": s3_bucket, - "region": s3_region, - "prefix": s3_prefix, - }, - }, + "storage_config": storage_config, }, ) @@ -2936,13 +2946,7 @@ def test_42_file_list(self, test_config, provider, model, vk_enabled): response = client.files.list( extra_query={ "provider": provider, - "storage_config": { - "s3": { - "bucket": s3_bucket, - "region": s3_region, - "prefix": s3_prefix, - }, - }, + "storage_config": storage_config, } ) @@ -2973,14 +2977,8 @@ def test_43_file_retrieve(self, test_config, provider, model, vk_enabled): if provider == "_no_providers_" or model == "_no_model_": pytest.skip("No providers configured for file_retrieve scenario") - config = get_config() - integration_settings = config.get_integration_settings("bedrock") - s3_bucket = integration_settings.get("s3_bucket") - s3_region = integration_settings.get("region", "us-west-2") - s3_prefix = integration_settings.get("output_s3_prefix", "bifrost-batch-output") - - if not s3_bucket: - pytest.skip("S3 bucket not configured for file tests") + # Resolve provider-specific storage (S3 for Bedrock, GCS for Vertex) + storage_config = get_file_storage_config(provider) # First upload a file jsonl_content = create_batch_jsonl_content(model=model, provider=provider, num_requests=1) @@ -2992,13 +2990,7 @@ def test_43_file_retrieve(self, test_config, provider, model, vk_enabled): purpose="batch", extra_body={ "provider": provider, - "storage_config": { - "s3": { - "bucket": s3_bucket, - "region": s3_region, - "prefix": s3_prefix, - }, - }, + "storage_config": storage_config, }, ) @@ -3028,32 +3020,23 @@ def test_43_file_retrieve(self, test_config, provider, model, vk_enabled): ) def test_44_file_delete(self, test_config, provider, model, vk_enabled): """Test Case 44: Delete an uploaded file""" + if provider == "_no_providers_" or model == "_no_model_": + pytest.skip("No providers configured for file_delete scenario") + + # Resolve provider-specific storage (S3 for Bedrock, GCS for Vertex) + storage_config = get_file_storage_config(provider) + # First upload a file jsonl_content = create_batch_jsonl_content(model=model, provider=provider, num_requests=1) client = get_provider_openai_client(provider, vk_enabled=vk_enabled) - config = get_config() - integration_settings = config.get_integration_settings("bedrock") - s3_bucket = integration_settings.get("s3_bucket") - s3_region = integration_settings.get("region", "us-west-2") - s3_prefix = integration_settings.get("output_s3_prefix", "bifrost-batch-output") - - if not s3_bucket: - pytest.skip("S3 bucket not configured for file tests") - uploaded_file = client.files.create( file=("test_delete.jsonl", jsonl_content.encode(), "application/jsonl"), purpose="batch", extra_body={ "provider": provider, - "storage_config": { - "s3": { - "bucket": s3_bucket, - "region": s3_region, - "prefix": s3_prefix, - }, - }, + "storage_config": storage_config, }, ) @@ -3076,16 +3059,10 @@ def test_45_file_content(self, test_config, provider, model, vk_enabled): """Test Case 45: Download file content""" if provider == "_no_providers_" or model == "_no_model_": - pytest.skip("No providers configured for file_download scenario") - - config = get_config() - integration_settings = config.get_integration_settings("bedrock") - s3_bucket = integration_settings.get("s3_bucket") - s3_region = integration_settings.get("region", "us-west-2") - s3_prefix = integration_settings.get("output_s3_prefix", "bifrost-batch-output") + pytest.skip("No providers configured for file_content scenario") - if not s3_bucket: - pytest.skip("S3 bucket not configured for file tests") + # Resolve provider-specific storage (S3 for Bedrock, GCS for Vertex) + storage_config = get_file_storage_config(provider) # Get provider-specific client client = get_provider_openai_client(provider, vk_enabled=vk_enabled) @@ -3098,13 +3075,7 @@ def test_45_file_content(self, test_config, provider, model, vk_enabled): purpose="batch", extra_body={ "provider": provider, - "storage_config": { - "s3": { - "bucket": s3_bucket, - "region": s3_region, - "prefix": s3_prefix, - }, - }, + "storage_config": storage_config, }, ) @@ -3205,6 +3176,48 @@ def test_46_batch_create_with_file(self, test_config, provider, model, vk_enable print(f"Info: Could not cancel batch (may already be processed): {e}") return + # Vertex uses a customer-owned GCS bucket for input and an output_folder (gs:// prefix) + # instead of Bedrock's S3 role/URI. The uploaded file id is base64-encoded by the + # integration; the batch routes decode it back to gs:// before reaching the provider. + if provider == "vertex": + storage_config = get_file_storage_config(provider) # skips if VERTEX_GCS_BUCKET unset + jsonl_content = create_batch_jsonl_content(model=model, num_requests=2, provider=provider) + uploaded_file = client.files.create( + file=("batch_create_file_test.jsonl", jsonl_content.encode(), "application/jsonl"), + purpose="batch", + extra_body={"provider": provider, "storage_config": storage_config}, + ) + batch = None + try: + batch = client.batches.create( + input_file_id=uploaded_file.id, + endpoint="/v1/chat/completions", + completion_window="24h", + extra_body={ + "provider": provider, + "model": model, + "output_folder": {"url": get_vertex_batch_dest_uri()}, + }, + ) + assert_valid_batch_response(batch) + assert ( + batch.input_file_id == uploaded_file.id + ), f"Input file ID should round-trip: expected {uploaded_file.id}, got {batch.input_file_id}" + print( + f"Success: Created file-based batch with ID: {batch.id}, status: {batch.status} for provider {provider}" + ) + finally: + if batch: + try: + client.batches.cancel(batch.id, extra_body={"provider": provider}) + except Exception as e: + print(f"Info: Could not cancel batch (may already be processed): {e}") + try: + client.files.delete(uploaded_file.id, extra_query={"provider": provider}) + except Exception as e: + print(f"Warning: Failed to clean up file: {e}") + return + # File-based batching for other providers (Bedrock, OpenAI) config = get_config() integration_settings = config.get_integration_settings("bedrock") @@ -3359,6 +3372,49 @@ def test_48_batch_retrieve(self, test_config, provider, model, vk_enabled): pass return + # Vertex: GCS-backed file input + output_folder (gs:// prefix). + if provider == "vertex": + storage_config = get_file_storage_config(provider) # skips if VERTEX_GCS_BUCKET unset + batch_id = None + uploaded_file = None + try: + jsonl_content = create_batch_jsonl_content(model=model, num_requests=1, provider=provider) + uploaded_file = client.files.create( + file=("batch_retrieve_test.jsonl", jsonl_content.encode(), "application/jsonl"), + purpose="batch", + extra_body={"provider": provider, "storage_config": storage_config}, + ) + batch = client.batches.create( + input_file_id=uploaded_file.id, + endpoint="/v1/chat/completions", + completion_window="24h", + extra_body={ + "provider": provider, + "model": model, + "output_folder": {"url": get_vertex_batch_dest_uri()}, + }, + ) + batch_id = batch.id + + retrieved_batch = client.batches.retrieve(batch_id, extra_query={"provider": provider}) + assert_valid_batch_response(retrieved_batch) + assert retrieved_batch.id == batch_id + print( + f"Success: Retrieved batch {batch_id}, status: {retrieved_batch.status} for provider {provider}" + ) + finally: + if batch_id: + try: + client.batches.cancel(batch_id, extra_body={"provider": provider}) + except Exception: + pass + if uploaded_file: + try: + client.files.delete(uploaded_file.id, extra_query={"provider": provider}) + except Exception: + pass + return + # File-based batching for other providers (Bedrock, OpenAI) config = get_config() integration_settings = config.get_integration_settings("bedrock") @@ -3480,6 +3536,42 @@ def test_49_batch_cancel(self, test_config, provider, model, vk_enabled): pass return + # Vertex: GCS-backed file input + output_folder (gs:// prefix). + if provider == "vertex": + storage_config = get_file_storage_config(provider) # skips if VERTEX_GCS_BUCKET unset + uploaded_file = None + try: + jsonl_content = create_batch_jsonl_content(model=model, num_requests=1, provider=provider) + uploaded_file = client.files.create( + file=("batch_cancel_test.jsonl", jsonl_content.encode(), "application/jsonl"), + purpose="batch", + extra_body={"provider": provider, "storage_config": storage_config}, + ) + batch = client.batches.create( + input_file_id=uploaded_file.id, + endpoint="/v1/chat/completions", + completion_window="24h", + extra_body={ + "provider": provider, + "model": model, + "output_folder": {"url": get_vertex_batch_dest_uri()}, + }, + ) + cancelled_batch = client.batches.cancel(batch.id, extra_body={"provider": provider}) + assert cancelled_batch is not None + assert cancelled_batch.id == batch.id + assert cancelled_batch.status in ["cancelling", "cancelled"] + print( + f"Success: Cancelled batch {batch.id}, status: {cancelled_batch.status} for provider {provider}" + ) + finally: + if uploaded_file: + try: + client.files.delete(uploaded_file.id, extra_query={"provider": provider}) + except Exception: + pass + return + # File-based batching for other providers (Bedrock, OpenAI) config = get_config() integration_settings = config.get_integration_settings("bedrock") @@ -3566,6 +3658,70 @@ def test_50_batch_e2e_file_api(self, test_config, provider, model, vk_enabled): if provider == "_no_providers_" or model == "_no_model_": pytest.skip("No providers configured for batch_file_upload scenario") + # Get provider-specific client + client = get_provider_openai_client(provider, vk_enabled=vk_enabled) + + # Vertex: GCS-backed e2e (upload -> create -> poll -> verify in list). Handled before + # the Bedrock S3 config/skip below, since Vertex uses its own GCS bucket. Vertex batches + # are long-running, so we poll a few times but do not wait for a terminal state. + if provider == "vertex": + storage_config = get_file_storage_config(provider) # skips if VERTEX_GCS_BUCKET unset + jsonl_content = create_batch_jsonl_content(model=model, num_requests=2, provider=provider) + print(f"Step 1: Uploading batch input file for provider {provider}...") + uploaded_file = client.files.create( + file=("batch_e2e_file_test.jsonl", jsonl_content.encode(), "application/jsonl"), + purpose="batch", + extra_body={"provider": provider, "storage_config": storage_config}, + ) + assert_valid_file_response(uploaded_file, expected_purpose="batch") + print(f" Uploaded file: {uploaded_file.id}") + batch = None + try: + print("Step 2: Creating batch job with file ID...") + batch = client.batches.create( + input_file_id=uploaded_file.id, + endpoint="/v1/chat/completions", + completion_window="24h", + metadata={"test": "e2e_file", "source": "bifrost-integration-tests"}, + extra_body={ + "provider": provider, + "model": model, + "output_folder": {"url": get_vertex_batch_dest_uri()}, + }, + ) + assert_valid_batch_response(batch) + print(f" Created batch: {batch.id}, status: {batch.status}") + + print("Step 3: Polling batch status...") + for i in range(5): + retrieved_batch = client.batches.retrieve( + batch.id, extra_query={"provider": provider} + ) + print(f" Poll {i+1}: status = {retrieved_batch.status}") + if retrieved_batch.status in ["completed", "failed", "expired", "cancelled"]: + break + time.sleep(2) + + print("Step 4: Verifying batch in list...") + batch_list = client.batches.list(limit=20, extra_query={"provider": provider}) + assert batch.id in [ + b.id for b in batch_list.data + ], f"Batch {batch.id} should be in the batch list" + print(f"Success: File API E2E completed for batch {batch.id} (provider: {provider})") + finally: + if batch: + try: + client.batches.cancel(batch.id, extra_body={"provider": provider}) + print(f"Cleanup: Cancelled batch {batch.id}") + except Exception as e: + print(f"Cleanup info: Could not cancel batch: {e}") + try: + client.files.delete(uploaded_file.id, extra_query={"provider": provider}) + print(f"Cleanup: Deleted file {uploaded_file.id}") + except Exception as e: + print(f"Cleanup info: Could not delete file: {e}") + return + config = get_config() integration_settings = config.get_integration_settings("bedrock") s3_bucket = integration_settings.get("s3_bucket") @@ -3575,9 +3731,6 @@ def test_50_batch_e2e_file_api(self, test_config, provider, model, vk_enabled): if not s3_bucket: pytest.skip("S3 bucket not configured for file tests") - # Get provider-specific client - client = get_provider_openai_client(provider, vk_enabled=vk_enabled) - # Anthropic uses inline requests instead of file-based batching if provider == "anthropic": batch = None diff --git a/tests/integrations/python/tests/utils/common.py b/tests/integrations/python/tests/utils/common.py index 30ca7dc234..dbcfe48c89 100644 --- a/tests/integrations/python/tests/utils/common.py +++ b/tests/integrations/python/tests/utils/common.py @@ -2535,6 +2535,168 @@ def skip_if_no_bedrock_s3(): pytest.skip("Bedrock S3 tests require AWS_S3_BUCKET environment variable") +def get_vertex_gcs_config() -> Dict[str, Optional[str]]: + """ + Get Vertex AI batch GCS configuration from environment variables. + + Vertex batch prediction reads inputs from / writes outputs to Google Cloud Storage, + so a bucket must be provided to exercise the batch API end-to-end. + + Returns: + Dictionary with GCS configuration: + - bucket: GCS bucket name (from VERTEX_GCS_BUCKET) + - prefix: Output object prefix (from VERTEX_GCS_PREFIX or a default) + """ + return { + "bucket": os.environ.get("VERTEX_GCS_BUCKET"), + "prefix": os.environ.get("VERTEX_GCS_PREFIX", "bifrost-batch-tests/"), + } + + +def is_vertex_gcs_configured() -> bool: + """ + Check if Vertex AI batch GCS configuration is available. + + Returns: + True if VERTEX_GCS_BUCKET is set, False otherwise + """ + config = get_vertex_gcs_config() + return config["bucket"] is not None and len(config["bucket"]) > 0 + + +def get_vertex_batch_dest_uri() -> str: + """ + Build the GCS output destination URI (gs:// prefix) for a Vertex batch job. + + Returns: + GCS URI string (e.g., gs://bucket/bifrost-batch-tests/output) + + Raises: + ValueError if VERTEX_GCS_BUCKET is not configured + """ + config = get_vertex_gcs_config() + if not config["bucket"]: + raise ValueError( + "VERTEX_GCS_BUCKET environment variable is required for Vertex batch API" + ) + prefix = (config["prefix"] or "").strip("/") + base = f"gs://{config['bucket']}" + if prefix: + base = f"{base}/{prefix}" + return f"{base}/output" + + +def skip_if_no_vertex_gcs(): + """ + Pytest skip helper for tests requiring Vertex GCS configuration. + Call skip_if_no_vertex_gcs() at the start of a test. + """ + import pytest + + if not is_vertex_gcs_configured(): + pytest.skip("Vertex batch tests require VERTEX_GCS_BUCKET environment variable") + + +def get_vertex_project() -> Optional[str]: + """Vertex project id for native batch prediction (from VERTEX_PROJECT_ID).""" + return os.environ.get("VERTEX_PROJECT_ID") + + +def get_vertex_location() -> str: + """Vertex regional location for native batch prediction (from GOOGLE_LOCATION).""" + return os.environ.get("GOOGLE_LOCATION", "us-central1") + + +def get_bifrost_base_url() -> str: + """Base URL of the Bifrost gateway (from BIFROST_BASE_URL).""" + return os.environ.get("BIFROST_BASE_URL", "http://localhost:8080") + + +def skip_if_no_vertex_native_batch(): + """ + Pytest skip helper for native Vertex batch tests (aiplatform JobServiceClient). + Requires both a project id and a GCS bucket. + """ + import pytest + + if not get_vertex_project(): + pytest.skip("Vertex native batch tests require VERTEX_PROJECT_ID environment variable") + if not is_vertex_gcs_configured(): + pytest.skip("Vertex native batch tests require VERTEX_GCS_BUCKET environment variable") + + +def get_vertex_google_credentials(scopes: Optional[List[str]] = None): + """ + Build google-auth credentials for direct GCS/Vertex calls in tests. + + The Vertex service-account key is provided to Bifrost via VERTEX_CREDENTIALS, which + may hold either the service-account JSON *content* or a path to a JSON file. ADC + (GOOGLE_APPLICATION_CREDENTIALS) only accepts a file path, so when the content is + inlined we must construct credentials explicitly instead of relying on ADC. + + Returns None if no usable credentials are found (caller falls back to ADC). + """ + import json + + from google.oauth2 import service_account + + raw = os.environ.get("VERTEX_CREDENTIALS") or os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + if not raw: + return None + raw = raw.strip() + + if os.path.isfile(raw): + creds = service_account.Credentials.from_service_account_file(raw) + else: + try: + info = json.loads(raw) + except (ValueError, TypeError): + return None + creds = service_account.Credentials.from_service_account_info(info) + + if scopes: + creds = creds.with_scopes(scopes) + return creds + + +def stage_vertex_batch_input(content: str, filename: str | None = None) -> str: + """ + Upload JSONL batch input to GCS and return its gs:// URI. + + Vertex batch prediction reads inputs from Cloud Storage, so the input file must + exist in GCS before creating the job (the native API has no inline mode). + + Args: + content: Newline-delimited JSON batch input + filename: Optional object filename (auto-generated if not provided) + + Returns: + gs:// URI of the uploaded input object + """ + import time + + from google.cloud import storage + + cfg = get_vertex_gcs_config() + if not cfg["bucket"]: + raise ValueError("VERTEX_GCS_BUCKET environment variable is required for Vertex batch API") + + if filename is None: + filename = f"batch-input-{int(time.time())}.jsonl" + prefix = (cfg["prefix"] or "").strip("/") + blob_name = f"{prefix}/input/{filename}" if prefix else f"input/{filename}" + + # Build credentials from VERTEX_CREDENTIALS (JSON content or path); fall back to ADC. + creds = get_vertex_google_credentials( + scopes=["https://www.googleapis.com/auth/cloud-platform"] + ) + client = storage.Client(project=get_vertex_project(), credentials=creds) + bucket = client.bucket(cfg["bucket"]) + blob = bucket.blob(blob_name) + blob.upload_from_string(content, content_type="application/jsonl") + return f"gs://{cfg['bucket']}/{blob_name}" + + def get_content_string_with_summary(response: Any) -> tuple[str, bool]: """ Extract content from response, handling both OpenAI API responses and LangChain AIMessage objects. diff --git a/tests/integrations/python/tests/utils/config_loader.py b/tests/integrations/python/tests/utils/config_loader.py index eed542fb13..394b6eab33 100644 --- a/tests/integrations/python/tests/utils/config_loader.py +++ b/tests/integrations/python/tests/utils/config_loader.py @@ -23,6 +23,7 @@ "pydanticai": "openai", # Pydantic AI defaults to OpenAI "bedrock": "bedrock", # Bedrock defaults to Amazon provider "azure": "azure", + "vertex": "vertex", } @dataclass diff --git a/transports/bifrost-http/handlers/config.go b/transports/bifrost-http/handlers/config.go index dab841caa0..8444d411c7 100644 --- a/transports/bifrost-http/handlers/config.go +++ b/transports/bifrost-http/handlers/config.go @@ -150,26 +150,23 @@ func (h *ConfigHandler) getConfig(ctx *fasthttp.RequestCtx) { } } mapConfig["auth_config"] = map[string]any{ - "admin_username": authConfig.AdminUserName, - "admin_password": passwordEnvVar, - "is_enabled": authConfig.IsEnabled, - "disable_auth_on_inference": authConfig.DisableAuthOnInference, + "admin_username": authConfig.AdminUserName, + "admin_password": passwordEnvVar, + "is_enabled": authConfig.IsEnabled, } } else { // No auth config exists yet, return default empty EnvVar values mapConfig["auth_config"] = map[string]any{ - "admin_username": &schemas.EnvVar{Val: "", EnvVar: "", FromEnv: false}, - "admin_password": &schemas.EnvVar{Val: "", EnvVar: "", FromEnv: false}, - "is_enabled": false, - "disable_auth_on_inference": true, + "admin_username": &schemas.EnvVar{Val: "", EnvVar: "", FromEnv: false}, + "admin_password": &schemas.EnvVar{Val: "", EnvVar: "", FromEnv: false}, + "is_enabled": false, } } } else { mapConfig["auth_config"] = map[string]any{ - "admin_username": &schemas.EnvVar{Val: "", EnvVar: "", FromEnv: false}, - "admin_password": &schemas.EnvVar{Val: "", EnvVar: "", FromEnv: false}, - "is_enabled": false, - "disable_auth_on_inference": true, + "admin_username": &schemas.EnvVar{Val: "", EnvVar: "", FromEnv: false}, + "admin_password": &schemas.EnvVar{Val: "", EnvVar: "", FromEnv: false}, + "is_enabled": false, } } mapConfig["is_db_connected"] = h.store.ConfigStore != nil @@ -288,6 +285,21 @@ func (h *ConfigHandler) updateConfig(ctx *fasthttp.RequestCtx) { return } + // Validate MCP library catalog URL override (only when set and non-default) + if payload.FrameworkConfig.MCPLibraryURL != nil && *payload.FrameworkConfig.MCPLibraryURL != "" && *payload.FrameworkConfig.MCPLibraryURL != modelcatalog.DefaultMCPLibraryURL { + if err := checkURLAccessibility(*payload.FrameworkConfig.MCPLibraryURL); err != nil { + logger.Warn("failed to check the accessibility of the MCP library URL: %v", err) + SendError(ctx, fasthttp.StatusBadRequest, fmt.Sprintf("failed to check the accessibility of the MCP library URL: %v", err)) + return + } + } + // Checking the MCP library sync interval + if payload.FrameworkConfig.MCPLibrarySyncInterval != nil && *payload.FrameworkConfig.MCPLibrarySyncInterval <= 0 { + logger.Warn("MCP library sync interval must be greater than 0") + SendError(ctx, fasthttp.StatusBadRequest, "MCP library sync interval must be greater than 0") + return + } + // Get current config with proper locking currentConfig := h.store.ClientConfig updatedConfig := currentConfig @@ -533,10 +545,12 @@ func (h *ConfigHandler) updateConfig(ctx *fasthttp.RequestCtx) { // if framework config is nil, we will use the default pricing config if frameworkConfig == nil { frameworkConfig = &configstoreTables.TableFrameworkConfig{ - ID: 0, - PricingURL: bifrost.Ptr(modelcatalog.DefaultPricingURL), - PricingSyncInterval: bifrost.Ptr(int64(modelcatalog.DefaultSyncInterval.Seconds())), - ModelParametersURL: bifrost.Ptr(modelcatalog.DefaultModelParametersURL), + ID: 0, + PricingURL: bifrost.Ptr(modelcatalog.DefaultPricingURL), + PricingSyncInterval: bifrost.Ptr(int64(modelcatalog.DefaultSyncInterval.Seconds())), + ModelParametersURL: bifrost.Ptr(modelcatalog.DefaultModelParametersURL), + MCPLibraryURL: bifrost.Ptr(modelcatalog.DefaultMCPLibraryURL), + MCPLibrarySyncInterval: bifrost.Ptr(int64(modelcatalog.DefaultSyncInterval.Seconds())), } } // Handling individual nil cases @@ -549,6 +563,12 @@ func (h *ConfigHandler) updateConfig(ctx *fasthttp.RequestCtx) { if frameworkConfig.ModelParametersURL == nil { frameworkConfig.ModelParametersURL = bifrost.Ptr(modelcatalog.DefaultModelParametersURL) } + if frameworkConfig.MCPLibraryURL == nil { + frameworkConfig.MCPLibraryURL = bifrost.Ptr(modelcatalog.DefaultMCPLibraryURL) + } + if frameworkConfig.MCPLibrarySyncInterval == nil { + frameworkConfig.MCPLibrarySyncInterval = bifrost.Ptr(int64(modelcatalog.DefaultSyncInterval.Seconds())) + } // Updating framework config shouldReloadFrameworkConfig := false if payload.FrameworkConfig.PricingURL != nil && *payload.FrameworkConfig.PricingURL != *frameworkConfig.PricingURL { @@ -584,6 +604,23 @@ func (h *ConfigHandler) updateConfig(ctx *fasthttp.RequestCtx) { shouldReloadFrameworkConfig = true } } + if payload.FrameworkConfig.MCPLibraryURL != nil { + effectiveMCPLibraryURL := *payload.FrameworkConfig.MCPLibraryURL + if effectiveMCPLibraryURL == "" { + effectiveMCPLibraryURL = modelcatalog.DefaultMCPLibraryURL + } + if frameworkConfig.MCPLibraryURL == nil || effectiveMCPLibraryURL != *frameworkConfig.MCPLibraryURL { + frameworkConfig.MCPLibraryURL = &effectiveMCPLibraryURL + shouldReloadFrameworkConfig = true + } + } + if payload.FrameworkConfig.MCPLibrarySyncInterval != nil { + syncInterval := *payload.FrameworkConfig.MCPLibrarySyncInterval + if frameworkConfig.MCPLibrarySyncInterval == nil || syncInterval != *frameworkConfig.MCPLibrarySyncInterval { + frameworkConfig.MCPLibrarySyncInterval = &syncInterval + shouldReloadFrameworkConfig = true + } + } // Reload config if required if shouldReloadFrameworkConfig { var syncSeconds int64 @@ -594,9 +631,11 @@ func (h *ConfigHandler) updateConfig(ctx *fasthttp.RequestCtx) { } h.store.FrameworkConfig = &framework.FrameworkConfig{ Pricing: &modelcatalog.Config{ - PricingURL: frameworkConfig.PricingURL, - PricingSyncInterval: &syncSeconds, - ModelParametersURL: frameworkConfig.ModelParametersURL, + PricingURL: frameworkConfig.PricingURL, + PricingSyncInterval: &syncSeconds, + ModelParametersURL: frameworkConfig.ModelParametersURL, + MCPLibraryURL: frameworkConfig.MCPLibraryURL, + MCPLibrarySyncInterval: frameworkConfig.MCPLibrarySyncInterval, }, } // Saving framework config diff --git a/transports/bifrost-http/handlers/governance.go b/transports/bifrost-http/handlers/governance.go index 52dd9a98b0..897b99e046 100644 --- a/transports/bifrost-http/handlers/governance.go +++ b/transports/bifrost-http/handlers/governance.go @@ -3,10 +3,12 @@ package handlers import ( + "bytes" "context" "encoding/json" "errors" "fmt" + "io" "math" "sort" "strconv" @@ -21,8 +23,11 @@ import ( "github.com/maximhq/bifrost/core/schemas" "github.com/maximhq/bifrost/framework/configstore" configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" + "github.com/maximhq/bifrost/framework/logstore" "github.com/maximhq/bifrost/framework/modelcatalog" "github.com/maximhq/bifrost/plugins/governance" + "github.com/maximhq/bifrost/plugins/governance/complexity" + "github.com/maximhq/bifrost/plugins/logging" "github.com/maximhq/bifrost/transports/bifrost-http/lib" "github.com/valyala/fasthttp" "gorm.io/gorm" @@ -56,6 +61,12 @@ type GovernanceManager interface { DeletePricingOverride(ctx context.Context, id string) error } +type complexityAnalyzerConfigReloader interface { + // HTTP server bridge signature: BifrostHTTPServer implements this and adapts + // to the governance plugin's in-memory ReloadComplexityAnalyzerConfig(config). + ReloadComplexityAnalyzerConfig(ctx context.Context, config *complexity.AnalyzerConfig) error +} + // GovernanceHandler manages HTTP requests for governance operations // ScopeNameResolver returns the human-readable name for a non-global model // config scope target (e.g. a virtual key's Name given its ID). The second @@ -97,13 +108,19 @@ func lookupScopeNameResolver(scope string) (ScopeNameResolver, bool) { type GovernanceHandler struct { configStore configstore.ConfigStore governanceManager GovernanceManager + // logManager sources actual per-model usage (from request logs) for the quota + // endpoint's model_usage breakdown. Optional: nil when the logging plugin is + // not enabled, in which case the breakdown is simply omitted. + logManager logging.LogManager } // NewGovernanceHandler creates a new governance handler instance. +// logManager is optional (may be nil); when supplied it powers the quota +// endpoint's per-budget actual per-model usage breakdown. // Side effect: ensures the default virtual_key scope-name resolver is // registered against the supplied configStore, so resolveModelConfigScopeName // can render VK names for OSS-only builds without further wiring. -func NewGovernanceHandler(manager GovernanceManager, configStore configstore.ConfigStore) (*GovernanceHandler, error) { +func NewGovernanceHandler(manager GovernanceManager, configStore configstore.ConfigStore, logManager logging.LogManager) (*GovernanceHandler, error) { if manager == nil { return nil, fmt.Errorf("governance manager is required") } @@ -120,6 +137,7 @@ func NewGovernanceHandler(manager GovernanceManager, configStore configstore.Con return &GovernanceHandler{ governanceManager: manager, configStore: configStore, + logManager: logManager, }, nil } @@ -951,6 +969,10 @@ type UpdateProviderGovernanceRequest struct { // RegisterRoutes registers all governance-related routes for the new hierarchical system func (h *GovernanceHandler) RegisterRoutes(r *router.Router, middlewares ...schemas.BifrostHTTPMiddleware) { + r.GET("/api/governance/complexity-analyzer-config", lib.ChainMiddlewares(h.getComplexityAnalyzerConfig, middlewares...)) + r.PUT("/api/governance/complexity-analyzer-config", lib.ChainMiddlewares(h.updateComplexityAnalyzerConfig, middlewares...)) + r.POST("/api/governance/complexity-analyzer-config/reset", lib.ChainMiddlewares(h.resetComplexityAnalyzerConfig, middlewares...)) + // Virtual Key CRUD operations r.GET("/api/governance/virtual-keys", lib.ChainMiddlewares(h.getVirtualKeys, middlewares...)) r.POST("/api/governance/virtual-keys", lib.ChainMiddlewares(h.createVirtualKey, middlewares...)) @@ -1008,6 +1030,88 @@ func (h *GovernanceHandler) RegisterRoutes(r *router.Router, middlewares ...sche r.GET("/api/governance/virtual-keys/quota", h.getVirtualKeyQuota) } +func (h *GovernanceHandler) getComplexityAnalyzerConfig(ctx *fasthttp.RequestCtx) { + if h.configStore == nil { + SendError(ctx, fasthttp.StatusServiceUnavailable, "config store not available") + return + } + + cfg, err := h.configStore.GetComplexityAnalyzerConfig(ctx) + if err != nil { + SendError(ctx, fasthttp.StatusInternalServerError, fmt.Sprintf("failed to get complexity analyzer config: %v", err)) + return + } + if cfg == nil { + defaults := complexity.DefaultAnalyzerConfig() + SendJSON(ctx, defaults) + return + } + SendJSON(ctx, cfg) +} + +func (h *GovernanceHandler) updateComplexityAnalyzerConfig(ctx *fasthttp.RequestCtx) { + if h.configStore == nil { + SendError(ctx, fasthttp.StatusServiceUnavailable, "config store not available") + return + } + + var payload complexity.AnalyzerConfig + decoder := json.NewDecoder(bytes.NewReader(ctx.PostBody())) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&payload); err != nil { + SendError(ctx, fasthttp.StatusBadRequest, fmt.Sprintf("invalid request format: %v", err)) + return + } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + SendError(ctx, fasthttp.StatusBadRequest, "invalid request format: multiple JSON values") + return + } + + normalized, err := complexity.ValidateAndNormalize(&payload) + if err != nil { + SendError(ctx, fasthttp.StatusBadRequest, err.Error()) + return + } + + if err := h.configStore.UpdateComplexityAnalyzerConfig(ctx, normalized); err != nil { + SendError(ctx, fasthttp.StatusInternalServerError, fmt.Sprintf("failed to update complexity analyzer config: %v", err)) + return + } + if err := h.reloadComplexityAnalyzerConfig(ctx, normalized); err != nil { + SendError(ctx, fasthttp.StatusInternalServerError, fmt.Sprintf("failed to reload complexity analyzer config in memory: %v, please restart bifrost to sync with the database", err)) + return + } + + SendJSON(ctx, normalized) +} + +func (h *GovernanceHandler) resetComplexityAnalyzerConfig(ctx *fasthttp.RequestCtx) { + if h.configStore == nil { + SendError(ctx, fasthttp.StatusServiceUnavailable, "config store not available") + return + } + + defaults := complexity.DefaultAnalyzerConfig() + if err := h.configStore.UpdateComplexityAnalyzerConfig(ctx, &defaults); err != nil { + SendError(ctx, fasthttp.StatusInternalServerError, fmt.Sprintf("failed to reset complexity analyzer config: %v", err)) + return + } + if err := h.reloadComplexityAnalyzerConfig(ctx, &defaults); err != nil { + SendError(ctx, fasthttp.StatusInternalServerError, fmt.Sprintf("failed to reload complexity analyzer config in memory: %v, please restart bifrost to sync with the database", err)) + return + } + + SendJSON(ctx, defaults) +} + +func (h *GovernanceHandler) reloadComplexityAnalyzerConfig(ctx context.Context, config *complexity.AnalyzerConfig) error { + reloader, ok := h.governanceManager.(complexityAnalyzerConfigReloader) + if !ok { + return fmt.Errorf("governance manager does not support complexity analyzer config reload") + } + return reloader.ReloadComplexityAnalyzerConfig(ctx, config) +} + // Virtual Key CRUD Operations // getVirtualKeys handles GET /api/governance/virtual-keys - Get all virtual keys with relationships @@ -4471,6 +4575,114 @@ func validateRoutingFallbacks(fallbacks []string) error { return nil } +// quotaModelUsage is one entry in the quota endpoint's per-model breakdown: the budgets +// and rate limit (with their current usage) for a specific model governed under this VK. +// Mirrors how provider_configs surface per-provider governance. +type quotaModelUsage struct { + ModelName string `json:"model_name"` + Provider *string `json:"provider,omitempty"` // nil means all providers + Budgets []configstoreTables.TableBudget `json:"budgets,omitempty"` + RateLimit *configstoreTables.TableRateLimit `json:"rate_limit,omitempty"` +} + +// collectVKModelUsage loads the VK-scoped model configs for vk in a single query, then +// (1) reverse-maps the wildcard ("*") configs onto the VK and its provider configs — the +// same hydration hydrateVKGovernance performs — and (2) returns a per-model usage list +// built from the specific-model configs. Surfacing only VK-scoped governance keeps this +// self-service endpoint reporting the key's own usage (global/shared per-model limits are +// intentionally not exposed here). Returns an error on load failure so the endpoint fails +// closed (500) rather than silently returning empty governance — an empty result here is +// indistinguishable from a key that legitimately has no model configs. +func (h *GovernanceHandler) collectVKModelUsage(ctx context.Context, vk *configstoreTables.TableVirtualKey) ([]quotaModelUsage, error) { + mcs, err := h.configStore.GetModelConfigsByScopeAndScopeIDs(ctx, configstoreTables.ModelConfigScopeVirtualKey, []string{vk.ID}) + if err != nil { + logger.Error("failed to load model configs for VK quota: %v", err) + return nil, err + } + + ptrs := make([]*configstoreTables.TableModelConfig, len(mcs)) + for i := range mcs { + ptrs[i] = &mcs[i] + } + applyVKGovernanceFromModelConfigs(vk, buildVKModelConfigIndex(ptrs)) + + models := make([]quotaModelUsage, 0) + for i := range mcs { + mc := &mcs[i] + if mc.ModelName == configstoreTables.ModelConfigAllModels { + continue // wildcard configs are the VK-/provider-level governance handled above + } + models = append(models, quotaModelUsage{ + ModelName: mc.ModelName, + Provider: mc.Provider, + Budgets: mc.Budgets, + RateLimit: mc.RateLimit, + }) + } + return models, nil +} + +// quotaModelSpend is one model's actual usage drawn from request logs (independent of +// whether any governance config exists for that model) within a budget's current cycle. +type quotaModelSpend struct { + Model string `json:"model"` + Provider string `json:"provider,omitempty"` + TotalRequests int64 `json:"total_requests"` + TotalTokens int64 `json:"total_tokens"` + TotalCost float64 `json:"total_cost"` +} + +// quotaBudget is a VK budget plus the actual per-model spend (from request logs) accumulated +// in its current cycle [last_reset, now]. The TableBudget is embedded so the budget's own +// fields (id, max_limit, reset_duration, last_reset, current_usage, …) render flat alongside +// the breakdown — no field is duplicated. The per-model totals reconcile with current_usage +// (both measured since last_reset). models is empty when logging is disabled. +type quotaBudget struct { + configstoreTables.TableBudget + Models []quotaModelSpend `json:"per_model_usage"` +} + +// buildVKBudgetsWithUsage wraps each hydrated VK budget with its per-model actual usage, +// queried from request logs over that budget's current cycle [last_reset, now]. Per-budget +// because a VK's budgets can have independent reset cycles (e.g. daily + monthly). When +// logging is disabled (logManager == nil) the budgets are returned with an empty models list +// — that is the only case where per_model_usage is empty. A log-store query failure instead +// returns an error so the endpoint fails closed (500) rather than reporting empty usage that +// callers cannot distinguish from "logging disabled". Callers must hydrate vk.Budgets (via +// collectVKModelUsage) before calling this. +func (h *GovernanceHandler) buildVKBudgetsWithUsage(ctx context.Context, vk *configstoreTables.TableVirtualKey, now time.Time) ([]quotaBudget, error) { + out := make([]quotaBudget, 0, len(vk.Budgets)) + for i := range vk.Budgets { + b := &vk.Budgets[i] + entry := quotaBudget{TableBudget: *b, Models: []quotaModelSpend{}} + if h.logManager != nil { + start := b.LastReset + ranking, err := h.logManager.GetModelRankings(ctx, &logstore.SearchFilters{ + VirtualKeyIDs: []string{vk.ID}, + StartTime: &start, + EndTime: &now, + }) + if err != nil { + logger.Error("failed to load per-model usage for VK quota (budget %s): %v", b.ID, err) + return nil, err + } + if ranking != nil { + for _, r := range ranking.Rankings { + entry.Models = append(entry.Models, quotaModelSpend{ + Model: r.Model, + Provider: r.Provider, + TotalRequests: r.TotalRequests, + TotalTokens: r.TotalTokens, + TotalCost: r.TotalCost, + }) + } + } + } + out = append(out, entry) + } + return out, nil +} + // getVirtualKeyQuota handles GET /api/governance/virtual-keys/quota // This is a self-service endpoint — no admin auth required. The VK value in the header is the credential. func (h *GovernanceHandler) getVirtualKeyQuota(ctx *fasthttp.RequestCtx) { @@ -4497,13 +4709,30 @@ func (h *GovernanceHandler) getVirtualKeyQuota(ctx *fasthttp.RequestCtx) { return } - h.hydrateVKGovernance(ctx, vk) + // collectVKModelUsage hydrates the wildcard VK/provider governance (in place) and + // returns the configured per-model limits — both from a single VK-scoped model-config load. + // Fail closed: a load error must not degrade to empty governance (it would leave vk.Budgets + // un-hydrated and report "budgets": [], silently hiding configured limits). + models, err := h.collectVKModelUsage(ctx, vk) + if err != nil { + SendError(ctx, 500, "Failed to load model configurations") + return + } + + // Each budget carries its actual per-model spend (from request logs) for the current + // cycle. Must run after collectVKModelUsage, which hydrates vk.Budgets. + budgets, err := h.buildVKBudgetsWithUsage(ctx, vk, time.Now()) + if err != nil { + SendError(ctx, 500, "Failed to load per-model usage") + return + } SendJSON(ctx, map[string]interface{}{ "virtual_key_name": vk.Name, "is_active": vk.IsActiveValue(), - "budgets": vk.Budgets, + "budgets": budgets, "rate_limit": vk.RateLimit, "provider_configs": vk.ProviderConfigs, + "model_configs": models, }) } diff --git a/transports/bifrost-http/handlers/governance_test.go b/transports/bifrost-http/handlers/governance_test.go index da6db83dac..ef9b051352 100644 --- a/transports/bifrost-http/handlers/governance_test.go +++ b/transports/bifrost-http/handlers/governance_test.go @@ -14,7 +14,10 @@ import ( "github.com/maximhq/bifrost/core/schemas" "github.com/maximhq/bifrost/framework/configstore" configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" + "github.com/maximhq/bifrost/framework/logstore" "github.com/maximhq/bifrost/plugins/governance" + "github.com/maximhq/bifrost/plugins/governance/complexity" + "github.com/maximhq/bifrost/plugins/logging" "github.com/valyala/fasthttp" "gorm.io/gorm" ) @@ -124,6 +127,164 @@ func (m *mockRotateGovernanceManager) ReloadVirtualKey(ctx context.Context, id s return m.store.GetVirtualKey(ctx, id) } +type mockComplexityGovernanceManager struct { + GovernanceManager + reloadedConfig *complexity.AnalyzerConfig + reloadCalls int + reloadErr error +} + +func (m *mockComplexityGovernanceManager) ReloadComplexityAnalyzerConfig(_ context.Context, config *complexity.AnalyzerConfig) error { + m.reloadCalls++ + m.reloadedConfig = config + return m.reloadErr +} + +func testComplexityAnalyzerPayload(t *testing.T, cfg complexity.AnalyzerConfig) string { + t.Helper() + body, err := json.Marshal(cfg) + if err != nil { + t.Fatalf("marshal complexity analyzer config: %v", err) + } + return string(body) +} + +func TestComplexityAnalyzerConfigGetReturnsDefaultsWhenUnset(t *testing.T) { + SetLogger(&mockLogger{}) + store := setupPricingOverrideHandlerStore(t) + handler := &GovernanceHandler{ + configStore: store, + governanceManager: &mockComplexityGovernanceManager{}, + } + + ctx := newTestRequestCtx("") + handler.getComplexityAnalyzerConfig(ctx) + + if ctx.Response.StatusCode() != fasthttp.StatusOK { + t.Fatalf("expected status 200, got %d: %s", ctx.Response.StatusCode(), string(ctx.Response.Body())) + } + var resp complexity.AnalyzerConfig + if err := json.Unmarshal(ctx.Response.Body(), &resp); err != nil { + t.Fatalf("unmarshal response: %v", err) + } + if resp.TierBoundaries != complexity.DefaultTierBoundaries() { + t.Fatalf("expected default boundaries, got %+v", resp.TierBoundaries) + } + if len(resp.Keywords.CodeKeywords) == 0 { + t.Fatalf("expected default code keywords") + } +} + +func TestComplexityAnalyzerConfigPutPersistsAndReloads(t *testing.T) { + SetLogger(&mockLogger{}) + store := setupPricingOverrideHandlerStore(t) + manager := &mockComplexityGovernanceManager{} + handler := &GovernanceHandler{ + configStore: store, + governanceManager: manager, + } + + cfg := complexity.DefaultAnalyzerConfig() + cfg.TierBoundaries.SimpleMedium = 0.12 + cfg.TierBoundaries.MediumComplex = 0.34 + cfg.TierBoundaries.ComplexReasoning = 0.78 + cfg.Keywords.CodeKeywords = []string{" Function ", "api", "API"} + + ctx := newTestRequestCtx(testComplexityAnalyzerPayload(t, cfg)) + handler.updateComplexityAnalyzerConfig(ctx) + + if ctx.Response.StatusCode() != fasthttp.StatusOK { + t.Fatalf("expected status 200, got %d: %s", ctx.Response.StatusCode(), string(ctx.Response.Body())) + } + if manager.reloadCalls != 1 { + t.Fatalf("expected one reload, got %d", manager.reloadCalls) + } + if manager.reloadedConfig == nil || manager.reloadedConfig.TierBoundaries.ComplexReasoning != 0.78 { + t.Fatalf("expected reload with normalized config, got %+v", manager.reloadedConfig) + } + + stored, err := store.GetComplexityAnalyzerConfig(context.Background()) + if err != nil { + t.Fatalf("get stored config: %v", err) + } + if stored == nil || len(stored.Keywords.CodeKeywords) != 2 || stored.Keywords.CodeKeywords[0] != "api" { + t.Fatalf("expected normalized stored keywords, got %+v", stored) + } +} + +func TestComplexityAnalyzerConfigPutRejectsInvalidPayloads(t *testing.T) { + SetLogger(&mockLogger{}) + store := setupPricingOverrideHandlerStore(t) + handler := &GovernanceHandler{ + configStore: store, + governanceManager: &mockComplexityGovernanceManager{}, + } + + valid := complexity.DefaultAnalyzerConfig() + validBody := testComplexityAnalyzerPayload(t, valid) + invalidBoundaries := valid + invalidBoundaries.TierBoundaries.MediumComplex = invalidBoundaries.TierBoundaries.SimpleMedium + emptyKeywords := valid + emptyKeywords.Keywords.CodeKeywords = nil + + tests := []struct { + name string + body string + want string + }{ + {name: "unknown field", body: strings.TrimSuffix(validBody, "}") + `,"extra":true}`, want: "unknown field"}, + {name: "multiple json values", body: validBody + `{}`, want: "multiple JSON values"}, + {name: "invalid boundaries", body: testComplexityAnalyzerPayload(t, invalidBoundaries), want: "tier boundaries"}, + {name: "empty keywords", body: testComplexityAnalyzerPayload(t, emptyKeywords), want: "keyword lists must be non-empty"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := newTestRequestCtx(tt.body) + handler.updateComplexityAnalyzerConfig(ctx) + if ctx.Response.StatusCode() != fasthttp.StatusBadRequest { + t.Fatalf("expected status 400, got %d: %s", ctx.Response.StatusCode(), string(ctx.Response.Body())) + } + if !strings.Contains(string(ctx.Response.Body()), tt.want) { + t.Fatalf("expected response to contain %q, got %s", tt.want, string(ctx.Response.Body())) + } + }) + } +} + +func TestComplexityAnalyzerConfigResetPersistsDefaultsAndReloads(t *testing.T) { + SetLogger(&mockLogger{}) + store := setupPricingOverrideHandlerStore(t) + manager := &mockComplexityGovernanceManager{} + handler := &GovernanceHandler{ + configStore: store, + governanceManager: manager, + } + + custom := complexity.DefaultAnalyzerConfig() + custom.TierBoundaries.ComplexReasoning = 0.80 + if err := store.UpdateComplexityAnalyzerConfig(context.Background(), &custom); err != nil { + t.Fatalf("seed custom config: %v", err) + } + + ctx := newTestRequestCtx("") + handler.resetComplexityAnalyzerConfig(ctx) + + if ctx.Response.StatusCode() != fasthttp.StatusOK { + t.Fatalf("expected status 200, got %d: %s", ctx.Response.StatusCode(), string(ctx.Response.Body())) + } + if manager.reloadCalls != 1 { + t.Fatalf("expected one reload, got %d", manager.reloadCalls) + } + stored, err := store.GetComplexityAnalyzerConfig(context.Background()) + if err != nil { + t.Fatalf("get stored config: %v", err) + } + if stored == nil || stored.TierBoundaries != complexity.DefaultTierBoundaries() { + t.Fatalf("expected stored defaults, got %+v", stored) + } +} + func TestApplyVirtualKeyOwnershipUpdatePreservesOmittedAssociation(t *testing.T) { teamID := "team-1" customerID := "customer-1" @@ -1190,14 +1351,16 @@ func TestRotateVirtualKeys_AllFailuresReturnsServerError(t *testing.T) { // mockQuotaConfigStore backs the self-service quota endpoint. It returns a VK from // GetVirtualKeyQuotaByValue (whose direct Budgets/RateLimit are empty post-PR-#3939) -// and serves the VK-scoped wildcard model configs that own the governance, so -// hydrateVKGovernance can reverse-map them onto the response. +// and serves the VK-scoped model configs that own the governance via the bulk query the +// quota path uses — wildcard ("*") configs are reverse-mapped onto the VK/provider +// configs, and specific-model configs surface as the per-model usage breakdown. type mockQuotaConfigStore struct { configstore.ConfigStore - vk *configstoreTables.TableVirtualKey - vkErr error - modelConfigs map[string]*configstoreTables.TableModelConfig - quotaCalls int + vk *configstoreTables.TableVirtualKey + vkErr error + modelConfigs []configstoreTables.TableModelConfig + modelConfigsErr error + quotaCalls int } func (m *mockQuotaConfigStore) GetVirtualKeyQuotaByValue(_ context.Context, _ string) (*configstoreTables.TableVirtualKey, error) { @@ -1208,16 +1371,51 @@ func (m *mockQuotaConfigStore) GetVirtualKeyQuotaByValue(_ context.Context, _ st return cloneTestVirtualKey(m.vk), nil } -func (m *mockQuotaConfigStore) GetModelConfig(_ context.Context, scope string, scopeID *string, modelName string, provider *string) (*configstoreTables.TableModelConfig, error) { - return lookupVKModelConfig(m.modelConfigs, scope, scopeID, modelName, provider) +func (m *mockQuotaConfigStore) GetModelConfigsByScopeAndScopeIDs(_ context.Context, scope string, scopeIDs []string) ([]configstoreTables.TableModelConfig, error) { + if m.modelConfigsErr != nil { + return nil, m.modelConfigsErr + } + want := make(map[string]bool, len(scopeIDs)) + for _, id := range scopeIDs { + want[id] = true + } + var out []configstoreTables.TableModelConfig + for _, mc := range m.modelConfigs { + if mc.Scope == scope && mc.ScopeID != nil && want[*mc.ScopeID] { + out = append(out, mc) + } + } + return out, nil +} + +// mockQuotaLogManager backs the quota endpoint's actual per-model usage breakdown. It +// embeds the LogManager interface (so the dozens of unused methods are satisfied) and +// overrides only GetModelRankings, recording the filters it was called with so tests can +// assert the per-budget cycle window. +type mockQuotaLogManager struct { + logging.LogManager + rankings *logstore.ModelRankingResult + rankErr error + calls []logstore.SearchFilters +} + +func (m *mockQuotaLogManager) GetModelRankings(_ context.Context, filters *logstore.SearchFilters) (*logstore.ModelRankingResult, error) { + if filters != nil { + m.calls = append(m.calls, *filters) + } + if m.rankErr != nil { + return nil, m.rankErr + } + return m.rankings, nil } type quotaResponse struct { VirtualKeyName string `json:"virtual_key_name"` IsActive bool `json:"is_active"` - Budgets []configstoreTables.TableBudget `json:"budgets"` + Budgets []quotaBudget `json:"budgets"` RateLimit *configstoreTables.TableRateLimit `json:"rate_limit"` ProviderConfigs []configstoreTables.TableVirtualKeyProviderConfig `json:"provider_configs"` + Models []quotaModelUsage `json:"model_configs"` } // TestGetVirtualKeyQuota_HydratesBudgetsFromModelConfigs is the regression test for @@ -1230,6 +1428,10 @@ func TestGetVirtualKeyQuota_HydratesBudgetsFromModelConfigs(t *testing.T) { active := true tokenMax := int64(1000) rlID := "rl-vk" + modelTokenMax := int64(500) + modelRLID := "rl-gpt4o" + // Deterministic cycle start so the per-model usage query window is asserted exactly. + cycleStart := time.Date(2026, time.January, 2, 15, 4, 5, 0, time.UTC) store := &mockQuotaConfigStore{ vk: &configstoreTables.TableVirtualKey{ ID: "vk-1", @@ -1240,28 +1442,53 @@ func TestGetVirtualKeyQuota_HydratesBudgetsFromModelConfigs(t *testing.T) { {ID: 7, VirtualKeyID: "vk-1", Provider: "openai"}, }, }, - modelConfigs: map[string]*configstoreTables.TableModelConfig{ - // VK top-level governance (provider == nil). - vkModelConfigIndexKey("vk-1", nil): { - ID: "mc-vk", - Scope: configstoreTables.ModelConfigScopeVirtualKey, + modelConfigs: []configstoreTables.TableModelConfig{ + // VK top-level governance (wildcard, provider == nil). + { + ID: "mc-vk", + Scope: configstoreTables.ModelConfigScopeVirtualKey, + ScopeID: schemas.Ptr("vk-1"), + ModelName: configstoreTables.ModelConfigAllModels, Budgets: []configstoreTables.TableBudget{ - {ID: "b-vk", MaxLimit: 100, CurrentUsage: 30, ResetDuration: "1d"}, + {ID: "b-vk", MaxLimit: 100, CurrentUsage: 30, ResetDuration: "1d", LastReset: cycleStart}, }, RateLimitID: &rlID, RateLimit: &configstoreTables.TableRateLimit{ID: rlID, TokenMaxLimit: &tokenMax, TokenCurrentUsage: 250}, }, - // Per-provider governance (provider == "openai"). - vkModelConfigIndexKey("vk-1", schemas.Ptr("openai")): { - ID: "mc-openai", - Scope: configstoreTables.ModelConfigScopeVirtualKey, + // Per-provider governance (wildcard, provider == "openai"). + { + ID: "mc-openai", + Scope: configstoreTables.ModelConfigScopeVirtualKey, + ScopeID: schemas.Ptr("vk-1"), + ModelName: configstoreTables.ModelConfigAllModels, + Provider: schemas.Ptr("openai"), Budgets: []configstoreTables.TableBudget{ {ID: "b-openai", MaxLimit: 50, CurrentUsage: 10, ResetDuration: "1d"}, }, }, + // Per-model governance (specific model) — surfaces as the per-model usage breakdown. + { + ID: "mc-gpt4o", + Scope: configstoreTables.ModelConfigScopeVirtualKey, + ScopeID: schemas.Ptr("vk-1"), + ModelName: "gpt-4o", + Provider: schemas.Ptr("openai"), + Budgets: []configstoreTables.TableBudget{ + {ID: "b-gpt4o", MaxLimit: 25, CurrentUsage: 7, ResetDuration: "1d"}, + }, + RateLimitID: &modelRLID, + RateLimit: &configstoreTables.TableRateLimit{ID: modelRLID, TokenMaxLimit: &modelTokenMax, TokenCurrentUsage: 120}, + }, }, } - h := &GovernanceHandler{configStore: store} + logMgr := &mockQuotaLogManager{ + rankings: &logstore.ModelRankingResult{ + Rankings: []logstore.ModelRankingWithTrend{ + {ModelRankingEntry: logstore.ModelRankingEntry{Model: "gpt-4o", Provider: "openai", TotalRequests: 12, TotalTokens: 3400, TotalCost: 1.25}}, + }, + }, + } + h := &GovernanceHandler{configStore: store, logManager: logMgr} ctx := &fasthttp.RequestCtx{} ctx.Request.Header.Set("x-bf-vk", "sk-bf-secret") @@ -1295,6 +1522,53 @@ func TestGetVirtualKeyQuota_HydratesBudgetsFromModelConfigs(t *testing.T) { if len(pcBudgets) != 1 || pcBudgets[0].ID != "b-openai" || pcBudgets[0].CurrentUsage != 10 { t.Fatalf("expected hydrated provider budget b-openai (usage 10), got %#v", pcBudgets) } + // Per-model usage: only the specific-model config (gpt-4o) — wildcard configs feed the + // VK/provider governance above and must not leak into the per-model list. + if len(resp.Models) != 1 { + t.Fatalf("expected one per-model usage entry, got %#v", resp.Models) + } + m := resp.Models[0] + if m.ModelName != "gpt-4o" || m.Provider == nil || *m.Provider != "openai" { + t.Fatalf("unexpected per-model identity: name=%q provider=%v", m.ModelName, m.Provider) + } + if len(m.Budgets) != 1 || m.Budgets[0].ID != "b-gpt4o" || m.Budgets[0].CurrentUsage != 7 { + t.Fatalf("expected per-model budget b-gpt4o (usage 7), got %#v", m.Budgets) + } + if m.RateLimit == nil || m.RateLimit.ID != "rl-gpt4o" || m.RateLimit.TokenCurrentUsage != 120 { + t.Fatalf("expected per-model rate limit rl-gpt4o (usage 120), got %#v", m.RateLimit) + } + + // Actual per-model spend from logs is now embedded in each budget. The VK has a single + // budget (b-vk), whose models list breaks down spend over its current cycle. + bu := resp.Budgets[0] + if bu.ID != "b-vk" || bu.ResetDuration != "1d" || bu.CurrentUsage != 30 { + t.Fatalf("unexpected budget envelope: %#v", bu) + } + if len(bu.Models) != 1 { + t.Fatalf("expected one model spend entry, got %#v", bu.Models) + } + spend := bu.Models[0] + if spend.Model != "gpt-4o" || spend.Provider != "openai" || spend.TotalRequests != 12 || spend.TotalTokens != 3400 || spend.TotalCost != 1.25 { + t.Fatalf("unexpected model spend: %#v", spend) + } + // The usage query must be scoped to this VK and windowed to the budget's current cycle. + if len(logMgr.calls) != 1 { + t.Fatalf("expected GetModelRankings called once, got %d", len(logMgr.calls)) + } + call := logMgr.calls[0] + if len(call.VirtualKeyIDs) != 1 || call.VirtualKeyIDs[0] != "vk-1" { + t.Fatalf("expected usage query scoped to vk-1, got %#v", call.VirtualKeyIDs) + } + if call.StartTime == nil || call.EndTime == nil { + t.Fatalf("expected usage query to carry a cycle window, got start=%v end=%v", call.StartTime, call.EndTime) + } + // The window must start exactly at the budget's last reset and end at/after it. + if !call.StartTime.Equal(cycleStart) { + t.Fatalf("expected StartTime=%v (budget last reset), got %v", cycleStart, *call.StartTime) + } + if call.EndTime.Before(cycleStart) { + t.Fatalf("expected EndTime >= StartTime, got start=%v end=%v", *call.StartTime, *call.EndTime) + } } // TestGetVirtualKeyQuota_NoGovernanceReturnsEmpty verifies that a VK without any @@ -1379,6 +1653,68 @@ func TestGetVirtualKeyQuota_NotFoundReturns401(t *testing.T) { } } +// TestGetVirtualKeyQuota_ModelConfigLoadErrorFailsClosed verifies the endpoint returns 500 +// (not a 200 with silently-empty governance) when the model-config lookup fails. Failing +// open here would leave vk.Budgets un-hydrated and report "budgets": [], hiding configured +// limits from a client that reads len(budgets)==0 as "no limits". +func TestGetVirtualKeyQuota_ModelConfigLoadErrorFailsClosed(t *testing.T) { + SetLogger(&mockLogger{}) + + active := true + store := &mockQuotaConfigStore{ + vk: &configstoreTables.TableVirtualKey{ID: "vk-1", Name: "Prod", IsActive: &active}, + modelConfigsErr: errors.New("db down"), + } + h := &GovernanceHandler{configStore: store} + + ctx := &fasthttp.RequestCtx{} + ctx.Request.Header.Set("x-bf-vk", "sk-bf-secret") + + h.getVirtualKeyQuota(ctx) + + if ctx.Response.StatusCode() != 500 { + t.Fatalf("expected status 500 on model-config load error, got %d: %s", ctx.Response.StatusCode(), string(ctx.Response.Body())) + } +} + +// TestGetVirtualKeyQuota_RankingsErrorFailsClosed verifies that a log-store failure fails +// closed (500) rather than returning per_model_usage: [], which is indistinguishable from a +// legitimately empty breakdown (logging disabled). +func TestGetVirtualKeyQuota_RankingsErrorFailsClosed(t *testing.T) { + SetLogger(&mockLogger{}) + + active := true + store := &mockQuotaConfigStore{ + vk: &configstoreTables.TableVirtualKey{ + ID: "vk-1", + Name: "Prod", + IsActive: &active, + }, + modelConfigs: []configstoreTables.TableModelConfig{ + { + ID: "mc-vk", + Scope: configstoreTables.ModelConfigScopeVirtualKey, + ScopeID: schemas.Ptr("vk-1"), + ModelName: configstoreTables.ModelConfigAllModels, + Budgets: []configstoreTables.TableBudget{ + {ID: "b-vk", MaxLimit: 100, CurrentUsage: 30, ResetDuration: "1d"}, + }, + }, + }, + } + logMgr := &mockQuotaLogManager{rankErr: errors.New("log store down")} + h := &GovernanceHandler{configStore: store, logManager: logMgr} + + ctx := &fasthttp.RequestCtx{} + ctx.Request.Header.Set("x-bf-vk", "sk-bf-secret") + + h.getVirtualKeyQuota(ctx) + + if ctx.Response.StatusCode() != 500 { + t.Fatalf("expected status 500 on rankings load error, got %d: %s", ctx.Response.StatusCode(), string(ctx.Response.Body())) + } +} + // TestGetVirtualKeyQuota_EndToEndWithRealStore exercises the full round-trip against // a real (SQLite) config store: create a VK, write its top-level and per-provider // governance as VK-scoped wildcard model configs (the same shape the create path @@ -1389,6 +1725,8 @@ func TestGetVirtualKeyQuota_NotFoundReturns401(t *testing.T) { func TestGetVirtualKeyQuota_EndToEndWithRealStore(t *testing.T) { SetLogger(&mockLogger{}) ctx := context.Background() + // Deterministic cycle start so the per-model usage query window can be asserted exactly. + cycleStart := time.Date(2026, time.January, 2, 15, 4, 5, 0, time.UTC) store, err := configstore.NewConfigStore(ctx, &configstore.Config{ Enabled: true, @@ -1422,7 +1760,7 @@ func TestGetVirtualKeyQuota_EndToEndWithRealStore(t *testing.T) { Scope: configstoreTables.ModelConfigScopeVirtualKey, ScopeID: &scopeID, Budgets: []configstoreTables.TableBudget{ - {ID: "b-vk-e2e", MaxLimit: 100, CurrentUsage: 30, ResetDuration: "1d"}, + {ID: "b-vk-e2e", MaxLimit: 100, CurrentUsage: 30, ResetDuration: "1d", LastReset: cycleStart}, }, } if err := store.CreateModelConfig(ctx, vkMC); err != nil { @@ -1443,8 +1781,31 @@ func TestGetVirtualKeyQuota_EndToEndWithRealStore(t *testing.T) { if err := store.CreateModelConfig(ctx, provMC); err != nil { t.Fatalf("failed to create provider-scoped model config: %v", err) } + // Per-model governance: (scope=virtual_key, model_name='gpt-4o', provider='openai'). + modelMC := &configstoreTables.TableModelConfig{ + ID: "mc-gpt4o-e2e", + ModelName: "gpt-4o", + Scope: configstoreTables.ModelConfigScopeVirtualKey, + ScopeID: &scopeID, + Provider: &openai, + Budgets: []configstoreTables.TableBudget{ + {ID: "b-gpt4o-e2e", MaxLimit: 25, CurrentUsage: 7, ResetDuration: "1d"}, + }, + } + if err := store.CreateModelConfig(ctx, modelMC); err != nil { + t.Fatalf("failed to create model-scoped model config: %v", err) + } - h := &GovernanceHandler{configStore: store} + // Exercise the log-manager path so per_model_usage and the cycle window are covered + // against the real store (mirrors the mocked unit test). + logMgr := &mockQuotaLogManager{ + rankings: &logstore.ModelRankingResult{ + Rankings: []logstore.ModelRankingWithTrend{ + {ModelRankingEntry: logstore.ModelRankingEntry{Model: "gpt-4o", Provider: "openai", TotalRequests: 3, TotalTokens: 900, TotalCost: 0.42}}, + }, + }, + } + h := &GovernanceHandler{configStore: store, logManager: logMgr} // The real store query uses the RequestCtx as a context.Context (Done/Err), which // nil-derefs on a non-Init'd RequestCtx — so initialize it like a live request. @@ -1481,6 +1842,41 @@ func TestGetVirtualKeyQuota_EndToEndWithRealStore(t *testing.T) { if b := pcBudgets[0]; b.ID != "b-openai-e2e" || b.MaxLimit != 50 || b.CurrentUsage != 10 { t.Fatalf("unexpected provider budget values: %#v", b) } + // Per-model usage: only the specific-model config (gpt-4o), not the wildcard configs. + if len(resp.Models) != 1 { + t.Fatalf("expected one per-model usage entry, got %#v", resp.Models) + } + m := resp.Models[0] + if m.ModelName != "gpt-4o" || m.Provider == nil || *m.Provider != "openai" { + t.Fatalf("unexpected per-model identity: name=%q provider=%v", m.ModelName, m.Provider) + } + if len(m.Budgets) != 1 { + t.Fatalf("expected one per-model budget, got %#v", m.Budgets) + } + if b := m.Budgets[0]; b.ID != "b-gpt4o-e2e" || b.MaxLimit != 25 || b.CurrentUsage != 7 { + t.Fatalf("unexpected per-model budget values: %#v", b) + } + // per_model_usage: the VK budget carries the actual per-model spend from the log manager. + if len(resp.Budgets[0].Models) != 1 { + t.Fatalf("expected one per_model_usage entry on the VK budget, got %#v", resp.Budgets[0].Models) + } + if s := resp.Budgets[0].Models[0]; s.Model != "gpt-4o" || s.Provider != "openai" || s.TotalRequests != 3 || s.TotalTokens != 900 || s.TotalCost != 0.42 { + t.Fatalf("unexpected per_model_usage spend: %#v", s) + } + // The usage query must be scoped to this VK and windowed at the budget's last reset. + if len(logMgr.calls) != 1 { + t.Fatalf("expected GetModelRankings called once, got %d", len(logMgr.calls)) + } + call := logMgr.calls[0] + if len(call.VirtualKeyIDs) != 1 || call.VirtualKeyIDs[0] != vkID { + t.Fatalf("expected usage query scoped to %q, got %#v", vkID, call.VirtualKeyIDs) + } + if call.StartTime == nil || !call.StartTime.Equal(cycleStart) { + t.Fatalf("expected StartTime=%v (budget last reset), got %v", cycleStart, call.StartTime) + } + if call.EndTime == nil || call.EndTime.Before(cycleStart) { + t.Fatalf("expected EndTime >= StartTime, got %v", call.EndTime) + } } // TestGetVirtualKeys_PaginatedEndpoint_ResponseShape verifies the JSON response diff --git a/transports/bifrost-http/handlers/inference.go b/transports/bifrost-http/handlers/inference.go index ed840445d9..aaeb895106 100644 --- a/transports/bifrost-http/handlers/inference.go +++ b/transports/bifrost-http/handlers/inference.go @@ -5,6 +5,7 @@ package handlers import ( "context" + "encoding/base64" "encoding/json" "fmt" "io" @@ -42,6 +43,26 @@ func forwardProviderHeadersFromContext(ctx *fasthttp.RequestCtx, bifrostCtx *sch } } +func parseListModelString(model string, defaultProvider schemas.ModelProvider) (schemas.ModelProvider, string) { + provider, parsedModel := schemas.ParseModelString(model, defaultProvider) + if !strings.Contains(model, "/") { + return provider, parsedModel + } + if provider != defaultProvider || parsedModel != model { + return provider, parsedModel + } + + parts := strings.SplitN(model, "/", 2) + if len(parts) == 2 { + normalizedProvider := strings.ToLower(parts[0]) + if schemas.IsKnownProvider(normalizedProvider) { + return schemas.ModelProvider(normalizedProvider), parts[1] + } + } + + return provider, parsedModel +} + // CompletionHandler manages HTTP requests for completion operations type CompletionHandler struct { client *bifrost.Bifrost @@ -56,27 +77,13 @@ func NewInferenceHandler(client *bifrost.Bifrost, config *lib.Config) *Completio } } -// resolveModelAndProvider parses the model string, validates it, and resolves -// the provider via model catalog when no provider prefix is present. Stores -// resolution metadata on the fasthttp context for ConvertToBifrostContext to -// emit the routing engine log. -func resolveModelAndProvider(ctx *fasthttp.RequestCtx, config *lib.Config, model string) (schemas.ModelProvider, string, error) { +// resolveModelAndProvider parses the model string. An empty provider is allowed here — +// the ModelCatalogResolver built-in PreRequestHook plugin fills it in as the last routing +// layer when no other routing plugin (governance routing rules, governance VK LB, enterprise +// LB) picked one. The empty-provider validation in handleRequest/handleStreamRequest catches +// the case where catalog resolution also fails. +func resolveModelAndProvider(_ *fasthttp.RequestCtx, _ *lib.Config, model string) (schemas.ModelProvider, string, error) { provider, modelName := schemas.ParseModelString(model, "") - if modelName == "" { - return "", "", fmt.Errorf("model is required") - } - if provider == "" { - providers := config.GetProvidersForModel(modelName) - if len(providers) == 0 { - return "", "", fmt.Errorf("provider is required in model field (format: provider/model) — no providers found for model %q in model catalog to auto-resolve", modelName) - } - ctx.SetUserValue(lib.FastHTTPUserValueModelCatalogResolution, &lib.ModelCatalogResolution{ - Model: modelName, - ResolvedProvider: providers[0], - AllProviders: providers, - }) - provider = providers[0] - } return provider, modelName, nil } @@ -355,6 +362,7 @@ var batchCreateParamsKnownFields = map[string]bool{ "input_file_id": true, "input_blob": true, "output_folder": true, + "display_name": true, "requests": true, "endpoint": true, "completion_window": true, @@ -589,6 +597,7 @@ type BatchCreateRequest struct { Requests []schemas.BatchRequestItem `json:"requests,omitempty"` // Anthropic-style inline requests InputBlob *string `json:"input_blob,omitempty"` // Azure-style blob storage input OutputFolder *schemas.BatchOutputFolder `json:"output_folder,omitempty"` // Azure-style output destination + DisplayName *string `json:"display_name,omitempty"` // Human-readable job name (e.g. Vertex displayName) Endpoint string `json:"endpoint,omitempty"` // e.g., "/v1/chat/completions" CompletionWindow string `json:"completion_window,omitempty"` // e.g., "24h" Metadata map[string]string `json:"metadata,omitempty"` @@ -873,7 +882,7 @@ func (h *CompletionHandler) listModels(ctx *fasthttp.RequestCtx) { // Add pricing data to the response if len(resp.Data) > 0 && h.config.ModelCatalog != nil { for i, modelEntry := range resp.Data { - provider, modelName := schemas.ParseModelString(modelEntry.ID, "") + provider, modelName := parseListModelString(modelEntry.ID, "") pricingEntry := h.config.ModelCatalog.GetPricingEntryForModel(modelName, provider) if pricingEntry == nil && modelEntry.Alias != nil { // Retry with alias @@ -2787,6 +2796,7 @@ func (h *CompletionHandler) batchCreate(ctx *fasthttp.RequestCtx) { InputFileID: req.InputFileID, InputBlob: req.InputBlob, OutputFolder: req.OutputFolder, + DisplayName: req.DisplayName, Requests: req.Requests, Endpoint: schemas.BatchEndpoint(req.Endpoint), CompletionWindow: req.CompletionWindow, @@ -3016,6 +3026,32 @@ func (h *CompletionHandler) batchResults(ctx *fasthttp.RequestCtx) { SendJSON(ctx, resp) } +// encodeStorageFileID makes a storage-URI file id (gs://...) opaque and path-safe so +// callers can use it in retrieve/delete/content without percent-encoding slashes. +// Non-URI ids (OpenAI/Gemini/Anthropic) pass through unchanged. The response's +// storage_uri still carries the raw gs:// URI for direct use (e.g. inference). +func encodeStorageFileID(id string) string { + if strings.HasPrefix(id, "gs://") { + return base64.RawURLEncoding.EncodeToString([]byte(id)) + } + return id +} + +// decodeStorageFileID reverses encodeStorageFileID. PathUnescape first (harmless for +// the unreserved RawURL alphabet, and tolerant of callers that percent-encode), then +// base64-decode the opaque gs:// id. Raw or percent-encoded gs:// ids passed directly +// also work: PathUnescape yields the gs:// URI and the base64 step is skipped (a +// gs:// string is not valid base64). +func decodeStorageFileID(id string) string { + if unescaped, err := url.PathUnescape(id); err == nil { + id = unescaped + } + if decoded, err := base64.RawURLEncoding.DecodeString(id); err == nil && strings.HasPrefix(string(decoded), "gs://") { + return string(decoded) + } + return id +} + // fileUpload handles POST /v1/files - Upload a file func (h *CompletionHandler) fileUpload(ctx *fasthttp.RequestCtx) { // Parse multipart form @@ -3049,36 +3085,71 @@ func (h *CompletionHandler) fileUpload(ctx *fasthttp.RequestCtx) { } purpose := purposeValues[0] - // Extract file (required) + // Extract file (optional for providers that support resumable uploads, e.g. Vertex/GCS; + // when omitted, the provider mints an upload session URL instead of receiving bytes) + var fileData []byte + var filename string fileHeaders := form.File["file"] - if len(fileHeaders) == 0 { - SendError(ctx, fasthttp.StatusBadRequest, "file is required") - return + if len(fileHeaders) > 0 { + fileHeader := fileHeaders[0] + filename = fileHeader.Filename + + // Open and read the file + file, err := fileHeader.Open() + if err != nil { + logger.Warn("Failed to open uploaded file: %v", err) + SendError(ctx, fasthttp.StatusInternalServerError, "Internal Server Error") + return + } + defer file.Close() + + // Read file data + fileData, err = io.ReadAll(file) + if err != nil { + logger.Warn("Failed to read uploaded file: %v", err) + SendError(ctx, fasthttp.StatusInternalServerError, "Internal Server Error") + return + } + } else if len(form.Value["filename"]) > 0 { + filename = form.Value["filename"][0] } - fileHeader := fileHeaders[0] + // Extract content type (used for resumable upload sessions and stored object metadata) + var contentType *string + if len(form.Value["content_type"]) > 0 && form.Value["content_type"][0] != "" { + contentType = &form.Value["content_type"][0] + } - // Open and read the file - file, err := fileHeader.Open() - if err != nil { - SendError(ctx, fasthttp.StatusInternalServerError, "Internal Server Error") - return + // GCS storage location for Vertex uploads: sent as individual multipart fields, + // parsed into the typed StorageConfig rather than passed opaquely via extra_params. + var storageConfig *schemas.FileStorageConfig + if len(form.Value["gcs_bucket"]) > 0 && form.Value["gcs_bucket"][0] != "" { + gcs := &schemas.GCSStorageConfig{Bucket: form.Value["gcs_bucket"][0]} + if len(form.Value["gcs_prefix"]) > 0 { + gcs.Prefix = form.Value["gcs_prefix"][0] + } + storageConfig = &schemas.FileStorageConfig{GCS: gcs} } - defer file.Close() - // Read file data - fileData, err := io.ReadAll(file) - if err != nil { - SendError(ctx, fasthttp.StatusInternalServerError, "Internal Server Error") - return + // Collect unknown form fields as extra params (multipart — cannot use extractExtraParams which expects JSON). + // gcs_bucket/gcs_prefix are consumed into StorageConfig above; other providers (e.g. Bedrock s3_bucket) still flow through here. + fileUploadKnownFields := map[string]bool{"file": true, "purpose": true, "provider": true, "filename": true, "content_type": true, "gcs_bucket": true, "gcs_prefix": true} + extraParams := map[string]interface{}{} + for k, vals := range form.Value { + if !fileUploadKnownFields[k] && len(vals) > 0 && vals[0] != "" { + extraParams[k] = vals[0] + } } // Build Bifrost file upload request bifrostFileReq := &schemas.BifrostFileUploadRequest{ - Provider: schemas.ModelProvider(provider), - File: fileData, - Filename: fileHeader.Filename, - Purpose: schemas.FilePurpose(purpose), + Provider: schemas.ModelProvider(provider), + File: fileData, + Filename: filename, + Purpose: schemas.FilePurpose(purpose), + ContentType: contentType, + StorageConfig: storageConfig, + ExtraParams: extraParams, } // Convert context @@ -3096,8 +3167,11 @@ func (h *CompletionHandler) fileUpload(ctx *fasthttp.RequestCtx) { return } - if resp != nil && resp.ExtraFields.ProviderResponseHeaders != nil { - forwardProviderHeaders(ctx, resp.ExtraFields.ProviderResponseHeaders) + if resp != nil { + resp.ID = encodeStorageFileID(resp.ID) + if resp.ExtraFields.ProviderResponseHeaders != nil { + forwardProviderHeaders(ctx, resp.ExtraFields.ProviderResponseHeaders) + } } if streamLargeResponseIfActive(ctx, bifrostCtx) { return @@ -3141,13 +3215,35 @@ func (h *CompletionHandler) fileList(ctx *fasthttp.RequestCtx) { order = &s } + // GCS storage location for Vertex listing: parsed into the typed StorageConfig + // rather than passed opaquely via extra_params. + var storageConfig *schemas.FileStorageConfig + if gcsBucket := string(ctx.QueryArgs().Peek("gcs_bucket")); gcsBucket != "" { + storageConfig = &schemas.FileStorageConfig{GCS: &schemas.GCSStorageConfig{ + Bucket: gcsBucket, + Prefix: string(ctx.QueryArgs().Peek("gcs_prefix")), + }} + } + + // Collect unknown query args as extra params. gcs_bucket/gcs_prefix are consumed into + // StorageConfig above; other providers (e.g. Bedrock s3_bucket) still flow through here. + fileListKnownArgs := map[string]bool{"provider": true, "x-model-provider": true, "purpose": true, "limit": true, "after": true, "order": true, "gcs_bucket": true, "gcs_prefix": true} + extraParams := map[string]interface{}{} + ctx.QueryArgs().VisitAll(func(k, v []byte) { + if argKey := string(k); !fileListKnownArgs[argKey] && len(v) > 0 { + extraParams[argKey] = string(v) + } + }) + // Build Bifrost file list request bifrostFileReq := &schemas.BifrostFileListRequest{ - Provider: schemas.ModelProvider(provider), - Purpose: schemas.FilePurpose(purpose), - Limit: limit, - After: after, - Order: order, + Provider: schemas.ModelProvider(provider), + Purpose: schemas.FilePurpose(purpose), + Limit: limit, + After: after, + Order: order, + StorageConfig: storageConfig, + ExtraParams: extraParams, } // Convert context @@ -3165,8 +3261,13 @@ func (h *CompletionHandler) fileList(ctx *fasthttp.RequestCtx) { return } - if resp != nil && resp.ExtraFields.ProviderResponseHeaders != nil { - forwardProviderHeaders(ctx, resp.ExtraFields.ProviderResponseHeaders) + if resp != nil { + for i := range resp.Data { + resp.Data[i].ID = encodeStorageFileID(resp.Data[i].ID) + } + if resp.ExtraFields.ProviderResponseHeaders != nil { + forwardProviderHeaders(ctx, resp.ExtraFields.ProviderResponseHeaders) + } } if streamLargeResponseIfActive(ctx, bifrostCtx) { return @@ -3183,6 +3284,11 @@ func (h *CompletionHandler) fileRetrieve(ctx *fasthttp.RequestCtx) { return } + // Vertex returns an opaque base64(gs://) id so callers don't percent-encode + // slashes in the path; decode it back (falls back to percent-decoding for raw + // or percent-encoded gs:// / s3:// ids passed directly). + fileID = decodeStorageFileID(fileID) + // Get provider from query parameters provider := string(ctx.QueryArgs().Peek("provider")) if provider == "" { @@ -3211,8 +3317,11 @@ func (h *CompletionHandler) fileRetrieve(ctx *fasthttp.RequestCtx) { return } - if resp != nil && resp.ExtraFields.ProviderResponseHeaders != nil { - forwardProviderHeaders(ctx, resp.ExtraFields.ProviderResponseHeaders) + if resp != nil { + resp.ID = encodeStorageFileID(resp.ID) + if resp.ExtraFields.ProviderResponseHeaders != nil { + forwardProviderHeaders(ctx, resp.ExtraFields.ProviderResponseHeaders) + } } if streamLargeResponseIfActive(ctx, bifrostCtx) { return @@ -3229,6 +3338,11 @@ func (h *CompletionHandler) fileDelete(ctx *fasthttp.RequestCtx) { return } + // Vertex returns an opaque base64(gs://) id so callers don't percent-encode + // slashes in the path; decode it back (falls back to percent-decoding for raw + // or percent-encoded gs:// / s3:// ids passed directly). + fileID = decodeStorageFileID(fileID) + // Get provider from query parameters provider := string(ctx.QueryArgs().Peek("provider")) if provider == "" { @@ -3257,8 +3371,11 @@ func (h *CompletionHandler) fileDelete(ctx *fasthttp.RequestCtx) { return } - if resp != nil && resp.ExtraFields.ProviderResponseHeaders != nil { - forwardProviderHeaders(ctx, resp.ExtraFields.ProviderResponseHeaders) + if resp != nil { + resp.ID = encodeStorageFileID(resp.ID) + if resp.ExtraFields.ProviderResponseHeaders != nil { + forwardProviderHeaders(ctx, resp.ExtraFields.ProviderResponseHeaders) + } } if streamLargeResponseIfActive(ctx, bifrostCtx) { return @@ -3275,6 +3392,11 @@ func (h *CompletionHandler) fileContent(ctx *fasthttp.RequestCtx) { return } + // Vertex returns an opaque base64(gs://) id so callers don't percent-encode + // slashes in the path; decode it back (falls back to percent-decoding for raw + // or percent-encoded gs:// / s3:// ids passed directly). + fileID = decodeStorageFileID(fileID) + // Get provider from query parameters provider := string(ctx.QueryArgs().Peek("provider")) if provider == "" { diff --git a/transports/bifrost-http/handlers/mcp.go b/transports/bifrost-http/handlers/mcp.go index 49a4bb9579..41689282df 100644 --- a/transports/bifrost-http/handlers/mcp.go +++ b/transports/bifrost-http/handlers/mcp.go @@ -20,6 +20,7 @@ import ( "github.com/maximhq/bifrost/core/schemas" "github.com/maximhq/bifrost/framework/configstore" configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" + "github.com/maximhq/bifrost/framework/modelcatalog" "github.com/maximhq/bifrost/transports/bifrost-http/lib" "github.com/valyala/fasthttp" "gorm.io/gorm" @@ -69,6 +70,11 @@ func NewMCPHandler(mcpManager MCPManager, governanceManager GovernanceManager, c // RegisterRoutes registers all MCP-related routes func (h *MCPHandler) RegisterRoutes(r *router.Router, middlewares ...schemas.BifrostHTTPMiddleware) { r.GET("/api/mcp/clients", lib.ChainMiddlewares(h.getMCPClients, middlewares...)) + r.GET("/api/mcp/library", lib.ChainMiddlewares(h.getMCPLibrary, middlewares...)) + r.GET("/api/mcp/library/filterdata", lib.ChainMiddlewares(h.getMCPLibraryFilterData, middlewares...)) + r.POST("/api/mcp/library/force-sync", lib.ChainMiddlewares(h.forceSyncMCPLibrary, middlewares...)) + r.POST("/api/mcp/library", lib.ChainMiddlewares(h.createMCPLibraryEntry, middlewares...)) + r.DELETE("/api/mcp/library/{id}", lib.ChainMiddlewares(h.deleteMCPLibraryEntry, middlewares...)) r.POST("/api/mcp/client", lib.ChainMiddlewares(h.addMCPClient, middlewares...)) r.PUT("/api/mcp/client/{id}", lib.ChainMiddlewares(h.updateMCPClient, middlewares...)) r.DELETE("/api/mcp/client/{id}", lib.ChainMiddlewares(h.deleteMCPClient, middlewares...)) @@ -112,6 +118,132 @@ func (h *MCPHandler) getMCPClients(ctx *fasthttp.RequestCtx) { h.getMCPClientsPaginated(ctx, limitStr, offsetStr, searchStr) } +// getMCPLibrary handles GET /api/mcp/library — paginated, searchable, filterable +// listing of the synced MCP server catalog. All query parameters are optional. +func (h *MCPHandler) getMCPLibrary(ctx *fasthttp.RequestCtx) { + emptyResponse := map[string]interface{}{ + "servers": []configstoreTables.TableMCPLibrary{}, + "count": 0, + "total_count": 0, + "limit": 0, + "offset": 0, + } + if h.store.ConfigStore == nil { + SendJSON(ctx, emptyResponse) + return + } + + params := configstore.MCPLibraryQueryParams{ + Search: string(ctx.QueryArgs().Peek("search")), + Categories: parseCommaSeparated(string(ctx.QueryArgs().Peek("category"))), + ConnectionTypes: parseCommaSeparated(string(ctx.QueryArgs().Peek("connection_type"))), + AuthTypes: parseCommaSeparated(string(ctx.QueryArgs().Peek("auth_type"))), + Tags: parseCommaSeparated(string(ctx.QueryArgs().Peek("tags"))), + SortBy: string(ctx.QueryArgs().Peek("sort_by")), + Order: string(ctx.QueryArgs().Peek("order")), + } + + if limitStr := string(ctx.QueryArgs().Peek("limit")); limitStr != "" { + n, err := strconv.Atoi(limitStr) + if err != nil { + SendError(ctx, 400, "Invalid limit parameter: must be a number") + return + } + if n < 0 { + SendError(ctx, 400, "Invalid limit parameter: must be non-negative") + return + } + params.Limit = n + } + if offsetStr := string(ctx.QueryArgs().Peek("offset")); offsetStr != "" { + n, err := strconv.Atoi(offsetStr) + if err != nil { + SendError(ctx, 400, "Invalid offset parameter: must be a number") + return + } + if n < 0 { + SendError(ctx, 400, "Invalid offset parameter: must be non-negative") + return + } + params.Offset = n + } + params.Limit, params.Offset = ClampPaginationParams(params.Limit, params.Offset) + + entries, totalCount, err := h.store.ConfigStore.GetMCPLibraryPaginated(ctx, params) + if err != nil { + logger.Error("failed to retrieve MCP library entries: %v", err) + SendError(ctx, 500, "Failed to retrieve MCP library entries") + return + } + + SendJSON(ctx, map[string]interface{}{ + "servers": entries, + "count": len(entries), + "total_count": totalCount, + "limit": params.Limit, + "offset": params.Offset, + }) +} + +// getMCPLibraryFilterData handles GET /api/mcp/library/filterdata — returns the +// distinct facet values (categories, connection types, auth types, tags) that +// drive the MCP library filter sidebar. +func (h *MCPHandler) getMCPLibraryFilterData(ctx *fasthttp.RequestCtx) { + emptyResponse := configstore.MCPLibraryFilterData{ + Categories: []string{}, + ConnectionTypes: []string{}, + AuthTypes: []string{}, + Tags: []string{}, + } + if h.store.ConfigStore == nil { + SendJSON(ctx, emptyResponse) + return + } + + data, err := h.store.ConfigStore.GetMCPLibraryFilterData(ctx) + if err != nil { + logger.Error("failed to retrieve MCP library filter data: %v", err) + SendError(ctx, 500, "Failed to retrieve MCP library filter data") + return + } + SendJSON(ctx, data) +} + +// forceSyncMCPLibrary handles POST /api/mcp/library/force-sync — triggers an +// immediate sync of the MCP server library catalog from the configured source. +// Mirrors ConfigHandler.forceSyncPricing → ForceReloadPricing. +func (h *MCPHandler) forceSyncMCPLibrary(ctx *fasthttp.RequestCtx) { + if h.store.ConfigStore == nil { + SendError(ctx, fasthttp.StatusServiceUnavailable, "config store not available") + return + } + + var count int + var err error + if h.store.ModelCatalog != nil { + count, err = h.store.ModelCatalog.ForceReloadMCPLibrary(ctx) + } else { + // Resolve the effective MCP library URL from framework config (DB → file → default). + mcpLibraryURL := modelcatalog.DefaultMCPLibraryURL + if h.store.FrameworkConfig != nil && h.store.FrameworkConfig.Pricing != nil && h.store.FrameworkConfig.Pricing.MCPLibraryURL != nil { + if u := *h.store.FrameworkConfig.Pricing.MCPLibraryURL; u != "" { + mcpLibraryURL = u + } + } + count, err = modelcatalog.SyncMCPLibrary(ctx, mcpLibraryURL, h.store.ConfigStore) + } + if err != nil { + logger.Error("failed to sync MCP library: %v", err) + SendError(ctx, fasthttp.StatusInternalServerError, fmt.Sprintf("failed to sync MCP library: %v", err)) + return + } + + SendJSON(ctx, map[string]any{ + "status": "success", + "message": fmt.Sprintf("MCP library sync completed, %d entries synced", count), + }) +} + // getMCPClientsPaginated handles the paginated path for GET /api/mcp/clients func (h *MCPHandler) getMCPClientsPaginated(ctx *fasthttp.RequestCtx, limitStr, offsetStr, searchStr string) { params := configstore.MCPClientsQueryParams{ @@ -1746,3 +1878,152 @@ func perUserHeaderKeysAdded(oldKeys, newKeys []string) bool { } return false } + +// CreateMCPLibraryEntryRequest is the body for POST /api/mcp/library. It carries +// the user-supplied fields of a custom library entry; DB-managed fields (id, +// slug, source, timestamps) are derived server-side. The slug is generated from +// Name, and the unique slug index enforces no-duplicate-name. +type CreateMCPLibraryEntryRequest struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + Category string `json:"category,omitempty"` + ConnectionType schemas.MCPConnectionType `json:"connection_type"` + ConnectionURL string `json:"connection_url,omitempty"` + StdioConfig *schemas.MCPStdioConfig `json:"stdio_config,omitempty"` + AuthType schemas.MCPAuthType `json:"auth_type,omitempty"` + RequiredHeaderKeys []string `json:"required_header_keys,omitempty"` + IconURL string `json:"icon_url,omitempty"` + DocsURL string `json:"docs_url,omitempty"` + Publisher string `json:"publisher,omitempty"` + Tags []string `json:"tags,omitempty"` +} + +// createMCPLibraryEntry handles POST /api/mcp/library — publishes an org-internal +// ("custom") MCP server into the library so other members can discover and +// install it. The entry is protected from the remote sync (see Source/skip-set +// in SyncMCPLibrary). A duplicate name (same generated slug) returns 409. +func (h *MCPHandler) createMCPLibraryEntry(ctx *fasthttp.RequestCtx) { + if h.store.ConfigStore == nil { + SendError(ctx, fasthttp.StatusServiceUnavailable, "MCP operations unavailable: config store is disabled") + return + } + + var req CreateMCPLibraryEntryRequest + if err := json.Unmarshal(ctx.PostBody(), &req); err != nil { + SendError(ctx, fasthttp.StatusBadRequest, fmt.Sprintf("Invalid request format: %v", err)) + return + } + + req.Name = strings.TrimSpace(req.Name) + if req.Name == "" { + SendError(ctx, fasthttp.StatusBadRequest, "name is required") + return + } + + // Validate connection type and the matching connection field. + switch req.ConnectionType { + case schemas.MCPConnectionTypeHTTP, schemas.MCPConnectionTypeSSE: + if strings.TrimSpace(req.ConnectionURL) == "" { + SendError(ctx, fasthttp.StatusBadRequest, "connection_url is required for http/sse connection types") + return + } + case schemas.MCPConnectionTypeSTDIO: + if req.StdioConfig == nil || strings.TrimSpace(req.StdioConfig.Command) == "" { + SendError(ctx, fasthttp.StatusBadRequest, "stdio_config.command is required for stdio connection type") + return + } + default: + SendError(ctx, fasthttp.StatusBadRequest, "connection_type must be one of: http, stdio, sse") + return + } + + // Default and validate auth type. + if req.AuthType == "" { + req.AuthType = schemas.MCPAuthTypeNone + } + switch req.AuthType { + case schemas.MCPAuthTypeNone, schemas.MCPAuthTypeHeaders, schemas.MCPAuthTypeOauth, + schemas.MCPAuthTypePerUserOauth, schemas.MCPAuthTypePerUserHeaders: + default: + SendError(ctx, fasthttp.StatusBadRequest, "invalid auth_type") + return + } + + slug := modelcatalog.Slugify(req.Name) + if slug == "" { + SendError(ctx, fasthttp.StatusBadRequest, "name must contain at least one alphanumeric character") + return + } + + now := time.Now() + entry := &configstoreTables.TableMCPLibrary{ + Slug: slug, + Name: req.Name, + Description: req.Description, + Category: req.Category, + ConnectionType: req.ConnectionType, + ConnectionURL: req.ConnectionURL, + StdioConfig: req.StdioConfig, + AuthType: req.AuthType, + RequiredHeaderKeys: req.RequiredHeaderKeys, + IconURL: req.IconURL, + DocsURL: req.DocsURL, + Publisher: req.Publisher, + Tags: req.Tags, + Source: "custom", + CreatedAt: now, + UpdatedAt: now, + } + + if err := h.store.ConfigStore.CreateCustomMCPLibraryEntry(ctx, entry); err != nil { + if errors.Is(err, configstore.ErrAlreadyExists) { + SendError(ctx, fasthttp.StatusConflict, "an MCP library server with this name already exists") + return + } + logger.Error("failed to create custom MCP library entry: %v", err) + SendError(ctx, fasthttp.StatusInternalServerError, "Failed to create MCP library entry") + return + } + + SendJSON(ctx, map[string]any{ + "status": "success", + "message": "MCP library server published successfully", + "entry": entry, + }) +} + +// deleteMCPLibraryEntry handles DELETE /api/mcp/library/{id} — soft-deletes a +// library entry (remote or custom) by numeric ID. The row is hidden from +// listings and the remote sync respects the tombstone, so a hidden remote entry +// is never resurrected. Also the escape hatch for a duplicate-name lockout. +func (h *MCPHandler) deleteMCPLibraryEntry(ctx *fasthttp.RequestCtx) { + if h.store.ConfigStore == nil { + SendError(ctx, fasthttp.StatusServiceUnavailable, "MCP operations unavailable: config store is disabled") + return + } + idStr, err := getIDFromCtx(ctx) + if err != nil { + SendError(ctx, fasthttp.StatusBadRequest, fmt.Sprintf("Invalid id: %v", err)) + return + } + id, err := strconv.ParseUint(idStr, 10, 64) + if err != nil { + SendError(ctx, fasthttp.StatusBadRequest, "id must be a positive integer") + return + } + + if err := h.store.ConfigStore.SoftDeleteMCPLibraryEntry(ctx, uint(id)); err != nil { + if errors.Is(err, configstore.ErrNotFound) { + SendError(ctx, fasthttp.StatusNotFound, "MCP library entry not found") + return + } + logger.Error("failed to soft-delete MCP library entry %d: %v", id, err) + SendError(ctx, fasthttp.StatusInternalServerError, "Failed to delete MCP library entry") + return + } + + SendJSON(ctx, map[string]any{ + "status": "success", + "message": "MCP library server removed successfully", + }) +} diff --git a/transports/bifrost-http/handlers/middlewares.go b/transports/bifrost-http/handlers/middlewares.go index 42bf86b86e..3eb4ddfbfc 100644 --- a/transports/bifrost-http/handlers/middlewares.go +++ b/transports/bifrost-http/handlers/middlewares.go @@ -722,12 +722,13 @@ func isRealtimeTransportEndpoint(path string) bool { // AuthMiddleware is a middleware that handles authentication for the API. type AuthMiddleware struct { - store configstore.ConfigStore - whitelistedRoutes atomic.Pointer[[]string] - authConfig atomic.Pointer[configstore.AuthConfig] - wsTicketStore *WSTicketStore - tempTokensService *temptoken.Service // optional; when nil, temp-token fallback is disabled - tempTokensEnabled atomic.Bool + store configstore.ConfigStore + whitelistedRoutes atomic.Pointer[[]string] + authConfig atomic.Pointer[configstore.AuthConfig] + wsTicketStore *WSTicketStore + tempTokensService *temptoken.Service // optional; when nil, temp-token fallback is disabled + tempTokensEnabled atomic.Bool + enforceAuthOnInference atomic.Bool } // InitAuthMiddleware initializes the auth middleware. The tempTokens service @@ -755,10 +756,12 @@ func InitAuthMiddleware(store configstore.ConfigStore, wsTicketStore *WSTicketSt if err == nil && clientConfig != nil { am.whitelistedRoutes.Store(&clientConfig.WhitelistedRoutes) am.tempTokensEnabled.Store(clientConfig.MCPEnableTempTokenAuth) + am.enforceAuthOnInference.Store(clientConfig.EnforceAuthOnInference) } else { emptyRoutes := []string{} am.whitelistedRoutes.Store(&emptyRoutes) am.tempTokensEnabled.Store(false) + am.enforceAuthOnInference.Store(false) } return am, nil @@ -778,6 +781,11 @@ func (m *AuthMiddleware) UpdateTempTokenAuthEnabled(enabled bool) { m.tempTokensEnabled.Store(enabled) } +// UpdateEnforceAuthOnInference updates whether auth is enforced on inference endpoints. +func (m *AuthMiddleware) UpdateEnforceAuthOnInference(enforce bool) { + m.enforceAuthOnInference.Store(enforce) +} + // tryTempTokenOrUnauthorized is the last-resort auth path: a request that // failed every conventional credential check (no Authorization header, no // valid cookie) is given one more chance to present an X-Bifrost-Temp-Token @@ -806,10 +814,13 @@ func (m *AuthMiddleware) tryTempTokenOrUnauthorized(ctx *fasthttp.RequestCtx, ne SendError(ctx, fasthttp.StatusUnauthorized, "Unauthorized") } -// InferenceMiddleware is for inference requests (including MCP routes) if authConfig is set, it will skip authentication if disableAuthOnInference is true. +// InferenceMiddleware is for inference requests (including MCP routes). It skips +// authentication when the ClientConfig.EnforceAuthOnInference switch is disabled. +// That switch is config/Helm-driven and survives restarts. Inference auth is open +// by default (raw binary); the Helm chart enables enforcement for production. func (m *AuthMiddleware) InferenceMiddleware() schemas.BifrostHTTPMiddleware { return m.middleware(func(authConfig *configstore.AuthConfig, url string) bool { - return authConfig.DisableAuthOnInference + return !m.enforceAuthOnInference.Load() }) } @@ -897,7 +908,7 @@ func (m *AuthMiddleware) middleware(shouldSkip func(*configstore.AuthConfig, str next(ctx) return } - if isRealtimeTransportEndpoint(string(ctx.Path())) { + if isRealtimeTransportEndpoint(url) { next(ctx) return } diff --git a/transports/bifrost-http/handlers/middlewares_test.go b/transports/bifrost-http/handlers/middlewares_test.go index 7290173507..ea2c43dcc2 100644 --- a/transports/bifrost-http/handlers/middlewares_test.go +++ b/transports/bifrost-http/handlers/middlewares_test.go @@ -738,6 +738,9 @@ func TestAuthMiddleware_InferenceMiddleware_RealtimeTransportBypassesAuth(t *tes AdminPassword: schemas.NewEnvVar("hashedpassword"), IsEnabled: true, }) + // Enforce auth on inference; realtime transport endpoints must still bypass it + // because browser clients connect with an ephemeral key, not admin credentials. + am.UpdateEnforceAuthOnInference(true) routes := []string{ "/v1/realtime", @@ -775,6 +778,9 @@ func TestAuthMiddleware_InferenceMiddleware_RealtimeMintingStillRequiresAuth(t * AdminPassword: schemas.NewEnvVar("hashedpassword"), IsEnabled: true, }) + // Enforce auth on inference. Minting endpoints follow the inference auth toggle with + // no exception (unlike the transport carve-out), so they require auth when enforced. + am.UpdateEnforceAuthOnInference(true) routes := []string{ "/v1/realtime/client_secrets", diff --git a/transports/bifrost-http/handlers/plugins.go b/transports/bifrost-http/handlers/plugins.go index 8018947db3..b74399600e 100644 --- a/transports/bifrost-http/handlers/plugins.go +++ b/transports/bifrost-http/handlers/plugins.go @@ -18,6 +18,7 @@ import ( type PluginsLoader interface { GetPluginStatus(ctx context.Context) map[string]schemas.PluginStatus + GetLoadedPluginNames() []string ReloadPlugin(ctx context.Context, name string, path *string, pluginConfig any, placement *schemas.PluginPlacement, order *int) error RemovePlugin(ctx context.Context, name string) error // NormalizePluginConfig converts a raw config map to DB-storage format using @@ -95,6 +96,7 @@ func (h *PluginsHandler) expandPluginConfigForAPI(name string, config map[string func (h *PluginsHandler) RegisterRoutes(r *router.Router, middlewares ...schemas.BifrostHTTPMiddleware) { r.GET("/api/plugins", lib.ChainMiddlewares(h.getPlugins, middlewares...)) r.GET("/api/plugins/builtins", lib.ChainMiddlewares(h.getBuiltinPlugins, middlewares...)) + r.GET("/api/plugins/loaded", lib.ChainMiddlewares(h.getLoadedPlugins, middlewares...)) r.GET("/api/plugins/{name}", lib.ChainMiddlewares(h.getPlugin, middlewares...)) r.POST("/api/plugins", lib.ChainMiddlewares(h.createPlugin, middlewares...)) r.PUT("/api/plugins/{name}", lib.ChainMiddlewares(h.updatePlugin, middlewares...)) @@ -159,13 +161,21 @@ func (h *PluginsHandler) buildPluginResponseWithStatuses(plugin *configstoreTabl } } -// getBuiltinPlugins returns the canonical list of built-in plugin names +// getBuiltinPlugins returns the canonical list of built-in plugin names. func (h *PluginsHandler) getBuiltinPlugins(ctx *fasthttp.RequestCtx) { SendJSON(ctx, map[string]any{ "plugins": lib.GetBuiltinPluginNames(), }) } +// getLoadedPlugins returns the names of all plugins currently loaded at runtime, whose +// spans an observability connector can filter. +func (h *PluginsHandler) getLoadedPlugins(ctx *fasthttp.RequestCtx) { + SendJSON(ctx, map[string]any{ + "plugins": h.pluginsLoader.GetLoadedPluginNames(), + }) +} + // getPlugins gets all plugins func (h *PluginsHandler) getPlugins(ctx *fasthttp.RequestCtx) { if h.configStore == nil { diff --git a/transports/bifrost-http/handlers/plugins_test.go b/transports/bifrost-http/handlers/plugins_test.go index d268fa74af..e9d6580844 100644 --- a/transports/bifrost-http/handlers/plugins_test.go +++ b/transports/bifrost-http/handlers/plugins_test.go @@ -51,6 +51,7 @@ func (noopPluginsLoader) RemovePlugin(_ context.Context, _ string) error { retur func (noopPluginsLoader) GetPluginStatus(_ context.Context) map[string]schemas.PluginStatus { return nil } +func (noopPluginsLoader) GetLoadedPluginNames() []string { return nil } func (noopPluginsLoader) NormalizePluginConfig(_ string, _ map[string]any) (map[string]any, error) { return nil, nil } @@ -163,3 +164,45 @@ func TestUpdatePlugin_ConfigMerge_NewPlugin(t *testing.T) { t.Fatalf("expected 200, got %d: %s", ctx.Response.StatusCode(), ctx.Response.Body()) } } + +// namedPluginsLoader is a noopPluginsLoader that returns a fixed set of loaded +// plugin names, used to assert the getLoadedPlugins response contract. +type namedPluginsLoader struct { + noopPluginsLoader + names []string +} + +func (l namedPluginsLoader) GetLoadedPluginNames() []string { return l.names } + +// TestGetLoadedPlugins verifies that getLoadedPlugins returns the loader's plugin +// names under the "plugins" JSON key, locking the response shape the UI depends on. +func TestGetLoadedPlugins(t *testing.T) { + want := []string{"logging", "telemetry", "enterprise-governance"} + h := &PluginsHandler{ + pluginsLoader: namedPluginsLoader{names: want}, + configStore: nil, + } + + ctx := &fasthttp.RequestCtx{} + ctx.Request.Header.SetMethod("GET") + h.getLoadedPlugins(ctx) + + if ctx.Response.StatusCode() != 200 { + t.Fatalf("expected 200, got %d: %s", ctx.Response.StatusCode(), ctx.Response.Body()) + } + + var response struct { + Plugins []string `json:"plugins"` + } + if err := json.Unmarshal(ctx.Response.Body(), &response); err != nil { + t.Fatalf("unmarshal response: %v", err) + } + if len(response.Plugins) != len(want) { + t.Fatalf("expected %d plugins, got %d: %v", len(want), len(response.Plugins), response.Plugins) + } + for i, name := range want { + if response.Plugins[i] != name { + t.Errorf("plugins[%d] = %q, want %q", i, response.Plugins[i], name) + } + } +} diff --git a/transports/bifrost-http/handlers/provider_keys.go b/transports/bifrost-http/handlers/provider_keys.go index 2e04417bee..449801dfb4 100644 --- a/transports/bifrost-http/handlers/provider_keys.go +++ b/transports/bifrost-http/handlers/provider_keys.go @@ -113,7 +113,7 @@ func (h *ProviderHandler) createProviderKey(ctx *fasthttp.RequestCtx) { return } - if err := key.Aliases.Validate(); err != nil { + if err := key.Aliases.Validate(baseProvider); err != nil { SendError(ctx, fasthttp.StatusBadRequest, fmt.Sprintf("Invalid aliases: %v", err)) return } @@ -139,8 +139,10 @@ func (h *ProviderHandler) createProviderKey(ctx *fasthttp.RequestCtx) { return } - if err := h.attemptModelDiscovery(ctx, provider, providerConfig.CustomProviderConfig); err != nil { - logger.Warn("Model discovery failed for provider %s after key create: %v", provider, err) + if providerConfig.CustomProviderConfig == nil || !providerConfig.CustomProviderConfig.IsKeyLess { + if err := h.modelsManager.OnKeyAdded(ctx, provider, key); err != nil { + logger.Warn("Catalog refresh failed for provider %s after key create: %v", provider, err) + } } redactedKey, err := h.inMemoryStore.GetProviderKeyRedacted(provider, key.ID) @@ -224,7 +226,7 @@ func (h *ProviderHandler) updateProviderKey(ctx *fasthttp.RequestCtx) { return } - if err := mergedKey.Aliases.Validate(); err != nil { + if err := mergedKey.Aliases.Validate(baseProvider); err != nil { SendError(ctx, fasthttp.StatusBadRequest, fmt.Sprintf("Invalid aliases: %v", err)) return } @@ -244,8 +246,10 @@ func (h *ProviderHandler) updateProviderKey(ctx *fasthttp.RequestCtx) { return } - if err := h.attemptModelDiscovery(ctx, provider, providerConfig.CustomProviderConfig); err != nil { - logger.Warn("Model discovery failed for provider %s after key update: %v", provider, err) + if providerConfig.CustomProviderConfig == nil || !providerConfig.CustomProviderConfig.IsKeyLess { + if err := h.modelsManager.OnKeyUpdated(ctx, provider, mergedKey); err != nil { + logger.Warn("Catalog refresh failed for provider %s after key update: %v", provider, err) + } } redactedKey, err := h.inMemoryStore.GetProviderKeyRedacted(provider, keyID) @@ -305,8 +309,8 @@ func (h *ProviderHandler) deleteProviderKey(ctx *fasthttp.RequestCtx) { return } - if err := h.attemptModelDiscovery(ctx, provider, providerConfig.CustomProviderConfig); err != nil { - logger.Warn("Model discovery failed for provider %s after key delete: %v", provider, err) + if err := h.modelsManager.OnKeyDeleted(ctx, provider, keyID); err != nil { + logger.Warn("Catalog refresh failed for provider %s after key delete: %v", provider, err) } SendJSON(ctx, redactedKey) diff --git a/transports/bifrost-http/handlers/providers.go b/transports/bifrost-http/handlers/providers.go index 059f5e138f..4d2ed7b0ae 100644 --- a/transports/bifrost-http/handlers/providers.go +++ b/transports/bifrost-http/handlers/providers.go @@ -31,6 +31,9 @@ type ModelsManager interface { GetModelsForProvider(provider schemas.ModelProvider) []string GetUnfilteredModelsForProvider(provider schemas.ModelProvider) []string UpsertModelPricingAttributes(ctx context.Context, entries []ModelPricingAttributesEntry) error + OnKeyAdded(ctx context.Context, provider schemas.ModelProvider, key schemas.Key) error + OnKeyUpdated(ctx context.Context, provider schemas.ModelProvider, key schemas.Key) error + OnKeyDeleted(ctx context.Context, provider schemas.ModelProvider, keyID string) error } // ModelPricingAttributesEntry is the wire shape for PUT /api/models/catalog. diff --git a/transports/bifrost-http/handlers/providers_test.go b/transports/bifrost-http/handlers/providers_test.go index 26755d046e..09398d9592 100644 --- a/transports/bifrost-http/handlers/providers_test.go +++ b/transports/bifrost-http/handlers/providers_test.go @@ -54,6 +54,18 @@ func (m *mockModelsManager) UpsertModelPricingAttributes(_ context.Context, _ [] return nil } +func (m *mockModelsManager) OnKeyAdded(_ context.Context, _ schemas.ModelProvider, _ schemas.Key) error { + return nil +} + +func (m *mockModelsManager) OnKeyUpdated(_ context.Context, _ schemas.ModelProvider, _ schemas.Key) error { + return nil +} + +func (m *mockModelsManager) OnKeyDeleted(_ context.Context, _ schemas.ModelProvider, _ string) error { + return nil +} + // providerHandlerForTest builds a handler with fixed provider config and model sets. func providerHandlerForTest(provider schemas.ModelProvider, keys []schemas.Key, filtered, unfiltered []string) *ProviderHandler { return &ProviderHandler{ @@ -394,7 +406,7 @@ func TestListModelDetails_UnknownKeysDoNotFilter(t *testing.T) { []string{"gpt-4o", "gpt-4o-mini"}, []string{"gpt-4o", "gpt-4o-mini"}, ) - h.inMemoryStore.ModelCatalog = &modelcatalog.ModelCatalog{} + h.inMemoryStore.ModelCatalog = modelcatalog.NewTestCatalog(nil) ctx := &fasthttp.RequestCtx{} ctx.Request.Header.SetMethod("GET") @@ -425,7 +437,7 @@ func TestListModelDetails_SkipsUnknownKeysAndFiltersWithValid(t *testing.T) { []string{"gpt-4o", "gpt-4o-mini"}, []string{"gpt-4o", "gpt-4o-mini"}, ) - h.inMemoryStore.ModelCatalog = &modelcatalog.ModelCatalog{} + h.inMemoryStore.ModelCatalog = modelcatalog.NewTestCatalog(nil) ctx := &fasthttp.RequestCtx{} ctx.Request.Header.SetMethod("GET") @@ -462,7 +474,7 @@ func TestListModelDetails_SkipsDisabledKeysAndFiltersWithValid(t *testing.T) { []string{"gpt-4o", "gpt-4o-mini"}, []string{"gpt-4o", "gpt-4o-mini"}, ) - h.inMemoryStore.ModelCatalog = &modelcatalog.ModelCatalog{} + h.inMemoryStore.ModelCatalog = modelcatalog.NewTestCatalog(nil) ctx := &fasthttp.RequestCtx{} ctx.Request.Header.SetMethod("GET") @@ -498,7 +510,7 @@ func TestListModelDetails_UnfilteredIgnoresKeys(t *testing.T) { []string{"gpt-4o"}, []string{"gpt-4o", "gpt-4o-mini"}, ) - h.inMemoryStore.ModelCatalog = &modelcatalog.ModelCatalog{} + h.inMemoryStore.ModelCatalog = modelcatalog.NewTestCatalog(nil) ctx := &fasthttp.RequestCtx{} ctx.Request.Header.SetMethod("GET") diff --git a/transports/bifrost-http/handlers/realtime_client_secrets.go b/transports/bifrost-http/handlers/realtime_client_secrets.go index 6b8f680e15..4b500ce0bf 100644 --- a/transports/bifrost-http/handlers/realtime_client_secrets.go +++ b/transports/bifrost-http/handlers/realtime_client_secrets.go @@ -11,6 +11,7 @@ import ( bifrost "github.com/maximhq/bifrost/core" "github.com/maximhq/bifrost/core/schemas" "github.com/maximhq/bifrost/plugins/governance" + "github.com/maximhq/bifrost/plugins/modelcatalogresolver" "github.com/maximhq/bifrost/transports/bifrost-http/integrations" "github.com/maximhq/bifrost/transports/bifrost-http/lib" "github.com/valyala/fasthttp" @@ -231,14 +232,14 @@ func resolveRealtimeClientSecretTarget(ctx *fasthttp.RequestCtx, config *lib.Con providerKey, model := schemas.ParseModelString(rawModel, defaultProvider) // Model catalog auto-resolution for bare model names on /v1 client secret routes if defaultProvider == "" && providerKey == "" && model != "" { - providers := config.GetProvidersForModel(model) - if len(providers) > 0 { + selected, candidates := modelcatalogresolver.ResolveProviderFromCatalog(nil, config.ModelCatalog, model) + if selected != "" { ctx.SetUserValue(lib.FastHTTPUserValueModelCatalogResolution, &lib.ModelCatalogResolution{ Model: model, - ResolvedProvider: providers[0], - AllProviders: providers, + ResolvedProvider: selected, + AllProviders: candidates, }) - providerKey = providers[0] + providerKey = selected } } if defaultProvider == "" && providerKey == "" { diff --git a/transports/bifrost-http/handlers/realtime_client_secrets_test.go b/transports/bifrost-http/handlers/realtime_client_secrets_test.go index 8c1b83dfa8..b7ed942b23 100644 --- a/transports/bifrost-http/handlers/realtime_client_secrets_test.go +++ b/transports/bifrost-http/handlers/realtime_client_secrets_test.go @@ -311,6 +311,10 @@ func (m *mockRealtimeMintingGovernancePlugin) HTTPTransportPostHook(_ *schemas.B return nil } +func (m *mockRealtimeMintingGovernancePlugin) PreRequestHook(_ *schemas.BifrostContext, _ *schemas.BifrostRequest) error { + return nil +} + func (m *mockRealtimeMintingGovernancePlugin) PreLLMHook(_ *schemas.BifrostContext, req *schemas.BifrostRequest) (*schemas.BifrostRequest, *schemas.LLMPluginShortCircuit, error) { return req, nil, nil } diff --git a/transports/bifrost-http/handlers/webrtc_realtime.go b/transports/bifrost-http/handlers/webrtc_realtime.go index 6119ee4a44..3f88d88655 100644 --- a/transports/bifrost-http/handlers/webrtc_realtime.go +++ b/transports/bifrost-http/handlers/webrtc_realtime.go @@ -15,6 +15,7 @@ import ( bifrost "github.com/maximhq/bifrost/core" "github.com/maximhq/bifrost/core/providers/openai" "github.com/maximhq/bifrost/core/schemas" + "github.com/maximhq/bifrost/plugins/modelcatalogresolver" "github.com/maximhq/bifrost/transports/bifrost-http/integrations" "github.com/maximhq/bifrost/transports/bifrost-http/lib" bfws "github.com/maximhq/bifrost/transports/bifrost-http/websocket" @@ -167,14 +168,14 @@ func parseCallsWebRTCRequest(ctx *fasthttp.RequestCtx, config *lib.Config) (stri providerKey, model := schemas.ParseModelString(rawModel, realtimeDefaultProviderForPath(path)) // Model catalog auto-resolution for bare model names on base /v1 routes if providerKey == "" && strings.TrimSpace(model) != "" { - providers := config.GetProvidersForModel(model) - if len(providers) > 0 { + selected, candidates := modelcatalogresolver.ResolveProviderFromCatalog(nil, config.ModelCatalog, model) + if selected != "" { ctx.SetUserValue(lib.FastHTTPUserValueModelCatalogResolution, &lib.ModelCatalogResolution{ Model: model, - ResolvedProvider: providers[0], - AllProviders: providers, + ResolvedProvider: selected, + AllProviders: candidates, }) - providerKey = providers[0] + providerKey = selected } } if providerKey == "" || strings.TrimSpace(model) == "" { @@ -199,14 +200,14 @@ func (h *WebRTCRealtimeHandler) handleLegacyRequest(ctx *fasthttp.RequestCtx, de providerKey, model := schemas.ParseModelString(rawModel, defaultProvider) // Model catalog auto-resolution for bare model names on base /v1 routes if providerKey == "" && strings.TrimSpace(model) != "" { - providers := h.config.GetProvidersForModel(model) - if len(providers) > 0 { + selected, candidates := modelcatalogresolver.ResolveProviderFromCatalog(nil, h.config.ModelCatalog, model) + if selected != "" { ctx.SetUserValue(lib.FastHTTPUserValueModelCatalogResolution, &lib.ModelCatalogResolution{ Model: model, - ResolvedProvider: providers[0], - AllProviders: providers, + ResolvedProvider: selected, + AllProviders: candidates, }) - providerKey = providers[0] + providerKey = selected } } if providerKey == "" || model == "" { @@ -1236,14 +1237,14 @@ func resolveRealtimeSDPTarget(ctx *fasthttp.RequestCtx, config *lib.Config, path providerKey, model := schemas.ParseModelString(strings.TrimSpace(rawModel), realtimeDefaultProviderForPath(path)) // Model catalog auto-resolution for bare model names in session body if providerKey == "" && strings.TrimSpace(model) != "" { - providers := config.GetProvidersForModel(model) - if len(providers) > 0 { + selected, candidates := modelcatalogresolver.ResolveProviderFromCatalog(nil, config.ModelCatalog, model) + if selected != "" { ctx.SetUserValue(lib.FastHTTPUserValueModelCatalogResolution, &lib.ModelCatalogResolution{ Model: model, - ResolvedProvider: providers[0], - AllProviders: providers, + ResolvedProvider: selected, + AllProviders: candidates, }) - providerKey = providers[0] + providerKey = selected } } if providerKey == "" || strings.TrimSpace(model) == "" { diff --git a/transports/bifrost-http/handlers/webrtc_realtime_test.go b/transports/bifrost-http/handlers/webrtc_realtime_test.go index 9fb8a1321a..d4b09b3f33 100644 --- a/transports/bifrost-http/handlers/webrtc_realtime_test.go +++ b/transports/bifrost-http/handlers/webrtc_realtime_test.go @@ -18,8 +18,7 @@ type testHandlerStore struct { kv *kvstore.Store } -func (s testHandlerStore) GetHeaderMatcher() *lib.HeaderMatcher { return nil } -func (s testHandlerStore) GetProvidersForModel(model string) []schemas.ModelProvider { return nil } +func (s testHandlerStore) GetHeaderMatcher() *lib.HeaderMatcher { return nil } func (s testHandlerStore) GetStreamChunkInterceptor() lib.StreamChunkInterceptor { return nil } diff --git a/transports/bifrost-http/handlers/wsrealtime.go b/transports/bifrost-http/handlers/wsrealtime.go index 81edd1496f..761bc269ef 100644 --- a/transports/bifrost-http/handlers/wsrealtime.go +++ b/transports/bifrost-http/handlers/wsrealtime.go @@ -92,6 +92,83 @@ func (h *WSRealtimeHandler) handleUpgrade(ctx *fasthttp.RequestCtx) { return } + // Run PreRequestHook to give governance + LB a chance to route the realtime connection. + // Realtime bypasses handleRequest (per-turn pipelines instead), so we invoke the routing + // phase explicitly here. Mutations to provider/model are read back into the local vars + // and copied to fasthttp user values so snapshotRealtimeMiddlewareValues picks up any + // ctx changes (governance team/customer IDs, routing engine logs). + preReqCtx, preReqCancel := createBifrostContextFromAuth(h.handlerStore, auth) + if preReqCtx == nil { + preReqCancel() + upgrader := h.websocketUpgrader("") + upgradeErr := upgrader.Upgrade(ctx, func(conn *ws.Conn) { + defer conn.Close() + clientConn := newRealtimeClientConn(conn) + clientConn.writeRealtimeError(newRealtimeWireBifrostError(500, "server_error", "failed to create request context")) + }) + if upgradeErr != nil { + logger.Warn("websocket upgrade failed for %s: %v", path, upgradeErr) + } + return + } + preReqCtx.SetValue(schemas.BifrostContextKeyHTTPRequestType, schemas.RealtimeRequest) + if realtimeDefaultProviderForPath(path) == schemas.OpenAI { + preReqCtx.SetValue(schemas.BifrostContextKeyIntegrationType, "openai") + } + // Surface full request headers + query params on the pre-request context so governance + // CEL routing rules (which read headers[...] / params[...]) see the same shape they would + // for normal HTTP requests. Mirrors lib/ctx.go ConvertToBifrostContext; the normal HTTP + // path doesn't run for WS upgrades, so we populate these explicitly. Keys are lowercased. + allHeaders := make(map[string]string) + ctx.Request.Header.All()(func(key, value []byte) bool { + allHeaders[strings.ToLower(string(key))] = string(value) + return true + }) + preReqCtx.SetValue(schemas.BifrostContextKeyRequestHeaders, allHeaders) + if queryArgs := ctx.Request.URI().QueryArgs(); queryArgs.Len() > 0 { + allQuery := make(map[string]string, queryArgs.Len()) + queryArgs.All()(func(key, value []byte) bool { + allQuery[strings.ToLower(string(key))] = string(value) + return true + }) + preReqCtx.SetValue(schemas.BifrostContextKeyRequestQuery, allQuery) + } + preReq := &schemas.BifrostRequest{ + RequestType: schemas.RealtimeRequest, + ResponsesRequest: &schemas.BifrostResponsesRequest{ + Provider: providerKey, + Model: model, + }, + } + h.client.RunPreRequestHooks(preReqCtx, preReq) + routedProvider, routedModel, _ := preReq.GetRequestFields() + if routedProvider == "" { + // Mirror the empty-provider check in core handleRequest. No routing layer + // (governance routing rules / LB / modelcatalogresolver) could pick a provider + // for this model — caller's input is unresolvable. + upgrader := h.websocketUpgrader("") + upgradeErr := upgrader.Upgrade(ctx, func(conn *ws.Conn) { + defer conn.Close() + clientConn := newRealtimeClientConn(conn) + clientConn.writeRealtimeError(newRealtimeWireBifrostError(400, "invalid_request_error", fmt.Sprintf("no provider could be resolved for model %q (set as provider/model or configure the model catalog)", model))) + }) + if upgradeErr != nil { + logger.Warn("websocket upgrade failed for %s: %v", path, upgradeErr) + } + preReqCancel() + return + } + providerKey = routedProvider + if routedModel != "" { + model = routedModel + } + // Mirror ctx values back to fasthttp user values so snapshotRealtimeMiddlewareValues + // (called below) picks them up — same mechanism TransportInterceptorMiddleware uses. + for k, v := range preReqCtx.GetUserValues() { + ctx.SetUserValue(k, v) + } + preReqCancel() + provider := h.client.GetProviderByKey(providerKey) rtProvider, ok := provider.(schemas.RealtimeProvider) if provider == nil || !ok || !rtProvider.SupportsRealtimeAPI() { @@ -517,7 +594,7 @@ func (h *WSRealtimeHandler) relayRealtimeProviderToClient( } } -func resolveRealtimeTarget(ctx *fasthttp.RequestCtx, config *lib.Config, path, modelParam, deploymentParam string) (schemas.ModelProvider, string, error) { +func resolveRealtimeTarget(_ *fasthttp.RequestCtx, _ *lib.Config, path, modelParam, deploymentParam string) (schemas.ModelProvider, string, error) { defaultProvider := realtimeDefaultProviderForPath(path) var rawParam string @@ -535,22 +612,9 @@ func resolveRealtimeTarget(ctx *fasthttp.RequestCtx, config *lib.Config, path, m return "", "", errRealtimeModelFormat } - // Model catalog auto-resolution: when no provider prefix is present and the - // path doesn't imply a default provider, look up the model catalog — same - // logic as resolveModelAndProvider in inference.go. - if provider == "" { - providers := config.GetProvidersForModel(model) - if len(providers) == 0 { - return "", "", errRealtimeModelFormat - } - ctx.SetUserValue(lib.FastHTTPUserValueModelCatalogResolution, &lib.ModelCatalogResolution{ - Model: model, - ResolvedProvider: providers[0], - AllProviders: providers, - }) - provider = providers[0] - } - + // Provider may be empty here when no path-default applies and the model has + // no explicit prefix. The modelcatalogresolver PreRequestHook will fill it in + // (or surface a clear error if no provider matches) — no inline lookup needed. return provider, model, nil } @@ -772,13 +836,11 @@ var realtimeMiddlewareKeys = []any{ // snapshotRealtimeMiddlewareValues reads governance/routing values from the fasthttp // context's UserValue store. TransportInterceptorMiddleware copies them there as -// individual key-value pairs (not inside a BifrostContext). -// -// It also processes FastHTTPUserValueModelCatalogResolution, which is set by -// resolveRealtimeTarget when a bare model name is auto-resolved via the model -// catalog. ConvertToBifrostContext normally handles this for regular inference, -// but WebSocket handlers use createBifrostContextFromAuth instead, so we do the -// same log/engine enrichment here. +// individual key-value pairs (not inside a BifrostContext). Routing engine logs +// emitted by PreRequestHook (governance routing rules, LB, modelcatalogresolver) +// are surfaced through the same mechanism — the hooks write them onto preReqCtx +// and handleUpgrade mirrors that ctx's user values onto the fasthttp ctx before +// this function is called. func snapshotRealtimeMiddlewareValues(ctx *fasthttp.RequestCtx) map[any]any { result := make(map[any]any) for _, key := range realtimeMiddlewareKeys { @@ -786,33 +848,6 @@ func snapshotRealtimeMiddlewareValues(ctx *fasthttp.RequestCtx) map[any]any { result[key] = value } } - - // Model catalog auto-resolution: replicate the routing engine log that - // ConvertToBifrostContext would normally emit (see lib/ctx.go). - if res, ok := ctx.UserValue(lib.FastHTTPUserValueModelCatalogResolution).(*lib.ModelCatalogResolution); ok && res != nil { - providerStrs := make([]string, len(res.AllProviders)) - for i, p := range res.AllProviders { - providerStrs[i] = string(p) - } - logEntry := schemas.RoutingEngineLogEntry{ - Engine: schemas.RoutingEngineModelCatalog, - Level: schemas.LogLevelInfo, - Message: fmt.Sprintf("No provider specified for model %s, found %d options in model catalog: [%s], selecting first: %s", res.Model, len(res.AllProviders), strings.Join(providerStrs, ", "), res.ResolvedProvider), - Timestamp: time.Now().UnixMilli(), - } - // Merge with any existing routing engine logs from governance middleware. - if existing, ok := result[schemas.BifrostContextKeyRoutingEngineLogs].([]schemas.RoutingEngineLogEntry); ok { - result[schemas.BifrostContextKeyRoutingEngineLogs] = append(existing, logEntry) - } else { - result[schemas.BifrostContextKeyRoutingEngineLogs] = []schemas.RoutingEngineLogEntry{logEntry} - } - if existing, ok := result[schemas.BifrostContextKeyRoutingEnginesUsed].([]string); ok { - result[schemas.BifrostContextKeyRoutingEnginesUsed] = append(existing, schemas.RoutingEngineModelCatalog) - } else { - result[schemas.BifrostContextKeyRoutingEnginesUsed] = []string{schemas.RoutingEngineModelCatalog} - } - } - if len(result) == 0 { return nil } diff --git a/transports/bifrost-http/handlers/wsresponses_test.go b/transports/bifrost-http/handlers/wsresponses_test.go index 424061ab65..0cb2c7a0d7 100644 --- a/transports/bifrost-http/handlers/wsresponses_test.go +++ b/transports/bifrost-http/handlers/wsresponses_test.go @@ -24,10 +24,6 @@ func (s testWSHandlerStore) GetHeaderMatcher() *lib.HeaderMatcher { return s.matcher } -func (s testWSHandlerStore) GetProvidersForModel(model string) []schemas.ModelProvider { - return nil -} - func (s testWSHandlerStore) GetStreamChunkInterceptor() lib.StreamChunkInterceptor { return nil } diff --git a/transports/bifrost-http/integrations/anthropic.go b/transports/bifrost-http/integrations/anthropic.go index 87743291bd..4d24b5fb4b 100644 --- a/transports/bifrost-http/integrations/anthropic.go +++ b/transports/bifrost-http/integrations/anthropic.go @@ -23,18 +23,6 @@ type AnthropicRouter struct { *GenericRouter } -// anthropicModelGetter extracts the model field from any Anthropic integration request type. -// It is called after body parsing, so req is fully populated. -func anthropicModelGetter(_ *fasthttp.RequestCtx, req interface{}) (string, error) { - switch r := req.(type) { - case *anthropic.AnthropicTextRequest: - return r.Model, nil - case *anthropic.AnthropicMessageRequest: - return r.Model, nil - } - return "", nil -} - // createAnthropicCompleteRouteConfig creates a route configuration for the `/v1/complete` endpoint. func createAnthropicCompleteRouteConfig(pathPrefix string) RouteConfig { return RouteConfig{ @@ -47,7 +35,6 @@ func createAnthropicCompleteRouteConfig(pathPrefix string) RouteConfig { GetRequestTypeInstance: func(ctx context.Context) interface{} { return &anthropic.AnthropicTextRequest{} }, - GetRequestModel: anthropicModelGetter, RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { if anthropicReq, ok := req.(*anthropic.AnthropicTextRequest); ok { return &schemas.BifrostRequest{ @@ -88,7 +75,6 @@ func createAnthropicMessagesRouteConfig(pathPrefix string, logger schemas.Logger GetRequestTypeInstance: func(ctx context.Context) interface{} { return &anthropic.AnthropicMessageRequest{} }, - GetRequestModel: anthropicModelGetter, RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { if anthropicReq, ok := req.(*anthropic.AnthropicMessageRequest); ok { bifrostReq := anthropicReq.ToBifrostResponsesRequest(ctx) @@ -320,19 +306,9 @@ func checkAnthropicPassthrough(ctx *fasthttp.RequestCtx, bifrostCtx *schemas.Bif switch r := req.(type) { case *anthropic.AnthropicTextRequest: provider, model = schemas.ParseModelString(r.Model, "") - // Check if model parameter explicitly has `anthropic/` prefix - if provider == schemas.Anthropic { - bifrostCtx.SetValue(schemas.BifrostContextKeySkipModelCatalogProviderSelection, true) - r.Model = model - } case *anthropic.AnthropicMessageRequest: provider, model = schemas.ParseModelString(r.Model, "") - // Check if model parameter explicitly has `anthropic/` prefix - if provider == schemas.Anthropic { - bifrostCtx.SetValue(schemas.BifrostContextKeySkipModelCatalogProviderSelection, true) - r.Model = model - } } headers := extractHeadersFromRequest(ctx) @@ -425,7 +401,6 @@ func CreateAnthropicCountTokensRouteConfigs(pathPrefix string, handlerStore lib. GetRequestTypeInstance: func(ctx context.Context) interface{} { return &anthropic.AnthropicMessageRequest{} }, - GetRequestModel: anthropicModelGetter, RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { if anthropicReq, ok := req.(*anthropic.AnthropicMessageRequest); ok { bifrostReq := anthropicReq.ToBifrostResponsesRequest(ctx) diff --git a/transports/bifrost-http/integrations/bedrock.go b/transports/bifrost-http/integrations/bedrock.go index 00da5932c8..89b784e440 100644 --- a/transports/bifrost-http/integrations/bedrock.go +++ b/transports/bifrost-http/integrations/bedrock.go @@ -20,23 +20,6 @@ type BedrockRouter struct { *GenericRouter } -// bedrockModelGetter extracts the model ID from any Bedrock integration request type. -// It is called after PreCallback, so req.ModelID is populated from the URL path param. -func bedrockModelGetter(_ *fasthttp.RequestCtx, req interface{}) (string, error) { - switch r := req.(type) { - case *bedrock.BedrockConverseRequest: - return r.ModelID, nil - case *bedrock.BedrockInvokeRequest: - return r.ModelID, nil - case *bedrock.BedrockCountTokensRequest: - if r.Input.Converse != nil { - return r.Input.Converse.ModelID, nil - } - return "", nil - } - return "", nil -} - // S3 context keys for storing request parameters const ( @@ -58,7 +41,6 @@ func createBedrockConverseRouteConfig(pathPrefix string, handlerStore lib.Handle GetHTTPRequestType: func(ctx *fasthttp.RequestCtx) schemas.RequestType { return schemas.ResponsesRequest }, - GetRequestModel: bedrockModelGetter, RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { if bedrockReq, ok := req.(*bedrock.BedrockConverseRequest); ok { bifrostReq, err := bedrockReq.ToBifrostResponsesRequest(ctx) @@ -94,7 +76,6 @@ func createBedrockConverseStreamRouteConfig(pathPrefix string, handlerStore lib. GetRequestTypeInstance: func(ctx context.Context) interface{} { return &bedrock.BedrockConverseRequest{} }, - GetRequestModel: bedrockModelGetter, RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { if bedrockReq, ok := req.(*bedrock.BedrockConverseRequest); ok { // Mark as streaming request @@ -145,7 +126,6 @@ func createBedrockInvokeWithResponseStreamRouteConfig(pathPrefix string, handler GetRequestTypeInstance: func(ctx context.Context) interface{} { return &bedrock.BedrockInvokeRequest{} }, - GetRequestModel: bedrockModelGetter, RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { if invokeReq, ok := req.(*bedrock.BedrockInvokeRequest); ok { requestType, _ := ctx.Value(schemas.BifrostContextKeyHTTPRequestType).(schemas.RequestType) @@ -220,7 +200,6 @@ func createBedrockInvokeRouteConfig(pathPrefix string, handlerStore lib.HandlerS GetRequestTypeInstance: func(ctx context.Context) interface{} { return &bedrock.BedrockInvokeRequest{} }, - GetRequestModel: bedrockModelGetter, RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { invokeReq, ok := req.(*bedrock.BedrockInvokeRequest) if !ok { @@ -276,7 +255,7 @@ func createBedrockInvokeRouteConfig(pathPrefix string, handlerStore lib.HandlerS return bedrock.ToBedrockInvokeMessagesResponse(ctx, resp) }, EmbeddingResponseConverter: func(ctx *schemas.BifrostContext, resp *schemas.BifrostEmbeddingResponse) (interface{}, error) { - return bedrock.ToBedrockEmbeddingInvokeResponse(resp) + return bedrock.ToBedrockEmbeddingInvokeResponse(ctx, resp) }, ImageGenerationResponseConverter: func(ctx *schemas.BifrostContext, resp *schemas.BifrostImageGenerationResponse) (interface{}, error) { return bedrock.ToBedrockInvokeImagesResponse(ctx, resp) @@ -337,7 +316,6 @@ func createBedrockCountTokensRouteConfig(pathPrefix string, handlerStore lib.Han GetHTTPRequestType: func(ctx *fasthttp.RequestCtx) schemas.RequestType { return schemas.CountTokensRequest }, - GetRequestModel: bedrockModelGetter, RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { if countTokensReq, ok := req.(*bedrock.BedrockCountTokensRequest); ok { if countTokensReq.Input.Converse == nil { diff --git a/transports/bifrost-http/integrations/bedrock_test.go b/transports/bifrost-http/integrations/bedrock_test.go index 42de6b3bc8..6b038c3758 100644 --- a/transports/bifrost-http/integrations/bedrock_test.go +++ b/transports/bifrost-http/integrations/bedrock_test.go @@ -25,10 +25,6 @@ func (m *mockHandlerStore) GetHeaderMatcher() *lib.HeaderMatcher { return m.headerMatcher } -func (m *mockHandlerStore) GetProvidersForModel(model string) []schemas.ModelProvider { - return m.availableProviders -} - func (m *mockHandlerStore) GetStreamChunkInterceptor() lib.StreamChunkInterceptor { return nil } diff --git a/transports/bifrost-http/integrations/cohere.go b/transports/bifrost-http/integrations/cohere.go index cf6b7ceaca..37aad1c1a8 100644 --- a/transports/bifrost-http/integrations/cohere.go +++ b/transports/bifrost-http/integrations/cohere.go @@ -69,22 +69,6 @@ func NewCohereRouter(client *bifrost.Bifrost, handlerStore lib.HandlerStore, log } } -// cohereModelGetter extracts the model field from any Cohere integration request type. -// It is called after body parsing, so req is fully populated. -func cohereModelGetter(_ *fasthttp.RequestCtx, req interface{}) (string, error) { - switch r := req.(type) { - case *cohere.CohereChatRequest: - return r.Model, nil - case *cohere.CohereEmbeddingRequest: - return r.Model, nil - case *cohere.CohereRerankRequest: - return r.Model, nil - case *cohere.CohereCountTokensRequest: - return r.Model, nil - } - return "", nil -} - // CreateCohereRouteConfigs creates route configurations for Cohere API endpoints. func CreateCohereRouteConfigs(pathPrefix string) []RouteConfig { var routes []RouteConfig @@ -101,7 +85,6 @@ func CreateCohereRouteConfigs(pathPrefix string) []RouteConfig { GetRequestTypeInstance: func(ctx context.Context) interface{} { return &cohere.CohereChatRequest{} }, - GetRequestModel: cohereModelGetter, RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { if cohereReq, ok := req.(*cohere.CohereChatRequest); ok { return &schemas.BifrostRequest{ @@ -148,7 +131,6 @@ func CreateCohereRouteConfigs(pathPrefix string) []RouteConfig { GetRequestTypeInstance: func(ctx context.Context) interface{} { return &cohere.CohereEmbeddingRequest{} }, - GetRequestModel: cohereModelGetter, RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { if cohereReq, ok := req.(*cohere.CohereEmbeddingRequest); ok { return &schemas.BifrostRequest{ @@ -182,7 +164,6 @@ func CreateCohereRouteConfigs(pathPrefix string) []RouteConfig { GetRequestTypeInstance: func(ctx context.Context) interface{} { return &cohere.CohereRerankRequest{} }, - GetRequestModel: cohereModelGetter, RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { if cohereReq, ok := req.(*cohere.CohereRerankRequest); ok { return &schemas.BifrostRequest{ @@ -216,7 +197,6 @@ func CreateCohereRouteConfigs(pathPrefix string) []RouteConfig { GetRequestTypeInstance: func(ctx context.Context) interface{} { return &cohere.CohereCountTokensRequest{} }, - GetRequestModel: cohereModelGetter, RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { if cohereReq, ok := req.(*cohere.CohereCountTokensRequest); ok { return &schemas.BifrostRequest{ diff --git a/transports/bifrost-http/integrations/genai.go b/transports/bifrost-http/integrations/genai.go index bc24f3fe27..930ca02cb5 100644 --- a/transports/bifrost-http/integrations/genai.go +++ b/transports/bifrost-http/integrations/genai.go @@ -38,33 +38,6 @@ type GenAIRouter struct { *GenericRouter } -// genAIModelGetter extracts the model name for GenAI routes. -// For request types populated by extractAndSetModelAndRequestType (the PreCallback), -// the model is already clean on the struct. For BifrostVideoRetrieveRequest (which has -// no model field), the provider-scoped model is extracted from the operation_id suffix -// (format: "op123:openai/gpt-4o") since the route pins the provider via operation_id. -func genAIModelGetter(ctx *fasthttp.RequestCtx, req interface{}) (string, error) { - switch r := req.(type) { - case *gemini.GeminiGenerationRequest: - return r.Model, nil - case *gemini.GeminiEmbeddingRequest: - return r.Model, nil - case *gemini.GeminiVideoGenerationRequest: - return r.Model, nil - case *gemini.GeminiBatchCreateRequest: - return r.Model, nil - case *schemas.BifrostVideoRetrieveRequest: - // operation_id encodes the full model string: "op123:gpt-4o" or "op123:openai/gpt-4o". - operationID, _ := ctx.UserValue("operation_id").(string) - parts := strings.Split(operationID, ":") - if len(parts) >= 2 && parts[len(parts)-1] != "" { - return parts[len(parts)-1], nil - } - return "", nil - } - return "", nil -} - // CreateGenAIRouteConfigs creates a route configurations for GenAI endpoints. func CreateGenAIRouteConfigs(pathPrefix string) []RouteConfig { var routes []RouteConfig @@ -81,7 +54,6 @@ func CreateGenAIRouteConfigs(pathPrefix string) []RouteConfig { GetRequestTypeInstance: func(ctx context.Context) interface{} { return &schemas.BifrostVideoRetrieveRequest{} }, - GetRequestModel: genAIModelGetter, RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { if videoRetrieveReq, ok := req.(*schemas.BifrostVideoRetrieveRequest); ok { return &schemas.BifrostRequest{ @@ -120,7 +92,6 @@ func CreateGenAIRouteConfigs(pathPrefix string) []RouteConfig { } return &gemini.GeminiGenerationRequest{} }, - GetRequestModel: genAIModelGetter, RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { if geminiReq, ok := req.(*gemini.GeminiGenerationRequest); ok { if geminiReq.IsCountTokens { @@ -705,6 +676,220 @@ func CreateGenAIBatchRouteConfigs(pathPrefix string, handlerStore lib.HandlerSto return routes } +// CreateVertexBatchRouteConfigs creates route configurations for the native Vertex AI +// batchPredictionJobs API (as used by the aiplatform JobServiceClient). Unlike the Gemini +// Developer batches surface, Vertex batch prediction is GCS-backed and addressed by the +// regional resource path projects/{project}/locations/{location}/batchPredictionJobs. +// Key/project selection happens in Bifrost from the vertex key config, so the project and +// location in the path are placeholders used only for routing the request shape. +func CreateVertexBatchRouteConfigs(pathPrefix string) []RouteConfig { + var routes []RouteConfig + + collectionPath := pathPrefix + "/v1/projects/{project}/locations/{location}/batchPredictionJobs" + itemPath := collectionPath + "/{batch_id}" + + // Create batch prediction job - POST .../batchPredictionJobs + routes = append(routes, RouteConfig{ + Type: RouteConfigTypeGenAI, + Path: collectionPath, + Method: "POST", + GetHTTPRequestType: func(ctx *fasthttp.RequestCtx) schemas.RequestType { + return schemas.BatchCreateRequest + }, + GetRequestTypeInstance: func(ctx context.Context) interface{} { + return &vertex.VertexBatchPredictionJob{} + }, + BatchRequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*BatchRequest, error) { + if job, ok := req.(*vertex.VertexBatchPredictionJob); ok { + createReq := vertex.ToBifrostBatchCreateRequest(job) + // The native body is already a Vertex BatchPredictionJob; carry it verbatim + // so BigQuery IO, non-JSONL formats and multi-URI inputs round-trip losslessly. + createReq.RawRequestBody = getGenAIRawRequestBody(ctx) + return &BatchRequest{ + Type: schemas.BatchCreateRequest, + CreateRequest: createReq, + }, nil + } + return nil, errors.New("invalid vertex batch create request type") + }, + BatchCreateResponseConverter: func(ctx *schemas.BifrostContext, resp *schemas.BifrostBatchCreateResponse) (interface{}, error) { + if resp.ExtraFields.Provider == schemas.Vertex && resp.ExtraFields.RawResponse != nil { + return resp.ExtraFields.RawResponse, nil + } + return vertex.ToVertexBatchCreateResponse(resp), nil + }, + ErrorConverter: func(ctx *schemas.BifrostContext, err *schemas.BifrostError) interface{} { + return gemini.ToGeminiError(err) + }, + // Native Vertex batch bodies pass through verbatim (see RawRequestBody above). + PreCallback: func(ctx *fasthttp.RequestCtx, bifrostCtx *schemas.BifrostContext, req interface{}) error { + setGenAIRawRequestBodyFromRequest(ctx, bifrostCtx) + bifrostCtx.SetValue(schemas.BifrostContextKeyUseRawRequestBody, true) + return nil + }, + }) + + // List batch prediction jobs - GET .../batchPredictionJobs + routes = append(routes, RouteConfig{ + Type: RouteConfigTypeGenAI, + Path: collectionPath, + Method: "GET", + GetHTTPRequestType: func(ctx *fasthttp.RequestCtx) schemas.RequestType { + return schemas.BatchListRequest + }, + GetRequestTypeInstance: func(ctx context.Context) interface{} { + return &schemas.BifrostBatchListRequest{} + }, + BatchRequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*BatchRequest, error) { + if listReq, ok := req.(*schemas.BifrostBatchListRequest); ok { + return &BatchRequest{Type: schemas.BatchListRequest, ListRequest: listReq}, nil + } + return nil, errors.New("invalid vertex batch list request type") + }, + BatchListResponseConverter: func(ctx *schemas.BifrostContext, resp *schemas.BifrostBatchListResponse) (interface{}, error) { + if resp.ExtraFields.Provider == schemas.Vertex && resp.ExtraFields.RawResponse != nil { + return resp.ExtraFields.RawResponse, nil + } + return vertex.ToVertexBatchListResponse(resp), nil + }, + ErrorConverter: func(ctx *schemas.BifrostContext, err *schemas.BifrostError) interface{} { + return gemini.ToGeminiError(err) + }, + PreCallback: extractVertexBatchPathParams, + }) + + // Retrieve batch prediction job - GET .../batchPredictionJobs/{batch_id} + routes = append(routes, RouteConfig{ + Type: RouteConfigTypeGenAI, + Path: itemPath, + Method: "GET", + GetHTTPRequestType: func(ctx *fasthttp.RequestCtx) schemas.RequestType { + return schemas.BatchRetrieveRequest + }, + GetRequestTypeInstance: func(ctx context.Context) interface{} { + return &schemas.BifrostBatchRetrieveRequest{} + }, + BatchRequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*BatchRequest, error) { + if retrieveReq, ok := req.(*schemas.BifrostBatchRetrieveRequest); ok { + return &BatchRequest{Type: schemas.BatchRetrieveRequest, RetrieveRequest: retrieveReq}, nil + } + return nil, errors.New("invalid vertex batch retrieve request type") + }, + BatchRetrieveResponseConverter: func(ctx *schemas.BifrostContext, resp *schemas.BifrostBatchRetrieveResponse) (interface{}, error) { + if resp.ExtraFields.Provider == schemas.Vertex && resp.ExtraFields.RawResponse != nil { + return resp.ExtraFields.RawResponse, nil + } + return vertex.ToVertexBatchRetrieveResponse(resp), nil + }, + ErrorConverter: func(ctx *schemas.BifrostContext, err *schemas.BifrostError) interface{} { + return gemini.ToGeminiError(err) + }, + PreCallback: extractVertexBatchPathParams, + }) + + // Cancel batch prediction job - POST .../batchPredictionJobs/{batch_id}:cancel + routes = append(routes, RouteConfig{ + Type: RouteConfigTypeGenAI, + Path: itemPath, + Method: "POST", + GetHTTPRequestType: func(ctx *fasthttp.RequestCtx) schemas.RequestType { + return schemas.BatchCancelRequest + }, + GetRequestTypeInstance: func(ctx context.Context) interface{} { + return &schemas.BifrostBatchCancelRequest{} + }, + BatchRequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*BatchRequest, error) { + if cancelReq, ok := req.(*schemas.BifrostBatchCancelRequest); ok { + return &BatchRequest{Type: schemas.BatchCancelRequest, CancelRequest: cancelReq}, nil + } + return nil, errors.New("invalid vertex batch cancel request type") + }, + BatchCancelResponseConverter: func(ctx *schemas.BifrostContext, resp *schemas.BifrostBatchCancelResponse) (interface{}, error) { + if resp.ExtraFields.Provider == schemas.Vertex && resp.ExtraFields.RawResponse != nil { + return resp.ExtraFields.RawResponse, nil + } + // Vertex batchPredictionJobs.cancel returns google.protobuf.Empty. + return map[string]interface{}{}, nil + }, + ErrorConverter: func(ctx *schemas.BifrostContext, err *schemas.BifrostError) interface{} { + return gemini.ToGeminiError(err) + }, + PreCallback: extractVertexBatchPathParams, + }) + + // Delete batch prediction job - DELETE .../batchPredictionJobs/{batch_id} + routes = append(routes, RouteConfig{ + Type: RouteConfigTypeGenAI, + Path: itemPath, + Method: "DELETE", + GetHTTPRequestType: func(ctx *fasthttp.RequestCtx) schemas.RequestType { + return schemas.BatchDeleteRequest + }, + GetRequestTypeInstance: func(ctx context.Context) interface{} { + return &schemas.BifrostBatchDeleteRequest{} + }, + BatchRequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*BatchRequest, error) { + if deleteReq, ok := req.(*schemas.BifrostBatchDeleteRequest); ok { + return &BatchRequest{Type: schemas.BatchDeleteRequest, DeleteRequest: deleteReq}, nil + } + return nil, errors.New("invalid vertex batch delete request type") + }, + BatchDeleteResponseConverter: func(ctx *schemas.BifrostContext, resp *schemas.BifrostBatchDeleteResponse) (interface{}, error) { + if resp.ExtraFields.Provider == schemas.Vertex && resp.ExtraFields.RawResponse != nil { + return resp.ExtraFields.RawResponse, nil + } + // Vertex batchPredictionJobs.delete returns a long-running Operation. + return map[string]interface{}{"done": true}, nil + }, + ErrorConverter: func(ctx *schemas.BifrostContext, err *schemas.BifrostError) interface{} { + return gemini.ToGeminiError(err) + }, + PreCallback: extractVertexBatchPathParams, + }) + + return routes +} + +// extractVertexBatchPathParams pins the provider to Vertex and extracts the bare batch_id +// (stripping any :cancel action suffix) for the native Vertex batch routes. The job ID is +// passed bare so the provider resolves project/region from its key config. +func extractVertexBatchPathParams(ctx *fasthttp.RequestCtx, bifrostCtx *schemas.BifrostContext, req interface{}) error { + batchID, _ := ctx.UserValue("batch_id").(string) + batchID = strings.TrimSuffix(batchID, ":cancel") + + switch r := req.(type) { + case *schemas.BifrostBatchListRequest: + r.Provider = schemas.Vertex + if pageSizeStr := string(ctx.QueryArgs().Peek("pageSize")); pageSizeStr != "" { + if pageSize, err := strconv.Atoi(pageSizeStr); err == nil { + r.Limit = pageSize + } + } + if pageToken := string(ctx.QueryArgs().Peek("pageToken")); pageToken != "" { + r.After = &pageToken + } + case *schemas.BifrostBatchRetrieveRequest: + if batchID == "" { + return errors.New("batch_id is required") + } + r.Provider = schemas.Vertex + r.BatchID = batchID + case *schemas.BifrostBatchCancelRequest: + if batchID == "" { + return errors.New("batch_id is required") + } + r.Provider = schemas.Vertex + r.BatchID = batchID + case *schemas.BifrostBatchDeleteRequest: + if batchID == "" { + return errors.New("batch_id is required") + } + r.Provider = schemas.Vertex + r.BatchID = batchID + } + return nil +} + // extractGeminiBatchIDFromPath extracts batch_id from path parameters for Gemini func extractGeminiBatchIDFromPath(ctx *fasthttp.RequestCtx, bifrostCtx *schemas.BifrostContext, req interface{}) error { provider := getProviderFromHeader(ctx, schemas.Gemini) @@ -826,12 +1011,6 @@ func createGenAIRerankRouteConfig(pathPrefix string) RouteConfig { GetRequestTypeInstance: func(ctx context.Context) interface{} { return &vertex.VertexRankRequest{} }, - GetRequestModel: func(_ *fasthttp.RequestCtx, req interface{}) (string, error) { - if r, ok := req.(*vertex.VertexRankRequest); ok && r.Model != nil { - return *r.Model, nil - } - return "", nil - }, RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { if vertexReq, ok := req.(*vertex.VertexRankRequest); ok { return &schemas.BifrostRequest{ @@ -932,210 +1111,219 @@ func extractGeminiCachedContentListQueryParams(ctx *fasthttp.RequestCtx, bifrost func CreateGenAICachedContentRouteConfigs(pathPrefix string, handlerStore lib.HandlerStore) []RouteConfig { var routes []RouteConfig - // POST /v1beta/cachedContents — create - routes = append(routes, RouteConfig{ - Type: RouteConfigTypeGenAI, - Path: pathPrefix + "/v1beta/cachedContents", - Method: "POST", - GetHTTPRequestType: func(ctx *fasthttp.RequestCtx) schemas.RequestType { - return schemas.CachedContentCreateRequest - }, - GetRequestTypeInstance: func(ctx context.Context) interface{} { - return &schemas.BifrostCachedContentCreateRequest{} - }, - RequestParser: func(ctx *fasthttp.RequestCtx, req interface{}) error { - createReq, ok := req.(*schemas.BifrostCachedContentCreateRequest) - if !ok { - return errors.New("invalid cached content create request type") - } - if body := ctx.Request.Body(); len(body) > 0 { - if !gjson.ValidBytes(body) { - return errors.New("invalid JSON") + // Register the lifecycle routes under both the flat Gemini path + // ("/v1beta/cachedContents") and the Vertex regional path + // ("/v1beta/projects/{project}/locations/{location}/cachedContents"). + cachedBasePaths := []string{ + pathPrefix + "/v1beta/cachedContents", + pathPrefix + "/v1beta/projects/{project}/locations/{location}/cachedContents", + } + for _, cachedBase := range cachedBasePaths { + // POST cachedContents — create + routes = append(routes, RouteConfig{ + Type: RouteConfigTypeGenAI, + Path: cachedBase, + Method: "POST", + GetHTTPRequestType: func(ctx *fasthttp.RequestCtx) schemas.RequestType { + return schemas.CachedContentCreateRequest + }, + GetRequestTypeInstance: func(ctx context.Context) interface{} { + return &schemas.BifrostCachedContentCreateRequest{} + }, + RequestParser: func(ctx *fasthttp.RequestCtx, req interface{}) error { + createReq, ok := req.(*schemas.BifrostCachedContentCreateRequest) + if !ok { + return errors.New("invalid cached content create request type") } - createReq.RawRequestBody = copyBytes(body) - model, err := requiredGJSONString(body, "model") - if err != nil { - return err + if body := ctx.Request.Body(); len(body) > 0 { + if !gjson.ValidBytes(body) { + return errors.New("invalid JSON") + } + createReq.RawRequestBody = copyBytes(body) + model, err := requiredGJSONString(body, "model") + if err != nil { + return err + } + displayName, err := optionalGJSONString(body, "displayName") + if err != nil { + return err + } + ttl, err := optionalGJSONString(body, "ttl") + if err != nil { + return err + } + expireTime, err := optionalGJSONString(body, "expireTime") + if err != nil { + return err + } + createReq.Model = strings.TrimPrefix(model, "models/") + createReq.DisplayName = displayName + createReq.TTL = ttl + createReq.ExpireTime = expireTime } - displayName, err := optionalGJSONString(body, "displayName") - if err != nil { - return err + return nil + }, + CachedContentRequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*CachedContentRequest, error) { + createReq, ok := req.(*schemas.BifrostCachedContentCreateRequest) + if !ok { + return nil, errors.New("invalid cached content create request type") } - ttl, err := optionalGJSONString(body, "ttl") - if err != nil { - return err + // Provider is set via PreCallback (setGeminiCachedContentCreateProvider). + if createReq.Provider == "" { + createReq.Provider = schemas.Gemini } - expireTime, err := optionalGJSONString(body, "expireTime") - if err != nil { - return err + if len(createReq.RawRequestBody) > 0 { + ctx.SetValue(schemas.BifrostContextKeyUseRawRequestBody, true) } - createReq.Model = strings.TrimPrefix(model, "models/") - createReq.DisplayName = displayName - createReq.TTL = ttl - createReq.ExpireTime = expireTime - } - return nil - }, - CachedContentRequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*CachedContentRequest, error) { - createReq, ok := req.(*schemas.BifrostCachedContentCreateRequest) - if !ok { - return nil, errors.New("invalid cached content create request type") - } - // Provider is set via PreCallback (setGeminiCachedContentCreateProvider). - if createReq.Provider == "" { - createReq.Provider = schemas.Gemini - } - if len(createReq.RawRequestBody) > 0 { - ctx.SetValue(schemas.BifrostContextKeyUseRawRequestBody, true) - } - return &CachedContentRequest{Type: schemas.CachedContentCreateRequest, CreateRequest: createReq}, nil - }, - CachedContentCreateResponseConverter: func(ctx *schemas.BifrostContext, resp *schemas.BifrostCachedContentCreateResponse) (interface{}, error) { - return gemini.ToGeminiCachedContentCreateResponse(resp), nil - }, - ErrorConverter: func(ctx *schemas.BifrostContext, err *schemas.BifrostError) interface{} { - return gemini.ToGeminiError(err) - }, - PreCallback: setGeminiCachedContentCreateProvider, - }) - - // GET /v1beta/cachedContents — list - routes = append(routes, RouteConfig{ - Type: RouteConfigTypeGenAI, - Path: pathPrefix + "/v1beta/cachedContents", - Method: "GET", - GetHTTPRequestType: func(ctx *fasthttp.RequestCtx) schemas.RequestType { - return schemas.CachedContentListRequest - }, - GetRequestTypeInstance: func(ctx context.Context) interface{} { - return &schemas.BifrostCachedContentListRequest{} - }, - CachedContentRequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*CachedContentRequest, error) { - listReq, ok := req.(*schemas.BifrostCachedContentListRequest) - if !ok { - return nil, errors.New("invalid cached content list request type") - } - return &CachedContentRequest{Type: schemas.CachedContentListRequest, ListRequest: listReq}, nil - }, - CachedContentListResponseConverter: func(ctx *schemas.BifrostContext, resp *schemas.BifrostCachedContentListResponse) (interface{}, error) { - return gemini.ToGeminiCachedContentListResponse(resp), nil - }, - ErrorConverter: func(ctx *schemas.BifrostContext, err *schemas.BifrostError) interface{} { - return gemini.ToGeminiError(err) - }, - PreCallback: extractGeminiCachedContentListQueryParams, - }) - - // GET /v1beta/cachedContents/{cached_id} — retrieve - routes = append(routes, RouteConfig{ - Type: RouteConfigTypeGenAI, - Path: pathPrefix + "/v1beta/cachedContents/{cached_id}", - Method: "GET", - GetHTTPRequestType: func(ctx *fasthttp.RequestCtx) schemas.RequestType { - return schemas.CachedContentRetrieveRequest - }, - GetRequestTypeInstance: func(ctx context.Context) interface{} { - return &schemas.BifrostCachedContentRetrieveRequest{} - }, - CachedContentRequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*CachedContentRequest, error) { - retrieveReq, ok := req.(*schemas.BifrostCachedContentRetrieveRequest) - if !ok { - return nil, errors.New("invalid cached content retrieve request type") - } - return &CachedContentRequest{Type: schemas.CachedContentRetrieveRequest, RetrieveRequest: retrieveReq}, nil - }, - CachedContentRetrieveResponseConverter: func(ctx *schemas.BifrostContext, resp *schemas.BifrostCachedContentRetrieveResponse) (interface{}, error) { - return gemini.ToGeminiCachedContentRetrieveResponse(resp), nil - }, - ErrorConverter: func(ctx *schemas.BifrostContext, err *schemas.BifrostError) interface{} { - return gemini.ToGeminiError(err) - }, - PreCallback: extractGeminiCachedContentNameFromPath, - }) - - // PATCH /v1beta/cachedContents/{cached_id} — update - routes = append(routes, RouteConfig{ - Type: RouteConfigTypeGenAI, - Path: pathPrefix + "/v1beta/cachedContents/{cached_id}", - Method: "PATCH", - GetHTTPRequestType: func(ctx *fasthttp.RequestCtx) schemas.RequestType { - return schemas.CachedContentUpdateRequest - }, - GetRequestTypeInstance: func(ctx context.Context) interface{} { - return &schemas.BifrostCachedContentUpdateRequest{} - }, - RequestParser: func(ctx *fasthttp.RequestCtx, req interface{}) error { - updateReq, ok := req.(*schemas.BifrostCachedContentUpdateRequest) - if !ok { - return errors.New("invalid cached content update request type") - } - if body := ctx.Request.Body(); len(body) > 0 { - if !gjson.ValidBytes(body) { - return errors.New("invalid JSON") + return &CachedContentRequest{Type: schemas.CachedContentCreateRequest, CreateRequest: createReq}, nil + }, + CachedContentCreateResponseConverter: func(ctx *schemas.BifrostContext, resp *schemas.BifrostCachedContentCreateResponse) (interface{}, error) { + return gemini.ToGeminiCachedContentCreateResponse(resp), nil + }, + ErrorConverter: func(ctx *schemas.BifrostContext, err *schemas.BifrostError) interface{} { + return gemini.ToGeminiError(err) + }, + PreCallback: setGeminiCachedContentCreateProvider, + }) + + // GET /v1beta/cachedContents — list + routes = append(routes, RouteConfig{ + Type: RouteConfigTypeGenAI, + Path: cachedBase, + Method: "GET", + GetHTTPRequestType: func(ctx *fasthttp.RequestCtx) schemas.RequestType { + return schemas.CachedContentListRequest + }, + GetRequestTypeInstance: func(ctx context.Context) interface{} { + return &schemas.BifrostCachedContentListRequest{} + }, + CachedContentRequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*CachedContentRequest, error) { + listReq, ok := req.(*schemas.BifrostCachedContentListRequest) + if !ok { + return nil, errors.New("invalid cached content list request type") } - updateReq.RawRequestBody = copyBytes(body) - ttl, err := optionalGJSONString(body, "ttl") - if err != nil { - return err + return &CachedContentRequest{Type: schemas.CachedContentListRequest, ListRequest: listReq}, nil + }, + CachedContentListResponseConverter: func(ctx *schemas.BifrostContext, resp *schemas.BifrostCachedContentListResponse) (interface{}, error) { + return gemini.ToGeminiCachedContentListResponse(resp), nil + }, + ErrorConverter: func(ctx *schemas.BifrostContext, err *schemas.BifrostError) interface{} { + return gemini.ToGeminiError(err) + }, + PreCallback: extractGeminiCachedContentListQueryParams, + }) + + // GET /v1beta/cachedContents/{cached_id} — retrieve + routes = append(routes, RouteConfig{ + Type: RouteConfigTypeGenAI, + Path: cachedBase + "/{cached_id}", + Method: "GET", + GetHTTPRequestType: func(ctx *fasthttp.RequestCtx) schemas.RequestType { + return schemas.CachedContentRetrieveRequest + }, + GetRequestTypeInstance: func(ctx context.Context) interface{} { + return &schemas.BifrostCachedContentRetrieveRequest{} + }, + CachedContentRequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*CachedContentRequest, error) { + retrieveReq, ok := req.(*schemas.BifrostCachedContentRetrieveRequest) + if !ok { + return nil, errors.New("invalid cached content retrieve request type") } - expireTime, err := optionalGJSONString(body, "expireTime") - if err != nil { - return err + return &CachedContentRequest{Type: schemas.CachedContentRetrieveRequest, RetrieveRequest: retrieveReq}, nil + }, + CachedContentRetrieveResponseConverter: func(ctx *schemas.BifrostContext, resp *schemas.BifrostCachedContentRetrieveResponse) (interface{}, error) { + return gemini.ToGeminiCachedContentRetrieveResponse(resp), nil + }, + ErrorConverter: func(ctx *schemas.BifrostContext, err *schemas.BifrostError) interface{} { + return gemini.ToGeminiError(err) + }, + PreCallback: extractGeminiCachedContentNameFromPath, + }) + + // PATCH /v1beta/cachedContents/{cached_id} — update + routes = append(routes, RouteConfig{ + Type: RouteConfigTypeGenAI, + Path: cachedBase + "/{cached_id}", + Method: "PATCH", + GetHTTPRequestType: func(ctx *fasthttp.RequestCtx) schemas.RequestType { + return schemas.CachedContentUpdateRequest + }, + GetRequestTypeInstance: func(ctx context.Context) interface{} { + return &schemas.BifrostCachedContentUpdateRequest{} + }, + RequestParser: func(ctx *fasthttp.RequestCtx, req interface{}) error { + updateReq, ok := req.(*schemas.BifrostCachedContentUpdateRequest) + if !ok { + return errors.New("invalid cached content update request type") } - updateReq.TTL = ttl - updateReq.ExpireTime = expireTime - } - return nil - }, - CachedContentRequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*CachedContentRequest, error) { - updateReq, ok := req.(*schemas.BifrostCachedContentUpdateRequest) - if !ok { - return nil, errors.New("invalid cached content update request type") - } - // Name is set via PreCallback (extractGeminiCachedContentNameFromPath). - if updateReq.Provider == "" { - updateReq.Provider = schemas.Gemini - } - if len(updateReq.RawRequestBody) > 0 { - ctx.SetValue(schemas.BifrostContextKeyUseRawRequestBody, true) - } - return &CachedContentRequest{Type: schemas.CachedContentUpdateRequest, UpdateRequest: updateReq}, nil - }, - CachedContentUpdateResponseConverter: func(ctx *schemas.BifrostContext, resp *schemas.BifrostCachedContentUpdateResponse) (interface{}, error) { - return gemini.ToGeminiCachedContentUpdateResponse(resp), nil - }, - ErrorConverter: func(ctx *schemas.BifrostContext, err *schemas.BifrostError) interface{} { - return gemini.ToGeminiError(err) - }, - PreCallback: extractGeminiCachedContentNameFromPath, - }) - - // DELETE /v1beta/cachedContents/{cached_id} — delete - routes = append(routes, RouteConfig{ - Type: RouteConfigTypeGenAI, - Path: pathPrefix + "/v1beta/cachedContents/{cached_id}", - Method: "DELETE", - GetHTTPRequestType: func(ctx *fasthttp.RequestCtx) schemas.RequestType { - return schemas.CachedContentDeleteRequest - }, - GetRequestTypeInstance: func(ctx context.Context) interface{} { - return &schemas.BifrostCachedContentDeleteRequest{} - }, - CachedContentRequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*CachedContentRequest, error) { - deleteReq, ok := req.(*schemas.BifrostCachedContentDeleteRequest) - if !ok { - return nil, errors.New("invalid cached content delete request type") - } - return &CachedContentRequest{Type: schemas.CachedContentDeleteRequest, DeleteRequest: deleteReq}, nil - }, - CachedContentDeleteResponseConverter: func(ctx *schemas.BifrostContext, resp *schemas.BifrostCachedContentDeleteResponse) (interface{}, error) { - return gemini.ToGeminiCachedContentDeleteResponse(resp), nil - }, - ErrorConverter: func(ctx *schemas.BifrostContext, err *schemas.BifrostError) interface{} { - return gemini.ToGeminiError(err) - }, - PreCallback: extractGeminiCachedContentNameFromPath, - }) + if body := ctx.Request.Body(); len(body) > 0 { + if !gjson.ValidBytes(body) { + return errors.New("invalid JSON") + } + updateReq.RawRequestBody = copyBytes(body) + ttl, err := optionalGJSONString(body, "ttl") + if err != nil { + return err + } + expireTime, err := optionalGJSONString(body, "expireTime") + if err != nil { + return err + } + updateReq.TTL = ttl + updateReq.ExpireTime = expireTime + } + return nil + }, + CachedContentRequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*CachedContentRequest, error) { + updateReq, ok := req.(*schemas.BifrostCachedContentUpdateRequest) + if !ok { + return nil, errors.New("invalid cached content update request type") + } + // Name is set via PreCallback (extractGeminiCachedContentNameFromPath). + if updateReq.Provider == "" { + updateReq.Provider = schemas.Gemini + } + if len(updateReq.RawRequestBody) > 0 { + ctx.SetValue(schemas.BifrostContextKeyUseRawRequestBody, true) + } + return &CachedContentRequest{Type: schemas.CachedContentUpdateRequest, UpdateRequest: updateReq}, nil + }, + CachedContentUpdateResponseConverter: func(ctx *schemas.BifrostContext, resp *schemas.BifrostCachedContentUpdateResponse) (interface{}, error) { + return gemini.ToGeminiCachedContentUpdateResponse(resp), nil + }, + ErrorConverter: func(ctx *schemas.BifrostContext, err *schemas.BifrostError) interface{} { + return gemini.ToGeminiError(err) + }, + PreCallback: extractGeminiCachedContentNameFromPath, + }) + + // DELETE /v1beta/cachedContents/{cached_id} — delete + routes = append(routes, RouteConfig{ + Type: RouteConfigTypeGenAI, + Path: cachedBase + "/{cached_id}", + Method: "DELETE", + GetHTTPRequestType: func(ctx *fasthttp.RequestCtx) schemas.RequestType { + return schemas.CachedContentDeleteRequest + }, + GetRequestTypeInstance: func(ctx context.Context) interface{} { + return &schemas.BifrostCachedContentDeleteRequest{} + }, + CachedContentRequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*CachedContentRequest, error) { + deleteReq, ok := req.(*schemas.BifrostCachedContentDeleteRequest) + if !ok { + return nil, errors.New("invalid cached content delete request type") + } + return &CachedContentRequest{Type: schemas.CachedContentDeleteRequest, DeleteRequest: deleteReq}, nil + }, + CachedContentDeleteResponseConverter: func(ctx *schemas.BifrostContext, resp *schemas.BifrostCachedContentDeleteResponse) (interface{}, error) { + return gemini.ToGeminiCachedContentDeleteResponse(resp), nil + }, + ErrorConverter: func(ctx *schemas.BifrostContext, err *schemas.BifrostError) interface{} { + return gemini.ToGeminiError(err) + }, + PreCallback: extractGeminiCachedContentNameFromPath, + }) + } return routes } @@ -1145,6 +1333,7 @@ func NewGenAIRouter(client *bifrost.Bifrost, handlerStore lib.HandlerStore, logg routes := CreateGenAIRouteConfigs("/genai") routes = append(routes, CreateGenAIFileRouteConfigs("/genai", handlerStore)...) routes = append(routes, CreateGenAIBatchRouteConfigs("/genai", handlerStore)...) + routes = append(routes, CreateVertexBatchRouteConfigs("/genai")...) routes = append(routes, CreateGenAICachedContentRouteConfigs("/genai", handlerStore)...) return &GenAIRouter{ diff --git a/transports/bifrost-http/integrations/openai.go b/transports/bifrost-http/integrations/openai.go index 7b2bfa3d9c..8a97171428 100644 --- a/transports/bifrost-http/integrations/openai.go +++ b/transports/bifrost-http/integrations/openai.go @@ -269,43 +269,14 @@ func AzureEndpointPreHook(handlerStore lib.HandlerStore) func(ctx *fasthttp.Requ } } -// openAIModelGetter extracts the model field from any OpenAI integration request type. -// It is called after body parsing and PreCallback, so req is fully populated. -func openAIModelGetter(_ *fasthttp.RequestCtx, req interface{}) (string, error) { - switch r := req.(type) { - case *openai.OpenAIChatRequest: - return r.Model, nil - case *openai.OpenAITextCompletionRequest: - return r.Model, nil - case *openai.OpenAIEmbeddingRequest: - return r.Model, nil - case *openai.OpenAIResponsesRequest: - return r.Model, nil - case *openai.OpenAISpeechRequest: - return r.Model, nil - case *openai.OpenAITranscriptionRequest: - return r.Model, nil - case *openai.OpenAIImageGenerationRequest: - return r.Model, nil - case *openai.OpenAIImageEditRequest: - return r.Model, nil - case *openai.OpenAIImageVariationRequest: - return r.Model, nil - case *openai.OpenAIVideoGenerationRequest: - return r.Model, nil - } - return "", nil -} - // CreateOpenAIRouteConfigs creates route configurations for OpenAI endpoints. func CreateOpenAIRouteConfigs(pathPrefix string, handlerStore lib.HandlerStore) []RouteConfig { var routes []RouteConfig routes = append(routes, RouteConfig{ - Type: RouteConfigTypeOpenAI, - Path: pathPrefix + "/openai/deployments/{deploymentPath:*}", - Method: "POST", - GetRequestModel: openAIModelGetter, + Type: RouteConfigTypeOpenAI, + Path: pathPrefix + "/openai/deployments/{deploymentPath:*}", + Method: "POST", GetHTTPRequestType: func(ctx *fasthttp.RequestCtx) schemas.RequestType { deploymentPathVal, ok := ctx.UserValue("deploymentPath").(string) if !ok { @@ -578,7 +549,6 @@ func CreateOpenAIRouteConfigs(pathPrefix string, handlerStore lib.HandlerStore) GetRequestTypeInstance: func(ctx context.Context) interface{} { return &openai.OpenAIChatRequest{} }, - GetRequestModel: openAIModelGetter, RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { if openaiReq, ok := req.(*openai.OpenAIChatRequest); ok { br := &schemas.BifrostRequest{ @@ -675,7 +645,6 @@ func CreateOpenAIRouteConfigs(pathPrefix string, handlerStore lib.HandlerStore) GetRequestTypeInstance: func(ctx context.Context) interface{} { return &openai.OpenAITextCompletionRequest{} }, - GetRequestModel: openAIModelGetter, RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { if openaiReq, ok := req.(*openai.OpenAITextCompletionRequest); ok { return &schemas.BifrostRequest{ @@ -727,7 +696,6 @@ func CreateOpenAIRouteConfigs(pathPrefix string, handlerStore lib.HandlerStore) GetRequestTypeInstance: func(ctx context.Context) interface{} { return &openai.OpenAIResponsesRequest{} }, - GetRequestModel: openAIModelGetter, RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { if openaiReq, ok := req.(*openai.OpenAIResponsesRequest); ok { return &schemas.BifrostRequest{ @@ -810,7 +778,6 @@ func CreateOpenAIRouteConfigs(pathPrefix string, handlerStore lib.HandlerStore) GetRequestTypeInstance: func(ctx context.Context) interface{} { return &openai.OpenAIResponsesRequest{} }, - GetRequestModel: openAIModelGetter, RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { if openaiReq, ok := req.(*openai.OpenAIResponsesRequest); ok { return &schemas.BifrostRequest{ @@ -840,9 +807,9 @@ func CreateOpenAIRouteConfigs(pathPrefix string, handlerStore lib.HandlerStore) "/openai/responses/compact", } { routes = append(routes, RouteConfig{ - Type: RouteConfigTypeOpenAI, - Path: pathPrefix + path, - Method: "POST", + Type: RouteConfigTypeOpenAI, + Path: pathPrefix + path, + Method: "POST", PreCallback: func(ctx *fasthttp.RequestCtx, bifrostCtx *schemas.BifrostContext, req interface{}) error { hydrateOpenAIRequestFromLargePayloadMetadata(ctx, bifrostCtx, req) schemas.ExtractAndSetUserAgentFromHeaders(extractHeadersFromRequest(ctx), bifrostCtx) @@ -857,12 +824,6 @@ func CreateOpenAIRouteConfigs(pathPrefix string, handlerStore lib.HandlerStore) GetRequestTypeInstance: func(ctx context.Context) interface{} { return &openai.OpenAICompactionRequest{} }, - GetRequestModel: func(_ *fasthttp.RequestCtx, req interface{}) (string, error) { - if r, ok := req.(*openai.OpenAICompactionRequest); ok { - return r.Model, nil - } - return "", nil - }, RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { if r, ok := req.(*openai.OpenAICompactionRequest); ok { return &schemas.BifrostRequest{ @@ -901,7 +862,6 @@ func CreateOpenAIRouteConfigs(pathPrefix string, handlerStore lib.HandlerStore) GetRequestTypeInstance: func(ctx context.Context) interface{} { return &openai.OpenAIEmbeddingRequest{} }, - GetRequestModel: openAIModelGetter, RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { if embeddingReq, ok := req.(*openai.OpenAIEmbeddingRequest); ok { return &schemas.BifrostRequest{ @@ -940,7 +900,6 @@ func CreateOpenAIRouteConfigs(pathPrefix string, handlerStore lib.HandlerStore) GetRequestTypeInstance: func(ctx context.Context) interface{} { return &openai.OpenAISpeechRequest{} }, - GetRequestModel: openAIModelGetter, RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { if speechReq, ok := req.(*openai.OpenAISpeechRequest); ok { return &schemas.BifrostRequest{ @@ -984,8 +943,7 @@ func CreateOpenAIRouteConfigs(pathPrefix string, handlerStore lib.HandlerStore) GetRequestTypeInstance: func(ctx context.Context) interface{} { return &openai.OpenAITranscriptionRequest{} }, - GetRequestModel: openAIModelGetter, - RequestParser: parseTranscriptionMultipartRequest, // Handle multipart form parsing + RequestParser: parseTranscriptionMultipartRequest, // Handle multipart form parsing RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { if transcriptionReq, ok := req.(*openai.OpenAITranscriptionRequest); ok { return &schemas.BifrostRequest{ @@ -1040,7 +998,6 @@ func CreateOpenAIRouteConfigs(pathPrefix string, handlerStore lib.HandlerStore) GetRequestTypeInstance: func(ctx context.Context) interface{} { return &openai.OpenAIImageGenerationRequest{} }, - GetRequestModel: openAIModelGetter, RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { if imageGenReq, ok := req.(*openai.OpenAIImageGenerationRequest); ok { return &schemas.BifrostRequest{ @@ -1091,8 +1048,7 @@ func CreateOpenAIRouteConfigs(pathPrefix string, handlerStore lib.HandlerStore) GetRequestTypeInstance: func(ctx context.Context) interface{} { return &openai.OpenAIImageEditRequest{} }, - GetRequestModel: openAIModelGetter, - RequestParser: parseOpenAIImageEditMultipartRequest, // Handle multipart form parsing + RequestParser: parseOpenAIImageEditMultipartRequest, // Handle multipart form parsing RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { if imageEditReq, ok := req.(*openai.OpenAIImageEditRequest); ok { return &schemas.BifrostRequest{ @@ -1142,8 +1098,7 @@ func CreateOpenAIRouteConfigs(pathPrefix string, handlerStore lib.HandlerStore) GetRequestTypeInstance: func(ctx context.Context) interface{} { return &openai.OpenAIImageVariationRequest{} }, - GetRequestModel: openAIModelGetter, - RequestParser: parseOpenAIImageVariationMultipartRequest, + RequestParser: parseOpenAIImageVariationMultipartRequest, RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { if imageVariationReq, ok := req.(*openai.OpenAIImageVariationRequest); ok { return &schemas.BifrostRequest{ @@ -1195,8 +1150,7 @@ func CreateOpenAIRouteConfigs(pathPrefix string, handlerStore lib.HandlerStore) GetRequestTypeInstance: func(ctx context.Context) interface{} { return &openai.OpenAIVideoGenerationRequest{} }, - GetRequestModel: openAIModelGetter, - RequestParser: parseOpenAIVideoGenerationMultipartRequest, + RequestParser: parseOpenAIVideoGenerationMultipartRequest, RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { if videoGenerationReq, ok := req.(*openai.OpenAIVideoGenerationRequest); ok { return &schemas.BifrostRequest{ @@ -1467,6 +1421,12 @@ func CreateOpenAIBatchRouteConfigs(pathPrefix string, handlerStore lib.HandlerSt openaiReq.InputFileID = string(decodedFileID) } } + case schemas.Vertex: + if openaiReq.InputFileID != "" { + if decodedFileID, err := base64.RawURLEncoding.DecodeString(openaiReq.InputFileID); err == nil { + openaiReq.InputFileID = string(decodedFileID) + } + } } return &BatchRequest{ Type: schemas.BatchCreateRequest, @@ -1483,6 +1443,12 @@ func CreateOpenAIBatchRouteConfigs(pathPrefix string, handlerStore lib.HandlerSt case schemas.Bedrock: resp.ID = base64.StdEncoding.EncodeToString([]byte(resp.ID)) resp.InputFileID = base64.StdEncoding.EncodeToString([]byte(resp.InputFileID)) + case schemas.Vertex: + // id is a full resource name (projects/.../batchPredictionJobs/{id}) and + // input_file_id is a gs:// URI; both contain slashes. RawURLEncoding keeps + // them path-safe so callers can use them in retrieve/cancel without escaping. + resp.ID = base64.RawURLEncoding.EncodeToString([]byte(resp.ID)) + resp.InputFileID = base64.RawURLEncoding.EncodeToString([]byte(resp.InputFileID)) } return resp, nil }, @@ -1549,13 +1515,15 @@ func CreateOpenAIBatchRouteConfigs(pathPrefix string, handlerStore lib.HandlerSt } } - // For Azure, extract inline requests from raw body - if createReq.Provider == schemas.Azure { + // Azure (input_blob + output_folder) and Vertex (output_folder, a gs:// prefix) + // carry their storage location in the request body rather than a managed file. + if createReq.Provider == schemas.Azure || createReq.Provider == schemas.Vertex { var extraFields map[string]interface{} if err := json.Unmarshal(ctx.Request.Body(), &extraFields); err == nil { - // Extract requests array for inline batching - if inputBlob, ok := extraFields["input_blob"].(string); ok { - createReq.InputBlob = &inputBlob + if createReq.Provider == schemas.Azure { + if inputBlob, ok := extraFields["input_blob"].(string); ok { + createReq.InputBlob = &inputBlob + } } if outputFolder, ok := extraFields["output_folder"].(map[string]interface{}); ok { outputURL, ok := outputFolder["url"].(string) @@ -1614,6 +1582,11 @@ func CreateOpenAIBatchRouteConfigs(pathPrefix string, handlerStore lib.HandlerSt resp.Data[i].ID = base64.StdEncoding.EncodeToString([]byte(batch.ID)) resp.Data[i].InputFileID = base64.StdEncoding.EncodeToString([]byte(batch.InputFileID)) } + case schemas.Vertex: + for i, batch := range resp.Data { + resp.Data[i].ID = base64.RawURLEncoding.EncodeToString([]byte(batch.ID)) + resp.Data[i].InputFileID = base64.RawURLEncoding.EncodeToString([]byte(batch.InputFileID)) + } } return resp, nil }, @@ -1653,6 +1626,11 @@ func CreateOpenAIBatchRouteConfigs(pathPrefix string, handlerStore lib.HandlerSt if decodedBatchID, err := base64.StdEncoding.DecodeString(retrieveReq.BatchID); err == nil { retrieveReq.BatchID = string(decodedBatchID) } + case schemas.Vertex: + // Reverse the RawURLEncoding applied to the full resource name. + if decodedBatchID, err := base64.RawURLEncoding.DecodeString(retrieveReq.BatchID); err == nil { + retrieveReq.BatchID = string(decodedBatchID) + } } return &BatchRequest{ Type: schemas.BatchRetrieveRequest, @@ -1669,6 +1647,9 @@ func CreateOpenAIBatchRouteConfigs(pathPrefix string, handlerStore lib.HandlerSt case schemas.Bedrock: resp.ID = base64.StdEncoding.EncodeToString([]byte(resp.ID)) resp.InputFileID = base64.StdEncoding.EncodeToString([]byte(resp.InputFileID)) + case schemas.Vertex: + resp.ID = base64.RawURLEncoding.EncodeToString([]byte(resp.ID)) + resp.InputFileID = base64.RawURLEncoding.EncodeToString([]byte(resp.InputFileID)) } return resp, nil }, @@ -1708,6 +1689,11 @@ func CreateOpenAIBatchRouteConfigs(pathPrefix string, handlerStore lib.HandlerSt if decodedBatchID, err := base64.StdEncoding.DecodeString(cancelReq.BatchID); err == nil { cancelReq.BatchID = string(decodedBatchID) } + case schemas.Vertex: + // Reverse the RawURLEncoding applied to the full resource name. + if decodedBatchID, err := base64.RawURLEncoding.DecodeString(cancelReq.BatchID); err == nil { + cancelReq.BatchID = string(decodedBatchID) + } } return &BatchRequest{ Type: schemas.BatchCancelRequest, @@ -1722,6 +1708,8 @@ func CreateOpenAIBatchRouteConfigs(pathPrefix string, handlerStore lib.HandlerSt resp.ID = strings.Replace(resp.ID, "batches/", "batches-", 1) case schemas.Bedrock: resp.ID = base64.StdEncoding.EncodeToString([]byte(resp.ID)) + case schemas.Vertex: + resp.ID = base64.RawURLEncoding.EncodeToString([]byte(resp.ID)) } return resp, nil }, @@ -1772,7 +1760,13 @@ func CreateOpenAIFileRouteConfigs(pathPrefix string, handlerStore lib.HandlerSto case schemas.Gemini: resp.ID = strings.Replace(resp.ID, "files/", "files-", 1) case schemas.Bedrock: + // s3:// ids contain slashes that break single-segment path routing; + // encode to an opaque id (StdEncoding, as originally shipped for Bedrock). resp.ID = base64.StdEncoding.EncodeToString([]byte(resp.ID)) + case schemas.Vertex: + // gs:// ids: RawURLEncoding is fully path-safe (no /, +, or = padding), + // so the id never needs percent-encoding. Matches the native handler. + resp.ID = base64.RawURLEncoding.EncodeToString([]byte(resp.ID)) default: return resp, nil } @@ -1838,6 +1832,10 @@ func CreateOpenAIFileRouteConfigs(pathPrefix string, handlerStore lib.HandlerSto for i, file := range resp.Data { resp.Data[i].ID = base64.StdEncoding.EncodeToString([]byte(file.ID)) } + case schemas.Vertex: + for i, file := range resp.Data { + resp.Data[i].ID = base64.RawURLEncoding.EncodeToString([]byte(file.ID)) + } } return resp, nil }, @@ -1885,7 +1883,13 @@ func CreateOpenAIFileRouteConfigs(pathPrefix string, handlerStore lib.HandlerSto case schemas.Gemini: resp.ID = strings.Replace(resp.ID, "files/", "files-", 1) case schemas.Bedrock: + // s3:// ids contain slashes that break single-segment path routing; + // encode to an opaque id (StdEncoding, as originally shipped for Bedrock). resp.ID = base64.StdEncoding.EncodeToString([]byte(resp.ID)) + case schemas.Vertex: + // gs:// ids: RawURLEncoding is fully path-safe (no /, +, or = padding), + // so the id never needs percent-encoding. Matches the native handler. + resp.ID = base64.RawURLEncoding.EncodeToString([]byte(resp.ID)) default: return resp, nil } @@ -1937,7 +1941,13 @@ func CreateOpenAIFileRouteConfigs(pathPrefix string, handlerStore lib.HandlerSto case schemas.Gemini: resp.ID = strings.Replace(resp.ID, "files/", "files-", 1) case schemas.Bedrock: + // s3:// ids contain slashes that break single-segment path routing; + // encode to an opaque id (StdEncoding, as originally shipped for Bedrock). resp.ID = base64.StdEncoding.EncodeToString([]byte(resp.ID)) + case schemas.Vertex: + // gs:// ids: RawURLEncoding is fully path-safe (no /, +, or = padding), + // so the id never needs percent-encoding. Matches the native handler. + resp.ID = base64.RawURLEncoding.EncodeToString([]byte(resp.ID)) default: return resp, nil } @@ -2164,6 +2174,28 @@ func extractFileListQueryParams(_ lib.HandlerStore) PreRequestCallback { } } + // Extract GCS storage config for Vertex (bracket notation: storage_config[gcs][bucket]) + if listReq.Provider == schemas.Vertex { + if gcsBucket := string(ctx.QueryArgs().Peek("storage_config[gcs][bucket]")); gcsBucket != "" { + if listReq.StorageConfig == nil { + listReq.StorageConfig = &schemas.FileStorageConfig{} + } + if listReq.StorageConfig.GCS == nil { + listReq.StorageConfig.GCS = &schemas.GCSStorageConfig{} + } + listReq.StorageConfig.GCS.Bucket = gcsBucket + } + if gcsPrefix := string(ctx.QueryArgs().Peek("storage_config[gcs][prefix]")); gcsPrefix != "" { + if listReq.StorageConfig == nil { + listReq.StorageConfig = &schemas.FileStorageConfig{} + } + if listReq.StorageConfig.GCS == nil { + listReq.StorageConfig.GCS = &schemas.GCSStorageConfig{} + } + listReq.StorageConfig.GCS.Prefix = gcsPrefix + } + } + // Extract purpose filter if purpose := string(ctx.QueryArgs().Peek("purpose")); purpose != "" { listReq.Purpose = schemas.FilePurpose(purpose) @@ -2215,7 +2247,15 @@ func extractFileIDFromPath(_ lib.HandlerStore) PreRequestCallback { } var storageConfig *schemas.FileStorageConfig - if provider == schemas.Bedrock { + if provider == schemas.Vertex { + // Vertex file ids are RawURL-base64-encoded gs:// URIs (see file response + // converters); decode back. The bucket is parsed from the gs:// URI by the + // provider, so no storage config is needed. This branch is provider-gated, + // so no extra gs:// guard is needed. + if decodedFileID, err := base64.RawURLEncoding.DecodeString(fileIDStr); err == nil { + fileIDStr = string(decodedFileID) + } + } else if provider == schemas.Bedrock { // Check fileIDStr is base64 encoded if decodedFileID, err := base64.StdEncoding.DecodeString(fileIDStr); err == nil { fileIDStr = string(decodedFileID) @@ -2350,6 +2390,28 @@ func parseOpenAIFileUploadMultipartRequest(ctx *fasthttp.RequestCtx, req interfa } } + // Extract GCS storage config for Vertex (bracket notation: storage_config[gcs][bucket]) + if uploadReq.Provider == schemas.Vertex { + if gcsBucketValues := form.Value["storage_config[gcs][bucket]"]; len(gcsBucketValues) > 0 && gcsBucketValues[0] != "" { + if uploadReq.StorageConfig == nil { + uploadReq.StorageConfig = &schemas.FileStorageConfig{} + } + if uploadReq.StorageConfig.GCS == nil { + uploadReq.StorageConfig.GCS = &schemas.GCSStorageConfig{} + } + uploadReq.StorageConfig.GCS.Bucket = gcsBucketValues[0] + } + if gcsPrefixValues := form.Value["storage_config[gcs][prefix]"]; len(gcsPrefixValues) > 0 && gcsPrefixValues[0] != "" { + if uploadReq.StorageConfig == nil { + uploadReq.StorageConfig = &schemas.FileStorageConfig{} + } + if uploadReq.StorageConfig.GCS == nil { + uploadReq.StorageConfig.GCS = &schemas.GCSStorageConfig{} + } + uploadReq.StorageConfig.GCS.Prefix = gcsPrefixValues[0] + } + } + return nil } diff --git a/transports/bifrost-http/integrations/router.go b/transports/bifrost-http/integrations/router.go index 6ba2fc8b5d..50c17bfa73 100644 --- a/transports/bifrost-http/integrations/router.go +++ b/transports/bifrost-http/integrations/router.go @@ -55,7 +55,6 @@ import ( "io" "mime" "mime/multipart" - "slices" "strconv" "strings" @@ -397,10 +396,6 @@ type PostRequestCallback func(ctx *fasthttp.RequestCtx, req interface{}, resp in // returns a schemas.RequestType indicating the HTTP request type derived from the context. type HTTPRequestTypeGetter func(ctx *fasthttp.RequestCtx) schemas.RequestType -// RequestModelGetter is a function type that accepts only a *fasthttp.RequestCtx and -// returns a string indicating the model derived from the context. -type RequestModelGetter func(ctx *fasthttp.RequestCtx, req interface{}) (string, error) - // ShortCircuit is a function that determines if the request should be short-circuited. type ShortCircuit func(ctx *fasthttp.RequestCtx, bifrostCtx *schemas.BifrostContext, req interface{}) (bool, error) @@ -444,14 +439,6 @@ const ( RouteConfigTypeCohere RouteConfigType = "cohere" ) -var RouteConfigTypeToProvider = map[RouteConfigType]schemas.ModelProvider{ - RouteConfigTypeOpenAI: schemas.OpenAI, - RouteConfigTypeAnthropic: schemas.Anthropic, - RouteConfigTypeGenAI: schemas.Gemini, - RouteConfigTypeBedrock: schemas.Bedrock, - RouteConfigTypeCohere: schemas.Cohere, -} - // RouteConfig defines the configuration for a single route in an integration. // It specifies the path, method, and handlers for request/response conversion. type RouteConfig struct { @@ -459,7 +446,6 @@ type RouteConfig struct { Path string // HTTP path pattern (e.g., "/openai/v1/chat/completions") Method string // HTTP method (POST, GET, PUT, DELETE) GetHTTPRequestType HTTPRequestTypeGetter // Function to get the HTTP request type from the context (SHOULD NOT BE NIL) - GetRequestModel RequestModelGetter // Function to get the model from the context (SHOULD NOT BE NIL) GetRequestTypeInstance func(ctx context.Context) interface{} // Factory function to create request instance (SHOULD NOT BE NIL) RequestParser RequestParser // Optional: custom request parsing (e.g., multipart/form-data) RequestConverter RequestConverter // Function to convert request to BifrostRequest (for inference requests) @@ -644,6 +630,8 @@ func (g *GenericRouter) RegisterRoutes(r *router.Router, middlewares ...schemas. r.PUT(route.Path, lib.ChainMiddlewares(handler, routeMiddlewares...)) case fasthttp.MethodDelete: r.DELETE(route.Path, lib.ChainMiddlewares(handler, routeMiddlewares...)) + case fasthttp.MethodPatch: + r.PATCH(route.Path, lib.ChainMiddlewares(handler, routeMiddlewares...)) case fasthttp.MethodHead: r.HEAD(route.Path, lib.ChainMiddlewares(handler, routeMiddlewares...)) default: @@ -690,7 +678,9 @@ func (g *GenericRouter) createHandler(config RouteConfig) fasthttp.RequestHandle } }() - // Set integration type to context + // Set integration type to context. Used by the ModelCatalogResolver built-in + // PreRequestHook (last routing layer) to prefer this integration's canonical + // provider when the model is unprefixed and the catalog returns multiple options. bifrostCtx.SetValue(schemas.BifrostContextKeyIntegrationType, string(config.Type)) // Async retrieve: check x-bf-async-id header early (before body parsing) @@ -781,75 +771,6 @@ func (g *GenericRouter) createHandler(config RouteConfig) fasthttp.RequestHandle } } - // Set available providers to context - if config.GetRequestModel != nil { - model, err := config.GetRequestModel(ctx, req) - if err != nil { - g.sendError(ctx, bifrostCtx, config.ErrorConverter, newBifrostError(err, "failed to get model from context")) - return - } - extractedProvider, extractedModel := schemas.ParseModelString(model, "") - // Skip model-catalog when governance already made a routing decision. - // Governance uses dot-notation aliases (e.g. "anthropic.claude-sonnet-4-6") which - // ParseModelString cannot extract a provider from (it only handles slash separators), - // causing a spurious model-catalog lookup that can override governance's selection. - skipModelCatalogProviderSelection, _ := bifrostCtx.Value(schemas.BifrostContextKeySkipModelCatalogProviderSelection).(bool) - if extractedProvider == "" && !skipModelCatalogProviderSelection { - availableProviders := g.handlerStore.GetProvidersForModel(extractedModel) - existingProviders, hasExistingProviders := bifrostCtx.Value(schemas.BifrostContextKeyAvailableProviders).([]schemas.ModelProvider) - if hasExistingProviders { - if len(existingProviders) == 0 { - availableProviders = []schemas.ModelProvider{} - } else if len(availableProviders) == 0 { - availableProviders = existingProviders - } else { - availableProviders = slices.DeleteFunc(availableProviders, func(provider schemas.ModelProvider) bool { - return !slices.Contains(existingProviders, provider) - }) - } - } - availableProvidersStrs := make([]string, len(availableProviders)) - for i, p := range availableProviders { - availableProvidersStrs[i] = string(p) - } - bifrostCtx.AppendRoutingEngineLog(schemas.RoutingEngineModelCatalog, schemas.LogLevelInfo, fmt.Sprintf( - "No provider specified for model %s, found %d options in model catalog: [%s]", - extractedModel, len(availableProviders), strings.Join(availableProvidersStrs, ", "), - )) - if len(availableProviders) > 0 { - if slices.Contains(availableProviders, RouteConfigTypeToProvider[config.Type]) { - availableProviders = []schemas.ModelProvider{RouteConfigTypeToProvider[config.Type]} - bifrostCtx.AppendRoutingEngineLog(schemas.RoutingEngineModelCatalog, schemas.LogLevelInfo, fmt.Sprintf( - "Integration route default provider %s is found in the available providers list, selecting it", - RouteConfigTypeToProvider[config.Type], - )) - } else { - bifrostCtx.AppendRoutingEngineLog(schemas.RoutingEngineModelCatalog, schemas.LogLevelInfo, fmt.Sprintf( - "Integration route default provider %s is not found in the available providers list, selecting first: %s", - RouteConfigTypeToProvider[config.Type], availableProviders[0], - )) - // For Anthropic-type routes, raw request body passthrough is only valid for - // providers that speak the Anthropic Messages API natively. When the model - // catalog falls back to a provider that doesn't (e.g. Bedrock), clear the - // flag so the provider performs its own format conversion. - firstProvider := availableProviders[0] - if config.Type == RouteConfigTypeAnthropic && - firstProvider != schemas.Anthropic && - firstProvider != schemas.Vertex && - firstProvider != schemas.Azure { - bifrostCtx.SetValue(schemas.BifrostContextKeyUseRawRequestBody, false) - bifrostCtx.SetValue(schemas.BifrostContextKeySendBackRawResponse, false) - bifrostCtx.SetValue(schemas.BifrostContextKeyPassthroughOverridesPresent, false) - } - } - bifrostCtx.SetValue(schemas.BifrostContextKeyAvailableProviders, availableProviders) - } else if hasExistingProviders { - bifrostCtx.SetValue(schemas.BifrostContextKeyAvailableProviders, []schemas.ModelProvider{}) - } - schemas.AppendToContextList(bifrostCtx, schemas.BifrostContextKeyRoutingEnginesUsed, schemas.RoutingEngineModelCatalog) - } - } - // Handle batch requests if BatchRequestConverter is set // GenAI has two cases: (1) Dedicated batch routes (list/retrieve) have only BatchRequestConverter — always use batch path. // (2) The models path has both BatchRequestConverter and RequestConverter — use batch path only for batch create. diff --git a/transports/bifrost-http/integrations/router_test.go b/transports/bifrost-http/integrations/router_test.go index f2bef00469..7acbc15318 100644 --- a/transports/bifrost-http/integrations/router_test.go +++ b/transports/bifrost-http/integrations/router_test.go @@ -377,95 +377,6 @@ func TestOpenAIChatStructuredOutputRequestParserAndConverter(t *testing.T) { assert.Contains(t, responseFormat, "json_schema") } -func TestCreateHandler_AnthropicRouteConstrainsCatalogProvidersWhenAvailableProvidersSet(t *testing.T) { - handlerStore := &mockHandlerStore{ - availableProviders: []schemas.ModelProvider{ - schemas.Anthropic, - schemas.Azure, - schemas.Bedrock, - schemas.Vertex, - }, - } - - var capturedProviders []schemas.ModelProvider - route := RouteConfig{ - Type: RouteConfigTypeAnthropic, - Path: "/v1/messages", - Method: fasthttp.MethodPost, - GetHTTPRequestType: func(ctx *fasthttp.RequestCtx) schemas.RequestType { - return schemas.ResponsesRequest - }, - GetRequestTypeInstance: func(ctx context.Context) interface{} { - return &anthropic.AnthropicMessageRequest{} - }, - GetRequestModel: anthropicModelGetter, - PreCallback: checkAnthropicPassthrough, - RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { - capturedProviders, _ = ctx.Value(schemas.BifrostContextKeyAvailableProviders).([]schemas.ModelProvider) - return nil, fmt.Errorf("stop before bifrost execution") - }, - ErrorConverter: func(ctx *schemas.BifrostContext, err *schemas.BifrostError) interface{} { - return err - }, - } - - router := NewGenericRouter(nil, handlerStore, nil, nil, nil) - ctx := &fasthttp.RequestCtx{} - ctx.Request.Header.SetMethod(fasthttp.MethodPost) - ctx.SetUserValue(schemas.BifrostContextKeyAvailableProviders, []schemas.ModelProvider{ - schemas.Azure, - schemas.OpenAI, - schemas.Ollama, - }) - ctx.Request.SetBodyString(`{"model":"claude-opus-4-8","max_tokens":1024,"messages":[{"role":"user","content":"hi"}]}`) - - router.createHandler(route)(ctx) - - require.Equal(t, fasthttp.StatusInternalServerError, ctx.Response.StatusCode()) - require.Equal(t, []schemas.ModelProvider{schemas.Azure}, capturedProviders) -} - -func TestCreateHandler_AnthropicRouteKeepsCatalogProvidersWhenAvailableProvidersUnset(t *testing.T) { - handlerStore := &mockHandlerStore{ - availableProviders: []schemas.ModelProvider{ - schemas.Bedrock, - schemas.Vertex, - }, - } - - var capturedProviders []schemas.ModelProvider - route := RouteConfig{ - Type: RouteConfigTypeAnthropic, - Path: "/v1/messages", - Method: fasthttp.MethodPost, - GetHTTPRequestType: func(ctx *fasthttp.RequestCtx) schemas.RequestType { - return schemas.ResponsesRequest - }, - GetRequestTypeInstance: func(ctx context.Context) interface{} { - return &anthropic.AnthropicMessageRequest{} - }, - GetRequestModel: anthropicModelGetter, - PreCallback: checkAnthropicPassthrough, - RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { - capturedProviders, _ = ctx.Value(schemas.BifrostContextKeyAvailableProviders).([]schemas.ModelProvider) - return nil, fmt.Errorf("stop before bifrost execution") - }, - ErrorConverter: func(ctx *schemas.BifrostContext, err *schemas.BifrostError) interface{} { - return err - }, - } - - router := NewGenericRouter(nil, handlerStore, nil, nil, nil) - ctx := &fasthttp.RequestCtx{} - ctx.Request.Header.SetMethod(fasthttp.MethodPost) - ctx.Request.SetBodyString(`{"model":"claude-opus-4-8","max_tokens":1024,"messages":[{"role":"user","content":"hi"}]}`) - - router.createHandler(route)(ctx) - - require.Equal(t, fasthttp.StatusInternalServerError, ctx.Response.StatusCode()) - require.Equal(t, []schemas.ModelProvider{schemas.Bedrock, schemas.Vertex}, capturedProviders) -} - func TestCreateHandler_AnthropicRouteClears_UseRawRequestBody_WhenCatalogSelectsBedrock(t *testing.T) { handlerStore := &mockHandlerStore{ availableProviders: []schemas.ModelProvider{schemas.Bedrock}, @@ -484,8 +395,7 @@ func TestCreateHandler_AnthropicRouteClears_UseRawRequestBody_WhenCatalogSelects GetRequestTypeInstance: func(ctx context.Context) interface{} { return &anthropic.AnthropicMessageRequest{} }, - GetRequestModel: anthropicModelGetter, - PreCallback: checkAnthropicPassthrough, + PreCallback: checkAnthropicPassthrough, RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { capturedUseRaw = ctx.Value(schemas.BifrostContextKeyUseRawRequestBody) capturedSendRawResponse = ctx.Value(schemas.BifrostContextKeySendBackRawResponse) diff --git a/transports/bifrost-http/integrations/utils.go b/transports/bifrost-http/integrations/utils.go index 29be18477c..e4c54590b6 100644 --- a/transports/bifrost-http/integrations/utils.go +++ b/transports/bifrost-http/integrations/utils.go @@ -5,7 +5,6 @@ import ( "fmt" "net/url" "reflect" - "slices" "strconv" "strings" @@ -349,11 +348,6 @@ func (g *GenericRouter) extractAndParseFallbacks(ctx *schemas.BifrostContext, re } provider, _, _ := bifrostReq.GetRequestFields() - var availableProviders []schemas.ModelProvider - var hasAvailableProviders bool - if ctx != nil { - availableProviders, hasAvailableProviders = ctx.Value(schemas.BifrostContextKeyAvailableProviders).([]schemas.ModelProvider) - } // Parse fallbacks from strings to Fallback structs parsedFallbacks := make([]schemas.Fallback, 0, len(fallbacks)) @@ -364,9 +358,6 @@ func (g *GenericRouter) extractAndParseFallbacks(ctx *schemas.BifrostContext, re // Use ParseModelString to extract provider and model provider, model := schemas.ParseModelString(fallbackStr, provider) - if hasAvailableProviders && !slices.Contains(availableProviders, provider) { - continue - } parsedFallback := schemas.Fallback{ Provider: provider, @@ -376,7 +367,6 @@ func (g *GenericRouter) extractAndParseFallbacks(ctx *schemas.BifrostContext, re } if len(parsedFallbacks) == 0 { - bifrostReq.SetFallbacks(nil) return nil // No valid fallbacks found } diff --git a/transports/bifrost-http/integrations/utils_test.go b/transports/bifrost-http/integrations/utils_test.go index e1e1dd09d3..50fe7e8216 100644 --- a/transports/bifrost-http/integrations/utils_test.go +++ b/transports/bifrost-http/integrations/utils_test.go @@ -69,59 +69,6 @@ func TestExtractAndParseFallbacks_GeminiGenerationRequest(t *testing.T) { assert.Equal(t, "gemini-3-flash-preview", bifrostReq.ResponsesRequest.Fallbacks[0].Model) } -func TestExtractAndParseFallbacks_FiltersByAvailableProviders(t *testing.T) { - router := newTestGenericRouter() - geminiReq := &gemini.GeminiGenerationRequest{ - Model: "gemini/gemini-3-flash-preview", - Fallbacks: []string{ - "azure/claude-opus-4-8", - "bedrock/claude-opus-4-8", - "vertex/claude-opus-4-8", - }, - } - bifrostReq := &schemas.BifrostRequest{ - ResponsesRequest: &schemas.BifrostResponsesRequest{ - Provider: schemas.Gemini, - Model: "gemini-3-flash-preview", - }, - } - ctx := newTestBifrostContext() - ctx.SetValue(schemas.BifrostContextKeyAvailableProviders, []schemas.ModelProvider{schemas.Azure}) - - err := router.extractAndParseFallbacks(ctx, geminiReq, bifrostReq) - - require.NoError(t, err) - require.NotNil(t, bifrostReq.ResponsesRequest) - require.Len(t, bifrostReq.ResponsesRequest.Fallbacks, 1) - assert.Equal(t, schemas.Azure, bifrostReq.ResponsesRequest.Fallbacks[0].Provider) - assert.Equal(t, "claude-opus-4-8", bifrostReq.ResponsesRequest.Fallbacks[0].Model) -} - -func TestExtractAndParseFallbacks_ClearsDisallowedPreparsedFallbacks(t *testing.T) { - router := newTestGenericRouter() - geminiReq := &gemini.GeminiGenerationRequest{ - Model: "gemini/gemini-3-flash-preview", - Fallbacks: []string{"bedrock/claude-opus-4-8"}, - } - bifrostReq := &schemas.BifrostRequest{ - ResponsesRequest: &schemas.BifrostResponsesRequest{ - Provider: schemas.Gemini, - Model: "gemini-3-flash-preview", - Fallbacks: []schemas.Fallback{ - {Provider: schemas.Bedrock, Model: "claude-opus-4-8"}, - }, - }, - } - ctx := newTestBifrostContext() - ctx.SetValue(schemas.BifrostContextKeyAvailableProviders, []schemas.ModelProvider{schemas.Azure}) - - err := router.extractAndParseFallbacks(ctx, geminiReq, bifrostReq) - - require.NoError(t, err) - require.NotNil(t, bifrostReq.ResponsesRequest) - require.Empty(t, bifrostReq.ResponsesRequest.Fallbacks) -} - // TestSendStreamError_PropagatesProviderStatusCode verifies that sendStreamError // sets the HTTP status code from the provider's BifrostError.StatusCode field. // All three providers (OpenAI, Anthropic, Bedrock) return actual HTTP error codes diff --git a/transports/bifrost-http/lib/config.go b/transports/bifrost-http/lib/config.go index c9a05f5d97..351c36dfe2 100644 --- a/transports/bifrost-http/lib/config.go +++ b/transports/bifrost-http/lib/config.go @@ -41,6 +41,7 @@ import ( "github.com/maximhq/bifrost/framework/vectorstore" "github.com/maximhq/bifrost/plugins/compat" "github.com/maximhq/bifrost/plugins/governance" + "github.com/maximhq/bifrost/plugins/governance/complexity" "github.com/maximhq/bifrost/plugins/logging" "github.com/maximhq/bifrost/plugins/maxim" "github.com/maximhq/bifrost/plugins/otel" @@ -65,8 +66,6 @@ type StreamChunkInterceptor interface { type HandlerStore interface { // GetHeaderMatcher returns the precompiled header matcher for header filtering GetHeaderMatcher() *HeaderMatcher - // GetProvidersForModel returns the list of providers that can serve a given model. - GetProvidersForModel(model string) []schemas.ModelProvider // GetStreamChunkInterceptor returns the interceptor for streaming chunks. // Returns nil if no plugins are loaded or streaming interception is not needed. GetStreamChunkInterceptor() StreamChunkInterceptor @@ -805,6 +804,8 @@ func LoadConfig(ctx context.Context, configDirPath string) (*Config, error) { if err := initEncryption(&configData); err != nil { return nil, err } + // 1a. Vault config acknowledgement (initialization handled by enterprise layer) + initVault(&configData) // 2. Stores (config, logs, vector) — creates defaults for absent configs if err := initStores(ctx, config, &configData, configDBPath, logsDBPath); err != nil { return nil, err @@ -1159,14 +1160,7 @@ func loadProviders(ctx context.Context, config *Config, configData *ConfigData) for providerName, providerCfgInFile := range configData.Providers { provider := schemas.ModelProvider(strings.ToLower(providerName)) existingCfg, exists := providersInConfigStore[provider] - if err = processAuthoritativeProvider(providerName, providerCfgInFile, existingCfg, exists, authoritativeProviders); err != nil { - logger.Warn("failed to process provider %s: %v", providerName, err) - // Preserve the existing persisted config so a single bad file entry - // does not prune the provider (and its DB-only keys) from the store. - if exists { - authoritativeProviders[provider] = existingCfg - } - } + processAuthoritativeProvider(providerName, providerCfgInFile, existingCfg, exists, authoritativeProviders) } providersInConfigStore = authoritativeProviders } else { @@ -1242,12 +1236,21 @@ func processProvider( ) error { provider := schemas.ModelProvider(strings.ToLower(providerName)) + if err := ValidateCustomProvider(providerCfgInFile, provider); err != nil { + return err + } + + baseProvider := provider + if providerCfgInFile.CustomProviderConfig != nil && providerCfgInFile.CustomProviderConfig.BaseProviderType != "" { + baseProvider = providerCfgInFile.CustomProviderConfig.BaseProviderType + } + // Process environment variables in keys (including key-level configs) for i, providerKeyInFile := range providerCfgInFile.Keys { if providerKeyInFile.ID == "" { providerCfgInFile.Keys[i].ID = uuid.NewString() } - if err := providerKeyInFile.Aliases.Validate(); err != nil { + if err := providerKeyInFile.Aliases.Validate(baseProvider); err != nil { return fmt.Errorf("invalid aliases for key %q in provider %s: %w", providerKeyInFile.Name, provider, err) } } @@ -1269,14 +1272,21 @@ func processAuthoritativeProvider( existingCfg configstore.ProviderConfig, exists bool, providers map[schemas.ModelProvider]configstore.ProviderConfig, -) error { +) { provider := schemas.ModelProvider(strings.ToLower(providerName)) + if err := ValidateCustomProvider(providerCfgInFile, provider); err != nil { + logger.Warn("invalid custom provider config for %s (writing through): %v", provider, err) + } + baseProvider := provider + if providerCfgInFile.CustomProviderConfig != nil && providerCfgInFile.CustomProviderConfig.BaseProviderType != "" { + baseProvider = providerCfgInFile.CustomProviderConfig.BaseProviderType + } for i, providerKeyInFile := range providerCfgInFile.Keys { if providerKeyInFile.ID == "" { providerCfgInFile.Keys[i].ID = uuid.NewString() } - if err := providerKeyInFile.Aliases.Validate(); err != nil { - return fmt.Errorf("invalid aliases for key %q in provider %s: %w", providerKeyInFile.Name, provider, err) + if err := providerKeyInFile.Aliases.Validate(baseProvider); err != nil { + logger.Warn("invalid aliases for key %q in provider %s (writing through): %v", providerKeyInFile.Name, provider, err) } } fileProviderConfigHash, err := providerCfgInFile.GenerateConfigHash(string(provider)) @@ -1290,7 +1300,6 @@ func processAuthoritativeProvider( providerCfgInFile.Description = existingCfg.Description } providers[provider] = providerCfgInFile - return nil } // mergeProviderWithHash merges provider config using hash-based reconciliation @@ -2057,7 +2066,13 @@ func mergeGovernanceConfig(ctx context.Context, config *Config, configData *Conf found := false for j, existingCustomer := range governanceConfig.Customers { - if existingCustomer.ID == newCustomer.ID { + idMatch := existingCustomer.ID == newCustomer.ID + nameMatch := newCustomer.ID == "" && existingCustomer.Name == newCustomer.Name + if idMatch || nameMatch { + if nameMatch { + // Config file has no ID; adopt the DB record's ID so updates use the right primary key. + configData.Governance.Customers[i].ID = existingCustomer.ID + } found = true if existingCustomer.ConfigHash != fileCustomerHash { logger.Debug("config hash mismatch for customer %s, syncing from config file", newCustomer.ID) @@ -2072,6 +2087,9 @@ func mergeGovernanceConfig(ctx context.Context, config *Config, configData *Conf } if !found { configData.Governance.Customers[i].ConfigHash = fileCustomerHash + if configData.Governance.Customers[i].ID == "" { + configData.Governance.Customers[i].ID = uuid.NewString() + } customersToAdd = append(customersToAdd, configData.Governance.Customers[i]) } } @@ -2088,7 +2106,13 @@ func mergeGovernanceConfig(ctx context.Context, config *Config, configData *Conf found := false for j, existingTeam := range governanceConfig.Teams { - if existingTeam.ID == newTeam.ID { + idMatch := existingTeam.ID == newTeam.ID + nameMatch := newTeam.ID == "" && existingTeam.Name == newTeam.Name + if idMatch || nameMatch { + if nameMatch { + // Config file has no ID; adopt the DB record's ID so updates use the right primary key. + configData.Governance.Teams[i].ID = existingTeam.ID + } found = true if existingTeam.ConfigHash != fileTeamHash { logger.Debug("config hash mismatch for team %s, syncing from config file", newTeam.ID) @@ -2103,6 +2127,9 @@ func mergeGovernanceConfig(ctx context.Context, config *Config, configData *Conf } if !found { configData.Governance.Teams[i].ConfigHash = fileTeamHash + if configData.Governance.Teams[i].ID == "" { + configData.Governance.Teams[i].ID = uuid.NewString() + } teamsToAdd = append(teamsToAdd, configData.Governance.Teams[i]) } } @@ -2119,10 +2146,16 @@ func mergeGovernanceConfig(ctx context.Context, config *Config, configData *Conf // Preparing hash found := false for j, existingVirtualKey := range governanceConfig.VirtualKeys { - if existingVirtualKey.ID == newVirtualKey.ID { + idMatch := existingVirtualKey.ID == newVirtualKey.ID + nameMatch := newVirtualKey.ID == "" && existingVirtualKey.Name == newVirtualKey.Name + if idMatch || nameMatch { + if nameMatch { + // Config file has no ID; adopt the DB record's ID so updates use the right primary key. + configData.Governance.VirtualKeys[i].ID = existingVirtualKey.ID + } found = true if existingVirtualKey.ConfigHash != fileVKHash { - logger.Debug("config hash mismatch for virtual key %s, syncing from config file", newVirtualKey.ID) + logger.Debug("config hash mismatch for virtual key %s, syncing from config file", existingVirtualKey.ID) configData.Governance.VirtualKeys[i].ConfigHash = fileVKHash // This is added for backward compatibility with existing configs if configData.Governance.VirtualKeys[i].Value == "" && existingVirtualKey.Value != "" { @@ -2158,6 +2191,9 @@ func mergeGovernanceConfig(ctx context.Context, config *Config, configData *Conf } if !found { configData.Governance.VirtualKeys[i].ConfigHash = fileVKHash + if configData.Governance.VirtualKeys[i].ID == "" { + configData.Governance.VirtualKeys[i].ID = uuid.NewString() + } // if the virtual key value is env.VIRTUAL_KEY_VALUE, then we will need to resolve the environment variable // Process environment variable for virtual key value if strings.HasPrefix(configData.Governance.VirtualKeys[i].Value, "env.") { @@ -2347,6 +2383,23 @@ func mergeGovernanceConfig(ctx context.Context, config *Config, configData *Conf logger.Fatal("failed to sync governance config: %v", err) } } + + // File config stays authoritative for analyzer tuning when present. + if configData.Governance.ComplexityAnalyzerConfig != nil { + normalized, err := complexity.ValidateAndNormalize(configData.Governance.ComplexityAnalyzerConfig) + if err != nil { + logger.Error("invalid complexity analyzer config in config file: %v", err) + } else if normalized != nil { + current := config.GovernanceConfig.ComplexityAnalyzerConfig + config.GovernanceConfig.ComplexityAnalyzerConfig = normalized + if config.ConfigStore != nil && (current == nil || !reflect.DeepEqual(current, normalized)) { + if err := config.ConfigStore.UpdateComplexityAnalyzerConfig(ctx, normalized); err != nil { + logger.Warn("failed to sync complexity analyzer config from config file: %v", err) + } + } + } + } + // Sync pricing overrides into the model catalog in one batch to avoid // rebuilding the lookup map on every iteration. if config.ModelCatalog != nil { @@ -3300,6 +3353,10 @@ func createGovernanceConfigInStore(ctx context.Context, config *Config) { virtualKey.ProviderConfigs = nil virtualKey.MCPConfigs = nil + if virtualKey.ID == "" { + virtualKey.ID = uuid.NewString() + } + if err := config.ConfigStore.CreateVirtualKey(ctx, virtualKey, tx); err != nil { logger.Error("failed to create virtual key %s: %v", virtualKey.ID, err) return fmt.Errorf("failed to create virtual key %s: %w", virtualKey.ID, err) @@ -3391,6 +3448,18 @@ func createGovernanceConfigInStore(ctx context.Context, config *Config) { } } + if config.GovernanceConfig.ComplexityAnalyzerConfig != nil { + normalized, err := complexity.ValidateAndNormalize(config.GovernanceConfig.ComplexityAnalyzerConfig) + if err != nil { + logger.Warn("invalid complexity analyzer config in config file: %v", err) + } else if normalized != nil { + config.GovernanceConfig.ComplexityAnalyzerConfig = normalized + if err := config.ConfigStore.UpdateComplexityAnalyzerConfig(ctx, normalized, tx); err != nil { + return fmt.Errorf("failed to create complexity analyzer config: %w", err) + } + } + } + return nil }); err != nil { logger.Warn("failed to update governance config: %v", err) @@ -3474,8 +3543,7 @@ func loadAuthConfig(ctx context.Context, config *Config, configData *ConfigData) // If DB already matches file config, skip hashing and DB write if dbAuthConfig != nil { usernameMatch := dbAuthConfig.AdminUserName.GetValue() == authConfig.AdminUserName.GetValue() - boolsMatch := dbAuthConfig.IsEnabled == authConfig.IsEnabled && - dbAuthConfig.DisableAuthOnInference == authConfig.DisableAuthOnInference + boolsMatch := dbAuthConfig.IsEnabled == authConfig.IsEnabled var passwordMatch bool if filePassword == "" { passwordMatch = dbAuthConfig.AdminPassword.GetValue() == "" @@ -3487,10 +3555,9 @@ func loadAuthConfig(ctx context.Context, config *Config, configData *ConfigData) if usernameMatch && passwordMatch && boolsMatch { // DB matches file -- use DB hash but preserve file env var references config.GovernanceConfig.AuthConfig = &configstore.AuthConfig{ - AdminUserName: authConfig.AdminUserName, - AdminPassword: preserveEnvVar(authConfig.AdminPassword, dbAuthConfig.AdminPassword.GetValue()), - IsEnabled: authConfig.IsEnabled, - DisableAuthOnInference: authConfig.DisableAuthOnInference, + AdminUserName: authConfig.AdminUserName, + AdminPassword: preserveEnvVar(authConfig.AdminPassword, dbAuthConfig.AdminPassword.GetValue()), + IsEnabled: authConfig.IsEnabled, } return } @@ -3517,10 +3584,9 @@ func loadAuthConfig(ctx context.Context, config *Config, configData *ConfigData) } // Build auth config with hashed password but preserve env var references config.GovernanceConfig.AuthConfig = &configstore.AuthConfig{ - AdminUserName: authConfig.AdminUserName, - AdminPassword: preserveEnvVar(authConfig.AdminPassword, hashedPassword), - IsEnabled: authConfig.IsEnabled, - DisableAuthOnInference: authConfig.DisableAuthOnInference, + AdminUserName: authConfig.AdminUserName, + AdminPassword: preserveEnvVar(authConfig.AdminPassword, hashedPassword), + IsEnabled: authConfig.IsEnabled, } // Persist to config store if err := config.ConfigStore.UpdateAuthConfig(ctx, config.GovernanceConfig.AuthConfig); err != nil { @@ -3797,8 +3863,11 @@ func ResolveFrameworkPricingConfig( filePricingURL := (*string)(nil) fileModelParametersURL := (*string)(nil) fileSyncSeconds := (*int64)(nil) + fileMCPLibraryURL := (*string)(nil) + fileMCPLibrarySyncSeconds := (*int64)(nil) skipURLBackfill := false // prevent DB backfill of unresolved env references skipModelParamsURLBackfill := false + skipMCPLibraryURLBackfill := false if fileConfig != nil && fileConfig.Pricing != nil { if fileConfig.Pricing.PricingURL != nil { raw := *fileConfig.Pricing.PricingURL @@ -3848,6 +3917,39 @@ func ResolveFrameworkPricingConfig( fileSyncSeconds = &val } } + if fileConfig.Pricing.MCPLibraryURL != nil { + raw := strings.TrimSpace(*fileConfig.Pricing.MCPLibraryURL) + if raw == "" { + // Blank is treated as "not set"; fall back to default. + } else if strings.HasPrefix(raw, "env.") { + resolvedURL, err := envutils.ProcessEnvValue(raw) + if err != nil { + logger.Warn("mcp_library_url: env variable not found (%v); keeping original value %q", err, raw) + fileMCPLibraryURL = &raw + skipMCPLibraryURLBackfill = true + } else { + resolved := strings.TrimSpace(resolvedURL) + if resolved != "" { + fileMCPLibraryURL = &resolved + } + } + } else { + fileMCPLibraryURL = &raw + } + } + if fileConfig.Pricing.MCPLibrarySyncInterval != nil { + val := *fileConfig.Pricing.MCPLibrarySyncInterval + switch { + case val <= 0: + logger.Warn("mcp_library_sync_interval in config.json is invalid (%d seconds), ignoring — using default (%d seconds)", val, defaultSyncSeconds) + case val < modelcatalog.MinimumPricingSyncIntervalSec: + clamped := modelcatalog.MinimumPricingSyncIntervalSec + logger.Warn("mcp_library_sync_interval in config.json is below minimum (%d seconds), clamping to %d seconds", val, clamped) + fileMCPLibrarySyncSeconds = &clamped + default: + fileMCPLibrarySyncSeconds = &val + } + } } // --- Phase 2: apply file config over defaults --- @@ -3856,6 +3958,11 @@ func ResolveFrameworkPricingConfig( resolvedModelParametersURL := &defaultModelParametersURL resolvedSyncSeconds := &defaultSyncSeconds + defaultMCPLibraryURL := modelcatalog.DefaultMCPLibraryURL + defaultMCPLibrarySyncSeconds := int64(modelcatalog.DefaultSyncInterval.Seconds()) + resolvedMCPLibraryURL := &defaultMCPLibraryURL + resolvedMCPLibrarySyncInterval := &defaultMCPLibrarySyncSeconds + if filePricingURL != nil { resolvedPricingURL = filePricingURL logger.Debug("pricing_url resolved from file") @@ -3869,6 +3976,16 @@ func ResolveFrameworkPricingConfig( logger.Debug("pricing_sync_interval resolved from file: %d seconds", *fileSyncSeconds) } + // MCP library catalog sync source mirrors the datasheet URL handling for + // defaults, env substitution, interval validation, and hash-gated config.json + // changes. DB precedence is applied in Phase 3 below. + if fileMCPLibraryURL != nil { + resolvedMCPLibraryURL = fileMCPLibraryURL + } + if fileMCPLibrarySyncSeconds != nil { + resolvedMCPLibrarySyncInterval = fileMCPLibrarySyncSeconds + } + // --- Phase 3: DB values applied; file wins on hash mismatch (file changed since last write) --- needsDBUpdate := false @@ -3876,8 +3993,22 @@ func ResolveFrameworkPricingConfig( // Hash the file-resolved values; skip if nothing valid survived Phase 1. fileHash := "" - if fileConfig != nil && fileConfig.Pricing != nil && !skipURLBackfill && (filePricingURL != nil || fileSyncSeconds != nil) { - h, err := configstore.GenerateFrameworkConfigHash(filePricingURL, fileModelParametersURL, fileSyncSeconds) + fileHasHashableMCPConfig := (fileMCPLibraryURL != nil && !skipMCPLibraryURLBackfill) || fileMCPLibrarySyncSeconds != nil + if fileConfig != nil && fileConfig.Pricing != nil && !skipURLBackfill && (filePricingURL != nil || fileSyncSeconds != nil || fileHasHashableMCPConfig) { + var h string + var err error + if fileHasHashableMCPConfig { + mcpHashURL := fileMCPLibraryURL + if skipMCPLibraryURLBackfill { + mcpHashURL = nil + } + h, err = configstore.GenerateFrameworkConfigHash(filePricingURL, fileModelParametersURL, fileSyncSeconds, configstore.FrameworkConfigHashOptions{ + MCPLibraryURL: mcpHashURL, + MCPLibrarySyncInterval: fileMCPLibrarySyncSeconds, + }) + } else { + h, err = configstore.GenerateFrameworkConfigHash(filePricingURL, fileModelParametersURL, fileSyncSeconds) + } if err != nil { logger.Warn("failed to compute framework config hash: %v", err) } else { @@ -3931,6 +4062,48 @@ func ResolveFrameworkPricingConfig( } else { needsDBUpdate = true } + + // MCP library config follows the same hash-gated config.json precedence as + // datasheet config: DB wins while the file is unchanged; file wins and is + // backfilled when the file changed since the last persisted hash. + if dbConfig.MCPLibraryURL != nil { + if trimmed := strings.TrimSpace(*dbConfig.MCPLibraryURL); trimmed != "" { + if fileChanged && fileMCPLibraryURL != nil && !skipMCPLibraryURLBackfill { + logger.Info("mcp_library_url from config.json overrides DB (file hash changed) — updating DB") + needsDBUpdate = true + } else { + resolvedMCPLibraryURL = &trimmed + } + } else if !skipMCPLibraryURLBackfill { + needsDBUpdate = true + } + } else if !skipMCPLibraryURLBackfill { + needsDBUpdate = true + } + if dbConfig.MCPLibrarySyncInterval != nil { + val := *dbConfig.MCPLibrarySyncInterval + switch { + case val <= 0: + logger.Warn("mcp_library_sync_interval in DB is corrupted (%d seconds), ignoring — backfilling with %d seconds", val, *resolvedMCPLibrarySyncInterval) + needsDBUpdate = true + case val < modelcatalog.MinimumPricingSyncIntervalSec: + logger.Warn("mcp_library_sync_interval in DB is below minimum (%d seconds) — backfilling", val) + if !fileChanged || fileMCPLibrarySyncSeconds == nil { + clamped := modelcatalog.MinimumPricingSyncIntervalSec + resolvedMCPLibrarySyncInterval = &clamped + } + needsDBUpdate = true + default: + if fileChanged && fileMCPLibrarySyncSeconds != nil { + logger.Info("mcp_library_sync_interval from config.json overrides DB (file hash changed): file=%d db=%d seconds — updating DB", *fileMCPLibrarySyncSeconds, val) + needsDBUpdate = true + } else { + resolvedMCPLibrarySyncInterval = dbConfig.MCPLibrarySyncInterval + } + } + } else { + needsDBUpdate = true + } } // --- Phase 4: nil guard --- @@ -3946,6 +4119,12 @@ func ResolveFrameworkPricingConfig( logger.Warn("invariant violation: pricing_sync_interval resolved to nil — falling back to default %d seconds", defaultSyncSeconds) resolvedSyncSeconds = &defaultSyncSeconds } + if resolvedMCPLibraryURL == nil { + resolvedMCPLibraryURL = &defaultMCPLibraryURL + } + if resolvedMCPLibrarySyncInterval == nil { + resolvedMCPLibrarySyncInterval = &defaultMCPLibrarySyncSeconds + } // Only update the stored hash when the file actually changed; preserve the // existing hash for correction-only DB updates (null backfill, corruption fix). @@ -3958,15 +4137,19 @@ func ResolveFrameworkPricingConfig( } return &configstoreTables.TableFrameworkConfig{ - ID: configID, - PricingURL: resolvedPricingURL, - PricingSyncInterval: resolvedSyncSeconds, - ModelParametersURL: resolvedModelParametersURL, - ConfigHash: persistedHash, + ID: configID, + PricingURL: resolvedPricingURL, + PricingSyncInterval: resolvedSyncSeconds, + ModelParametersURL: resolvedModelParametersURL, + MCPLibraryURL: resolvedMCPLibraryURL, + MCPLibrarySyncInterval: resolvedMCPLibrarySyncInterval, + ConfigHash: persistedHash, }, &modelcatalog.Config{ - PricingURL: resolvedPricingURL, - PricingSyncInterval: resolvedSyncSeconds, - ModelParametersURL: resolvedModelParametersURL, + PricingURL: resolvedPricingURL, + PricingSyncInterval: resolvedSyncSeconds, + ModelParametersURL: resolvedModelParametersURL, + MCPLibraryURL: resolvedMCPLibraryURL, + MCPLibrarySyncInterval: resolvedMCPLibrarySyncInterval, }, needsDBUpdate } @@ -4075,6 +4258,10 @@ func initEncryption(configData *ConfigData) error { return nil } +// initVault is a no-op stub at the OSS level. +// Vault initialization is performed by the enterprise layer via config_store.vault_store. +func initVault(_ *ConfigData) {} + // syncEncryption encrypts all plaintext rows in the config store if encryption is enabled. // Called during bootup after encryption key is initialized and all config data has been loaded. func syncEncryption(ctx context.Context, config *Config) { @@ -4373,28 +4560,6 @@ func (c *Config) GetAllowOnAllVirtualKeysClients() map[string]string { return result } -// GetProvidersForModel returns the list of providers for a given model, sorted -// deterministically so callers picking providers[0] always get the same result. -func (c *Config) GetProvidersForModel(model string) []schemas.ModelProvider { - if c.ModelCatalog == nil { - return []schemas.ModelProvider{} - } - providersInCatalog := c.ModelCatalog.GetProvidersForModel(model) - // Filter out the providers which are not present in the configured provider list for the client - c.Mu.RLock() - defer c.Mu.RUnlock() - allowedProviders := make([]schemas.ModelProvider, 0, len(providersInCatalog)) - for configuredProvider := range c.Providers { - if slices.Contains(providersInCatalog, configuredProvider) { - allowedProviders = append(allowedProviders, configuredProvider) - } - } - slices.SortFunc(allowedProviders, func(a, b schemas.ModelProvider) int { - return strings.Compare(string(a), string(b)) - }) - return allowedProviders -} - // GetPluginOrder returns the names of all base plugins in their sorted placement order. // This method is lock-free and safe for concurrent access from hot paths. // Do not modify the returned slice; it is a shared snapshot and must be treated read-only. @@ -4417,6 +4582,30 @@ func (c *Config) GetLoadedLLMPlugins() []schemas.LLMPlugin { return nil } +// GetLoadedPluginNames returns the sanitized names of every currently loaded plugin, +// matching the names embedded in their trace span names. +func (c *Config) GetLoadedPluginNames() []string { + plugins := c.BasePlugins.Load() + if plugins == nil { + return nil + } + seen := make(map[string]struct{}, len(*plugins)) + names := make([]string, 0, len(*plugins)) + for _, p := range *plugins { + name := schemas.SanitizePluginSpanName(p.GetName()) + if name == "" { + continue + } + if _, ok := seen[name]; ok { + continue + } + seen[name] = struct{}{} + names = append(names, name) + } + slices.Sort(names) + return names +} + // pluginChunkInterceptor implements StreamChunkInterceptor by calling plugin hooks type pluginChunkInterceptor struct { plugins []schemas.HTTPTransportPlugin diff --git a/transports/bifrost-http/lib/config_test.go b/transports/bifrost-http/lib/config_test.go index a1b612d270..77119ec479 100644 --- a/transports/bifrost-http/lib/config_test.go +++ b/transports/bifrost-http/lib/config_test.go @@ -635,6 +635,35 @@ func (m *MockConfigStore) GetMCPClientsPaginated(ctx context.Context, params con return nil, 0, nil } +func (m *MockConfigStore) GetMCPLibraryPaginated(ctx context.Context, params configstore.MCPLibraryQueryParams) ([]tables.TableMCPLibrary, int64, error) { + return nil, 0, nil +} + +func (m *MockConfigStore) GetMCPLibraryFilterData(ctx context.Context) (*configstore.MCPLibraryFilterData, error) { + return &configstore.MCPLibraryFilterData{ + Categories: []string{}, + ConnectionTypes: []string{}, + AuthTypes: []string{}, + Tags: []string{}, + }, nil +} + +func (m *MockConfigStore) UpsertMCPLibraryEntry(ctx context.Context, entry *tables.TableMCPLibrary, tx ...*gorm.DB) error { + return nil +} + +func (m *MockConfigStore) CreateCustomMCPLibraryEntry(ctx context.Context, entry *tables.TableMCPLibrary) error { + return nil +} + +func (m *MockConfigStore) SoftDeleteMCPLibraryEntry(ctx context.Context, id uint) error { + return nil +} + +func (m *MockConfigStore) GetProtectedMCPLibrarySlugs(ctx context.Context) ([]string, error) { + return []string{}, nil +} + func (m *MockConfigStore) DeleteMCPClientConfig(ctx context.Context, id string) error { if m.mcpConfig == nil { return nil @@ -943,6 +972,21 @@ func (m *MockConfigStore) UpdateConfig(ctx context.Context, config *tables.Table return nil } +func (m *MockConfigStore) GetComplexityAnalyzerConfig(ctx context.Context) (*configstore.ComplexityAnalyzerConfig, error) { + if m.governanceConfig == nil { + return nil, nil + } + return m.governanceConfig.ComplexityAnalyzerConfig, nil +} + +func (m *MockConfigStore) UpdateComplexityAnalyzerConfig(ctx context.Context, config *configstore.ComplexityAnalyzerConfig, tx ...*gorm.DB) error { + if m.governanceConfig == nil { + m.governanceConfig = &configstore.GovernanceConfig{} + } + m.governanceConfig.ComplexityAnalyzerConfig = config + return nil +} + // Plugins func (m *MockConfigStore) GetPlugins(ctx context.Context) ([]*tables.TablePlugin, error) { return m.plugins, nil @@ -1406,6 +1450,44 @@ func (m *MockConfigStore) DeleteRoutingRule(ctx context.Context, id string, tx . return nil } +func TestMergeGovernanceConfig_SyncsComplexityAnalyzerConfig(t *testing.T) { + initTestLogger() + + store := NewMockConfigStore() + dbGovernance := &configstore.GovernanceConfig{} + config := &Config{ + ConfigStore: store, + GovernanceConfig: dbGovernance, + } + fileConfig := &configstore.ComplexityAnalyzerConfig{ + TierBoundaries: configstore.ComplexityTierBoundaries{ + SimpleMedium: 0.11, + MediumComplex: 0.33, + ComplexReasoning: 0.77, + }, + Keywords: configstore.ComplexityEditableKeywordConfig{ + CodeKeywords: []string{" Function ", "api", "API"}, + ReasoningKeywords: []string{"tradeoffs"}, + TechnicalKeywords: []string{"latency"}, + SimpleKeywords: []string{"hello"}, + }, + } + configData := &ConfigData{ + Governance: &configstore.GovernanceConfig{ + ComplexityAnalyzerConfig: fileConfig, + }, + } + + mergeGovernanceConfig(context.Background(), config, configData, dbGovernance) + + stored, err := store.GetComplexityAnalyzerConfig(context.Background()) + require.NoError(t, err) + require.NotNil(t, stored) + require.Equal(t, 0.77, stored.TierBoundaries.ComplexReasoning) + require.Equal(t, []string{"api", "function"}, stored.Keywords.CodeKeywords) + require.Equal(t, stored, config.GovernanceConfig.ComplexityAnalyzerConfig) +} + // Prompt Repository - Folders func (m *MockConfigStore) GetFolders(ctx context.Context) ([]tables.TableFolder, error) { return nil, nil @@ -2695,7 +2777,7 @@ func TestGenerateKeyHash(t *testing.T) { Value: *schemas.NewEnvVar("sk-123"), Models: []string{"gpt-4", "gpt-3.5-turbo"}, Weight: 1.5, - Aliases: schemas.KeyAliases{"gpt-4": "gpt-4-deployment"}, + Aliases: schemas.KeyAliases{"gpt-4": {ModelID: "gpt-4-deployment"}}, } hashWithAliases, err := configstore.GenerateKeyHash(keyWithAliases) @@ -5139,7 +5221,7 @@ func TestKeyHashComparison_AzureConfigSyncScenarios(t *testing.T) { Name: "azure-key", Value: *schemas.NewEnvVar("azure-api-key-123"), Weight: 1, - Aliases: schemas.KeyAliases{"gpt-4": "gpt-4-deployment"}, + Aliases: schemas.KeyAliases{"gpt-4": {ModelID: "gpt-4-deployment"}}, AzureKeyConfig: &schemas.AzureKeyConfig{ Endpoint: *schemas.NewEnvVar("https://myazure.openai.azure.com"), }, @@ -5150,7 +5232,7 @@ func TestKeyHashComparison_AzureConfigSyncScenarios(t *testing.T) { Name: "azure-key", Value: *schemas.NewEnvVar("azure-api-key-123"), Weight: 1, - Aliases: schemas.KeyAliases{"gpt-4": "gpt-4-deployment"}, + Aliases: schemas.KeyAliases{"gpt-4": {ModelID: "gpt-4-deployment"}}, AzureKeyConfig: &schemas.AzureKeyConfig{ Endpoint: *schemas.NewEnvVar("https://myazure.openai.azure.com"), }, @@ -5172,7 +5254,7 @@ func TestKeyHashComparison_AzureConfigSyncScenarios(t *testing.T) { Name: "azure-key", Value: *schemas.NewEnvVar("azure-api-key-123"), Weight: 1, - Aliases: schemas.KeyAliases{"gpt-4": "gpt-4-deployment"}, + Aliases: schemas.KeyAliases{"gpt-4": {ModelID: "gpt-4-deployment"}}, AzureKeyConfig: &schemas.AzureKeyConfig{ Endpoint: *schemas.NewEnvVar("https://myazure.openai.azure.com"), }, @@ -5183,7 +5265,7 @@ func TestKeyHashComparison_AzureConfigSyncScenarios(t *testing.T) { Name: "azure-key", Value: *schemas.NewEnvVar("azure-api-key-123"), Weight: 1, - Aliases: schemas.KeyAliases{"gpt-4": "gpt-4-deployment"}, + Aliases: schemas.KeyAliases{"gpt-4": {ModelID: "gpt-4-deployment"}}, AzureKeyConfig: &schemas.AzureKeyConfig{ Endpoint: *schemas.NewEnvVar("https://different-azure.openai.azure.com"), // Changed! }, @@ -5205,7 +5287,7 @@ func TestKeyHashComparison_AzureConfigSyncScenarios(t *testing.T) { Name: "azure-key", Value: *schemas.NewEnvVar("azure-api-key-123"), Weight: 1, - Aliases: schemas.KeyAliases{"gpt-4": "gpt-4-deployment"}, + Aliases: schemas.KeyAliases{"gpt-4": {ModelID: "gpt-4-deployment"}}, AzureKeyConfig: &schemas.AzureKeyConfig{ Endpoint: *schemas.NewEnvVar("https://myazure.openai.azure.com"), }, @@ -5216,7 +5298,7 @@ func TestKeyHashComparison_AzureConfigSyncScenarios(t *testing.T) { Name: "azure-key", Value: *schemas.NewEnvVar("azure-api-key-123"), Weight: 1, - Aliases: schemas.KeyAliases{"gpt-4": "gpt-4-deployment", "gpt-3.5-turbo": "gpt-35-turbo-deployment"}, + Aliases: schemas.KeyAliases{"gpt-4": {ModelID: "gpt-4-deployment"}, "gpt-3.5-turbo": {ModelID: "gpt-35-turbo-deployment"}}, AzureKeyConfig: &schemas.AzureKeyConfig{ Endpoint: *schemas.NewEnvVar("https://myazure.openai.azure.com"), }, @@ -5304,7 +5386,7 @@ func TestKeyHashComparison_BedrockConfigSyncScenarios(t *testing.T) { Name: "bedrock-key", Value: *schemas.NewEnvVar("bedrock-api-key-123"), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3": "claude-3-inference-profile"}, + Aliases: schemas.KeyAliases{"claude-3": {ModelID: "claude-3-inference-profile"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAIOSFODNN7EXAMPLE"), SecretKey: *schemas.NewEnvVar("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"), @@ -5317,7 +5399,7 @@ func TestKeyHashComparison_BedrockConfigSyncScenarios(t *testing.T) { Name: "bedrock-key", Value: *schemas.NewEnvVar("bedrock-api-key-123"), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3": "claude-3-inference-profile"}, + Aliases: schemas.KeyAliases{"claude-3": {ModelID: "claude-3-inference-profile"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAIOSFODNN7EXAMPLE"), SecretKey: *schemas.NewEnvVar("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"), @@ -5341,7 +5423,7 @@ func TestKeyHashComparison_BedrockConfigSyncScenarios(t *testing.T) { Name: "bedrock-key", Value: *schemas.NewEnvVar("bedrock-api-key-123"), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3": "claude-3-inference-profile"}, + Aliases: schemas.KeyAliases{"claude-3": {ModelID: "claude-3-inference-profile"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAIOSFODNN7EXAMPLE"), SecretKey: *schemas.NewEnvVar("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"), @@ -5354,7 +5436,7 @@ func TestKeyHashComparison_BedrockConfigSyncScenarios(t *testing.T) { Name: "bedrock-key", Value: *schemas.NewEnvVar("bedrock-api-key-123"), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3": "claude-3-inference-profile"}, + Aliases: schemas.KeyAliases{"claude-3": {ModelID: "claude-3-inference-profile"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAI44QH8DHBEXAMPLE"), // Changed! SecretKey: *schemas.NewEnvVar("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"), @@ -5378,7 +5460,7 @@ func TestKeyHashComparison_BedrockConfigSyncScenarios(t *testing.T) { Name: "bedrock-key", Value: *schemas.NewEnvVar("bedrock-api-key-123"), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3": "claude-3-inference-profile"}, + Aliases: schemas.KeyAliases{"claude-3": {ModelID: "claude-3-inference-profile"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAIOSFODNN7EXAMPLE"), SecretKey: *schemas.NewEnvVar("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"), @@ -5391,7 +5473,7 @@ func TestKeyHashComparison_BedrockConfigSyncScenarios(t *testing.T) { Name: "bedrock-key", Value: *schemas.NewEnvVar("bedrock-api-key-123"), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3": "claude-3-inference-profile"}, + Aliases: schemas.KeyAliases{"claude-3": {ModelID: "claude-3-inference-profile"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAIOSFODNN7EXAMPLE"), SecretKey: *schemas.NewEnvVar("differentSecretKey/NEWKEY/bPxRfiCYEXAMPLEKEY"), // Changed! @@ -5415,7 +5497,7 @@ func TestKeyHashComparison_BedrockConfigSyncScenarios(t *testing.T) { Name: "bedrock-key", Value: *schemas.NewEnvVar("bedrock-api-key-123"), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3": "claude-3-inference-profile"}, + Aliases: schemas.KeyAliases{"claude-3": {ModelID: "claude-3-inference-profile"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAIOSFODNN7EXAMPLE"), SecretKey: *schemas.NewEnvVar("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"), @@ -5428,7 +5510,7 @@ func TestKeyHashComparison_BedrockConfigSyncScenarios(t *testing.T) { Name: "bedrock-key", Value: *schemas.NewEnvVar("bedrock-api-key-123"), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3": "claude-3-inference-profile"}, + Aliases: schemas.KeyAliases{"claude-3": {ModelID: "claude-3-inference-profile"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAIOSFODNN7EXAMPLE"), SecretKey: *schemas.NewEnvVar("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"), @@ -5452,7 +5534,7 @@ func TestKeyHashComparison_BedrockConfigSyncScenarios(t *testing.T) { Name: "bedrock-key", Value: *schemas.NewEnvVar("bedrock-api-key-123"), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3": "claude-3-inference-profile"}, + Aliases: schemas.KeyAliases{"claude-3": {ModelID: "claude-3-inference-profile"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAIOSFODNN7EXAMPLE"), SecretKey: *schemas.NewEnvVar("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"), @@ -5466,7 +5548,7 @@ func TestKeyHashComparison_BedrockConfigSyncScenarios(t *testing.T) { Name: "bedrock-key", Value: *schemas.NewEnvVar("bedrock-api-key-123"), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3": "claude-3-inference-profile"}, + Aliases: schemas.KeyAliases{"claude-3": {ModelID: "claude-3-inference-profile"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAIOSFODNN7EXAMPLE"), SecretKey: *schemas.NewEnvVar("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"), @@ -5491,7 +5573,7 @@ func TestKeyHashComparison_BedrockConfigSyncScenarios(t *testing.T) { Name: "bedrock-key", Value: *schemas.NewEnvVar("bedrock-api-key-123"), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3": "claude-3-inference-profile"}, + Aliases: schemas.KeyAliases{"claude-3": {ModelID: "claude-3-inference-profile"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAIOSFODNN7EXAMPLE"), SecretKey: *schemas.NewEnvVar("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"), @@ -5504,7 +5586,7 @@ func TestKeyHashComparison_BedrockConfigSyncScenarios(t *testing.T) { Name: "bedrock-key", Value: *schemas.NewEnvVar("bedrock-api-key-123"), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3": "claude-3-inference-profile", "claude-3.5": "claude-35-inference-profile"}, + Aliases: schemas.KeyAliases{"claude-3": {ModelID: "claude-3-inference-profile"}, "claude-3.5": {ModelID: "claude-35-inference-profile"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAIOSFODNN7EXAMPLE"), SecretKey: *schemas.NewEnvVar("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"), @@ -5594,7 +5676,7 @@ func TestKeyHashComparison_BedrockConfigSyncScenarios(t *testing.T) { Name: "bedrock-key", Value: *schemas.NewEnvVar("bedrock-api-key-123"), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3": "claude-3-inference-profile"}, + Aliases: schemas.KeyAliases{"claude-3": {ModelID: "claude-3-inference-profile"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAIOSFODNN7EXAMPLE"), SecretKey: *schemas.NewEnvVar("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"), @@ -5608,7 +5690,7 @@ func TestKeyHashComparison_BedrockConfigSyncScenarios(t *testing.T) { Name: "bedrock-key", Value: *schemas.NewEnvVar("bedrock-api-key-123"), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3": "claude-3-inference-profile"}, + Aliases: schemas.KeyAliases{"claude-3": {ModelID: "claude-3-inference-profile"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAIOSFODNN7EXAMPLE"), SecretKey: *schemas.NewEnvVar("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"), @@ -5634,7 +5716,7 @@ func TestKeyHashComparison_BedrockConfigSyncScenarios(t *testing.T) { Name: "bedrock-key", Value: *schemas.NewEnvVar(""), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3": "claude-3-inference-profile"}, + Aliases: schemas.KeyAliases{"claude-3": {ModelID: "claude-3-inference-profile"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar(""), // Empty for IAM role auth SecretKey: *schemas.NewEnvVar(""), // Empty for IAM role auth @@ -5648,7 +5730,7 @@ func TestKeyHashComparison_BedrockConfigSyncScenarios(t *testing.T) { Name: "bedrock-key", Value: *schemas.NewEnvVar(""), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3": "claude-3-inference-profile"}, + Aliases: schemas.KeyAliases{"claude-3": {ModelID: "claude-3-inference-profile"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAIOSFODNN7EXAMPLE"), SecretKey: *schemas.NewEnvVar("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"), @@ -5674,7 +5756,7 @@ func TestProviderHashComparison_AzureProviderFullLifecycle(t *testing.T) { Name: "azure-openai-key", Value: *schemas.NewEnvVar("azure-api-key-initial"), Weight: 1, - Aliases: schemas.KeyAliases{"gpt-4": "gpt-4-deployment"}, + Aliases: schemas.KeyAliases{"gpt-4": {ModelID: "gpt-4-deployment"}}, AzureKeyConfig: &schemas.AzureKeyConfig{ Endpoint: *schemas.NewEnvVar("https://myazure.openai.azure.com"), }, @@ -5708,7 +5790,7 @@ func TestProviderHashComparison_AzureProviderFullLifecycle(t *testing.T) { Name: "azure-openai-key", Value: *schemas.NewEnvVar("azure-api-key-dashboard-edited"), // Changed via dashboard! Weight: 1, - Aliases: schemas.KeyAliases{"gpt-4": "gpt-4-deployment"}, + Aliases: schemas.KeyAliases{"gpt-4": {ModelID: "gpt-4-deployment"}}, AzureKeyConfig: &schemas.AzureKeyConfig{ Endpoint: *schemas.NewEnvVar("https://myazure.openai.azure.com"), }, @@ -5740,7 +5822,7 @@ func TestProviderHashComparison_AzureProviderFullLifecycle(t *testing.T) { Name: "azure-openai-key", Value: *schemas.NewEnvVar("azure-api-key-initial"), // Original value from file Weight: 1, - Aliases: schemas.KeyAliases{"gpt-4": "gpt-4-deployment"}, + Aliases: schemas.KeyAliases{"gpt-4": {ModelID: "gpt-4-deployment"}}, AzureKeyConfig: &schemas.AzureKeyConfig{ Endpoint: *schemas.NewEnvVar("https://myazure.openai.azure.com"), }, @@ -5776,7 +5858,7 @@ func TestProviderHashComparison_AzureProviderFullLifecycle(t *testing.T) { Name: "azure-openai-key", Value: *schemas.NewEnvVar("azure-api-key-initial"), Weight: 1, - Aliases: schemas.KeyAliases{"gpt-4": "gpt-4-deployment", "gpt-4o": "gpt-4o-deployment"}, + Aliases: schemas.KeyAliases{"gpt-4": {ModelID: "gpt-4-deployment"}, "gpt-4o": {ModelID: "gpt-4o-deployment"}}, AzureKeyConfig: &schemas.AzureKeyConfig{ Endpoint: *schemas.NewEnvVar("https://new-azure.openai.azure.com"), // Changed! }, @@ -5882,7 +5964,7 @@ func TestProviderHashComparison_BedrockProviderFullLifecycle(t *testing.T) { Name: "aws-bedrock-key", Value: *schemas.NewEnvVar(""), // Empty for Bedrock with IAM or AccessKey auth Weight: 1, - Aliases: schemas.KeyAliases{"claude-3-sonnet": "anthropic.claude-3-sonnet-20240229-v1:0"}, + Aliases: schemas.KeyAliases{"claude-3-sonnet": {ModelID: "anthropic.claude-3-sonnet-20240229-v1:0"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAIOSFODNN7EXAMPLE"), SecretKey: *schemas.NewEnvVar("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"), @@ -5917,7 +5999,7 @@ func TestProviderHashComparison_BedrockProviderFullLifecycle(t *testing.T) { Name: "aws-bedrock-key-eu", Value: *schemas.NewEnvVar(""), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3-sonnet": "anthropic.claude-3-sonnet-20240229-v1:0"}, + Aliases: schemas.KeyAliases{"claude-3-sonnet": {ModelID: "anthropic.claude-3-sonnet-20240229-v1:0"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAI44QH8DHBEXAMPLE"), SecretKey: *schemas.NewEnvVar("je7MtGbClwBF/2Zp9Utk/h3yCo8nvbEXAMPLEKEY"), @@ -5944,7 +6026,7 @@ func TestProviderHashComparison_BedrockProviderFullLifecycle(t *testing.T) { Name: "aws-bedrock-key", Value: *schemas.NewEnvVar(""), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3-sonnet": "anthropic.claude-3-sonnet-20240229-v1:0"}, + Aliases: schemas.KeyAliases{"claude-3-sonnet": {ModelID: "anthropic.claude-3-sonnet-20240229-v1:0"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAIOSFODNN7EXAMPLE"), SecretKey: *schemas.NewEnvVar("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"), @@ -5983,7 +6065,7 @@ func TestProviderHashComparison_BedrockProviderFullLifecycle(t *testing.T) { Name: "aws-bedrock-key", Value: *schemas.NewEnvVar(""), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3-sonnet": "anthropic.claude-3-sonnet-20240229-v1:0", "claude-3-opus": "anthropic.claude-3-opus-20240229-v1:0"}, + Aliases: schemas.KeyAliases{"claude-3-sonnet": {ModelID: "anthropic.claude-3-sonnet-20240229-v1:0"}, "claude-3-opus": {ModelID: "anthropic.claude-3-opus-20240229-v1:0"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAIOSFODNN7EXAMPLE"), SecretKey: *schemas.NewEnvVar("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"), @@ -6111,7 +6193,7 @@ func TestProviderHashComparison_BedrockProviderFullLifecycle(t *testing.T) { Name: "aws-bedrock-key", Value: *schemas.NewEnvVar(""), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3-sonnet": "anthropic.claude-3-sonnet-20240229-v1:0", "claude-3-opus": "anthropic.claude-3-opus-20240229-v1:0"}, + Aliases: schemas.KeyAliases{"claude-3-sonnet": {ModelID: "anthropic.claude-3-sonnet-20240229-v1:0"}, "claude-3-opus": {ModelID: "anthropic.claude-3-opus-20240229-v1:0"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAIOSFODNN7EXAMPLE"), SecretKey: *schemas.NewEnvVar("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"), @@ -6154,7 +6236,7 @@ func TestProviderHashComparison_AzureNewProviderFromConfig(t *testing.T) { Name: "azure-openai-key", Value: *schemas.NewEnvVar("azure-api-key-123"), Weight: 1, - Aliases: schemas.KeyAliases{"gpt-4": "gpt-4-deployment"}, + Aliases: schemas.KeyAliases{"gpt-4": {ModelID: "gpt-4-deployment"}}, AzureKeyConfig: &schemas.AzureKeyConfig{ Endpoint: *schemas.NewEnvVar("https://myazure.openai.azure.com"), }, @@ -6220,7 +6302,7 @@ func TestProviderHashComparison_BedrockNewProviderFromConfig(t *testing.T) { Name: "aws-bedrock-key", Value: *schemas.NewEnvVar(""), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3": "anthropic.claude-3-sonnet-20240229-v1:0"}, + Aliases: schemas.KeyAliases{"claude-3": {ModelID: "anthropic.claude-3-sonnet-20240229-v1:0"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAIOSFODNN7EXAMPLE"), SecretKey: *schemas.NewEnvVar("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"), @@ -6289,7 +6371,7 @@ func TestProviderHashComparison_AzureDBValuePreservedWhenHashMatches(t *testing. Name: "azure-openai-key", Value: *schemas.NewEnvVar("DASHBOARD-EDITED-SECRET-KEY"), // Dashboard edited this! Weight: 1, - Aliases: schemas.KeyAliases{"gpt-4": "gpt-4-deployment"}, + Aliases: schemas.KeyAliases{"gpt-4": {ModelID: "gpt-4-deployment"}}, AzureKeyConfig: &schemas.AzureKeyConfig{ Endpoint: *schemas.NewEnvVar("https://myazure.openai.azure.com"), }, @@ -6317,7 +6399,7 @@ func TestProviderHashComparison_AzureDBValuePreservedWhenHashMatches(t *testing. Name: "azure-openai-key", Value: *schemas.NewEnvVar("original-key-from-file"), // Different value than DB! Weight: 1, - Aliases: schemas.KeyAliases{"gpt-4": "gpt-4-deployment"}, + Aliases: schemas.KeyAliases{"gpt-4": {ModelID: "gpt-4-deployment"}}, AzureKeyConfig: &schemas.AzureKeyConfig{ Endpoint: *schemas.NewEnvVar("https://myazure.openai.azure.com"), // Same }, @@ -6373,7 +6455,7 @@ func TestProviderHashComparison_BedrockDBValuePreservedWhenHashMatches(t *testin Name: "aws-bedrock-key", Value: *schemas.NewEnvVar(""), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3": "anthropic.claude-3-sonnet-20240229-v1:0"}, + Aliases: schemas.KeyAliases{"claude-3": {ModelID: "anthropic.claude-3-sonnet-20240229-v1:0"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("DASHBOARD-EDITED-ACCESS-KEY"), // Dashboard edited! SecretKey: *schemas.NewEnvVar("DASHBOARD-EDITED-SECRET-KEY"), // Dashboard edited! @@ -6403,7 +6485,7 @@ func TestProviderHashComparison_BedrockDBValuePreservedWhenHashMatches(t *testin Name: "aws-bedrock-key", Value: *schemas.NewEnvVar(""), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3": "anthropic.claude-3-sonnet-20240229-v1:0"}, + Aliases: schemas.KeyAliases{"claude-3": {ModelID: "anthropic.claude-3-sonnet-20240229-v1:0"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAIOSFODNN7EXAMPLE"), // Different! SecretKey: *schemas.NewEnvVar("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"), // Different! @@ -6491,7 +6573,7 @@ func TestProviderHashComparison_AzureConfigChangedInFile(t *testing.T) { Name: "azure-openai-key", Value: *schemas.NewEnvVar("azure-api-key-123"), Weight: 1, - Aliases: schemas.KeyAliases{"gpt-4o": "gpt-4o-deployment"}, + Aliases: schemas.KeyAliases{"gpt-4o": {ModelID: "gpt-4o-deployment"}}, AzureKeyConfig: &schemas.AzureKeyConfig{ Endpoint: *schemas.NewEnvVar("https://NEW-azure.openai.azure.com"), // Changed! }, @@ -6575,7 +6657,7 @@ func TestProviderHashComparison_BedrockConfigChangedInFile(t *testing.T) { Name: "aws-bedrock-key", Value: *schemas.NewEnvVar(""), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3-opus": "anthropic.claude-3-opus-20240229-v1:0"}, + Aliases: schemas.KeyAliases{"claude-3-opus": {ModelID: "anthropic.claude-3-opus-20240229-v1:0"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAIOSFODNN7EXAMPLE"), SecretKey: *schemas.NewEnvVar("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"), @@ -12243,6 +12325,10 @@ type mockLLMPlugin struct { mockPlugin } +func (p *mockLLMPlugin) PreRequestHook(_ *schemas.BifrostContext, _ *schemas.BifrostRequest) error { + return nil +} + func (p *mockLLMPlugin) PreLLMHook(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) (*schemas.BifrostRequest, *schemas.LLMPluginShortCircuit, error) { return req, nil, nil } @@ -14033,7 +14119,7 @@ func TestGenerateKeyHash_RuntimeVsMigrationParity(t *testing.T) { Value: *schemas.NewEnvVar("azure-key-value"), Weight: ptrFloat64(1.0), AzureKeyConfig: azureConfig, - Aliases: schemas.KeyAliases{"gpt-4": "gpt-4-deployment"}, + Aliases: schemas.KeyAliases{"gpt-4": {ModelID: "gpt-4-deployment"}}, } schemaKey := schemas.Key{ @@ -14985,7 +15071,7 @@ func TestKeyHashComparison_VertexConfigSyncScenarios(t *testing.T) { Name: "vertex-key", Value: *schemas.NewEnvVar("vertex-api-key-123"), Weight: 1, - Aliases: schemas.KeyAliases{"gemini-pro": "gemini-pro-endpoint"}, + Aliases: schemas.KeyAliases{"gemini-pro": {ModelID: "gemini-pro-endpoint"}}, VertexKeyConfig: &schemas.VertexKeyConfig{ ProjectID: *schemas.NewEnvVar("my-project-123"), ProjectNumber: *schemas.NewEnvVar("123456789"), @@ -14999,7 +15085,7 @@ func TestKeyHashComparison_VertexConfigSyncScenarios(t *testing.T) { Name: "vertex-key", Value: *schemas.NewEnvVar("vertex-api-key-123"), Weight: 1, - Aliases: schemas.KeyAliases{"gemini-pro": "gemini-pro-endpoint"}, + Aliases: schemas.KeyAliases{"gemini-pro": {ModelID: "gemini-pro-endpoint"}}, VertexKeyConfig: &schemas.VertexKeyConfig{ ProjectID: *schemas.NewEnvVar("my-project-123"), ProjectNumber: *schemas.NewEnvVar("123456789"), @@ -15129,7 +15215,7 @@ func TestKeyHashComparison_VertexConfigSyncScenarios(t *testing.T) { Name: "vertex-key", Value: *schemas.NewEnvVar("vertex-api-key-123"), Weight: 1, - Aliases: schemas.KeyAliases{"gemini-pro": "gemini-pro-endpoint"}, + Aliases: schemas.KeyAliases{"gemini-pro": {ModelID: "gemini-pro-endpoint"}}, VertexKeyConfig: &schemas.VertexKeyConfig{ ProjectID: *schemas.NewEnvVar("my-project-123"), Region: *schemas.NewEnvVar("us-central1"), @@ -15141,7 +15227,7 @@ func TestKeyHashComparison_VertexConfigSyncScenarios(t *testing.T) { Name: "vertex-key", Value: *schemas.NewEnvVar("vertex-api-key-123"), Weight: 1, - Aliases: schemas.KeyAliases{"gemini-pro": "gemini-pro-endpoint", "gemini-1.5-pro": "gemini-15-pro-endpoint"}, + Aliases: schemas.KeyAliases{"gemini-pro": {ModelID: "gemini-pro-endpoint"}, "gemini-1.5-pro": {ModelID: "gemini-15-pro-endpoint"}}, VertexKeyConfig: &schemas.VertexKeyConfig{ ProjectID: *schemas.NewEnvVar("my-project-123"), Region: *schemas.NewEnvVar("us-central1"), @@ -15528,7 +15614,7 @@ func TestKeyHashComparison_AzureDeploymentsChange(t *testing.T) { Name: "azure-key", Value: *schemas.NewEnvVar("azure-api-key"), Weight: 1, - Aliases: schemas.KeyAliases{"gpt-4": "gpt-4-deployment"}, + Aliases: schemas.KeyAliases{"gpt-4": {ModelID: "gpt-4-deployment"}}, AzureKeyConfig: &schemas.AzureKeyConfig{ Endpoint: *schemas.NewEnvVar("https://myazure.openai.azure.com"), }, @@ -15539,7 +15625,7 @@ func TestKeyHashComparison_AzureDeploymentsChange(t *testing.T) { Name: "azure-key", Value: *schemas.NewEnvVar("azure-api-key"), Weight: 1, - Aliases: schemas.KeyAliases{"gpt-4": "gpt-4-deployment", "gpt-4o": "gpt-4o-deployment"}, + Aliases: schemas.KeyAliases{"gpt-4": {ModelID: "gpt-4-deployment"}, "gpt-4o": {ModelID: "gpt-4o-deployment"}}, AzureKeyConfig: &schemas.AzureKeyConfig{ Endpoint: *schemas.NewEnvVar("https://myazure.openai.azure.com"), }, @@ -15559,7 +15645,7 @@ func TestKeyHashComparison_AzureDeploymentsChange(t *testing.T) { Name: "azure-key", Value: *schemas.NewEnvVar("azure-api-key"), Weight: 1, - Aliases: schemas.KeyAliases{"gpt-4": "gpt-4-deployment", "gpt-4o": "gpt-4o-deployment"}, + Aliases: schemas.KeyAliases{"gpt-4": {ModelID: "gpt-4-deployment"}, "gpt-4o": {ModelID: "gpt-4o-deployment"}}, AzureKeyConfig: &schemas.AzureKeyConfig{ Endpoint: *schemas.NewEnvVar("https://myazure.openai.azure.com"), }, @@ -15570,7 +15656,7 @@ func TestKeyHashComparison_AzureDeploymentsChange(t *testing.T) { Name: "azure-key", Value: *schemas.NewEnvVar("azure-api-key"), Weight: 1, - Aliases: schemas.KeyAliases{"gpt-4": "gpt-4-deployment"}, + Aliases: schemas.KeyAliases{"gpt-4": {ModelID: "gpt-4-deployment"}}, AzureKeyConfig: &schemas.AzureKeyConfig{ Endpoint: *schemas.NewEnvVar("https://myazure.openai.azure.com"), }, @@ -15590,7 +15676,7 @@ func TestKeyHashComparison_AzureDeploymentsChange(t *testing.T) { Name: "azure-key", Value: *schemas.NewEnvVar("azure-api-key"), Weight: 1, - Aliases: schemas.KeyAliases{"gpt-4": "gpt-4-deployment-v1"}, + Aliases: schemas.KeyAliases{"gpt-4": {ModelID: "gpt-4-deployment-v1"}}, AzureKeyConfig: &schemas.AzureKeyConfig{ Endpoint: *schemas.NewEnvVar("https://myazure.openai.azure.com"), }, @@ -15601,7 +15687,7 @@ func TestKeyHashComparison_AzureDeploymentsChange(t *testing.T) { Name: "azure-key", Value: *schemas.NewEnvVar("azure-api-key"), Weight: 1, - Aliases: schemas.KeyAliases{"gpt-4": "gpt-4-deployment-v2"}, + Aliases: schemas.KeyAliases{"gpt-4": {ModelID: "gpt-4-deployment-v2"}}, AzureKeyConfig: &schemas.AzureKeyConfig{ Endpoint: *schemas.NewEnvVar("https://myazure.openai.azure.com"), }, @@ -15631,7 +15717,7 @@ func TestKeyHashComparison_AzureDeploymentsChange(t *testing.T) { Name: "azure-key", Value: *schemas.NewEnvVar("azure-api-key"), Weight: 1, - Aliases: schemas.KeyAliases{"gpt-4": "gpt-4-deployment"}, + Aliases: schemas.KeyAliases{"gpt-4": {ModelID: "gpt-4-deployment"}}, AzureKeyConfig: &schemas.AzureKeyConfig{ Endpoint: *schemas.NewEnvVar("https://myazure.openai.azure.com"), }, @@ -15654,7 +15740,7 @@ func TestKeyHashComparison_BedrockDeploymentsChange(t *testing.T) { Name: "bedrock-key", Value: *schemas.NewEnvVar("bedrock-key"), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3": "arn:aws:bedrock:us-east-1::inference-profile/claude-3"}, + Aliases: schemas.KeyAliases{"claude-3": {ModelID: "arn:aws:bedrock:us-east-1::inference-profile/claude-3"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAIOSFODNN7EXAMPLE"), SecretKey: *schemas.NewEnvVar("wJalrXUtnFEMI"), @@ -15667,7 +15753,7 @@ func TestKeyHashComparison_BedrockDeploymentsChange(t *testing.T) { Name: "bedrock-key", Value: *schemas.NewEnvVar("bedrock-key"), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3": "arn:aws:bedrock:us-east-1::inference-profile/claude-3", "claude-3.5": "arn:aws:bedrock:us-east-1::inference-profile/claude-3.5"}, + Aliases: schemas.KeyAliases{"claude-3": {ModelID: "arn:aws:bedrock:us-east-1::inference-profile/claude-3"}, "claude-3.5": {ModelID: "arn:aws:bedrock:us-east-1::inference-profile/claude-3.5"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAIOSFODNN7EXAMPLE"), SecretKey: *schemas.NewEnvVar("wJalrXUtnFEMI"), @@ -15689,7 +15775,7 @@ func TestKeyHashComparison_BedrockDeploymentsChange(t *testing.T) { Name: "bedrock-key", Value: *schemas.NewEnvVar("bedrock-key"), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3": "arn:aws:bedrock:us-east-1::inference-profile/claude-3", "claude-3.5": "arn:aws:bedrock:us-east-1::inference-profile/claude-3.5"}, + Aliases: schemas.KeyAliases{"claude-3": {ModelID: "arn:aws:bedrock:us-east-1::inference-profile/claude-3"}, "claude-3.5": {ModelID: "arn:aws:bedrock:us-east-1::inference-profile/claude-3.5"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAIOSFODNN7EXAMPLE"), SecretKey: *schemas.NewEnvVar("wJalrXUtnFEMI"), @@ -15702,7 +15788,7 @@ func TestKeyHashComparison_BedrockDeploymentsChange(t *testing.T) { Name: "bedrock-key", Value: *schemas.NewEnvVar("bedrock-key"), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3": "arn:aws:bedrock:us-east-1::inference-profile/claude-3"}, + Aliases: schemas.KeyAliases{"claude-3": {ModelID: "arn:aws:bedrock:us-east-1::inference-profile/claude-3"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAIOSFODNN7EXAMPLE"), SecretKey: *schemas.NewEnvVar("wJalrXUtnFEMI"), @@ -15724,7 +15810,7 @@ func TestKeyHashComparison_BedrockDeploymentsChange(t *testing.T) { Name: "bedrock-key", Value: *schemas.NewEnvVar("bedrock-key"), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3": "arn:aws:bedrock:us-east-1::inference-profile/claude-3-old"}, + Aliases: schemas.KeyAliases{"claude-3": {ModelID: "arn:aws:bedrock:us-east-1::inference-profile/claude-3-old"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAIOSFODNN7EXAMPLE"), SecretKey: *schemas.NewEnvVar("wJalrXUtnFEMI"), @@ -15737,7 +15823,7 @@ func TestKeyHashComparison_BedrockDeploymentsChange(t *testing.T) { Name: "bedrock-key", Value: *schemas.NewEnvVar("bedrock-key"), Weight: 1, - Aliases: schemas.KeyAliases{"claude-3": "arn:aws:bedrock:us-east-1::inference-profile/claude-3-new"}, + Aliases: schemas.KeyAliases{"claude-3": {ModelID: "arn:aws:bedrock:us-east-1::inference-profile/claude-3-new"}}, BedrockKeyConfig: &schemas.BedrockKeyConfig{ AccessKey: *schemas.NewEnvVar("AKIAIOSFODNN7EXAMPLE"), SecretKey: *schemas.NewEnvVar("wJalrXUtnFEMI"), @@ -15762,7 +15848,7 @@ func TestKeyHashComparison_VertexDeploymentsChange(t *testing.T) { Name: "vertex-key", Value: *schemas.NewEnvVar("vertex-creds"), Weight: 1, - Aliases: schemas.KeyAliases{"gemini-pro": "gemini-pro-endpoint"}, + Aliases: schemas.KeyAliases{"gemini-pro": {ModelID: "gemini-pro-endpoint"}}, VertexKeyConfig: &schemas.VertexKeyConfig{ ProjectID: *schemas.NewEnvVar("my-project"), Region: *schemas.NewEnvVar("us-central1"), @@ -15774,7 +15860,7 @@ func TestKeyHashComparison_VertexDeploymentsChange(t *testing.T) { Name: "vertex-key", Value: *schemas.NewEnvVar("vertex-creds"), Weight: 1, - Aliases: schemas.KeyAliases{"gemini-pro": "gemini-pro-endpoint", "gemini-1.5-pro": "gemini-15-pro-endpoint"}, + Aliases: schemas.KeyAliases{"gemini-pro": {ModelID: "gemini-pro-endpoint"}, "gemini-1.5-pro": {ModelID: "gemini-15-pro-endpoint"}}, VertexKeyConfig: &schemas.VertexKeyConfig{ ProjectID: *schemas.NewEnvVar("my-project"), Region: *schemas.NewEnvVar("us-central1"), @@ -15795,7 +15881,7 @@ func TestKeyHashComparison_VertexDeploymentsChange(t *testing.T) { Name: "vertex-key", Value: *schemas.NewEnvVar("vertex-creds"), Weight: 1, - Aliases: schemas.KeyAliases{"gemini-pro": "gemini-pro-endpoint", "gemini-1.5-pro": "gemini-15-pro-endpoint"}, + Aliases: schemas.KeyAliases{"gemini-pro": {ModelID: "gemini-pro-endpoint"}, "gemini-1.5-pro": {ModelID: "gemini-15-pro-endpoint"}}, VertexKeyConfig: &schemas.VertexKeyConfig{ ProjectID: *schemas.NewEnvVar("my-project"), Region: *schemas.NewEnvVar("us-central1"), @@ -15807,7 +15893,7 @@ func TestKeyHashComparison_VertexDeploymentsChange(t *testing.T) { Name: "vertex-key", Value: *schemas.NewEnvVar("vertex-creds"), Weight: 1, - Aliases: schemas.KeyAliases{"gemini-pro": "gemini-pro-endpoint"}, + Aliases: schemas.KeyAliases{"gemini-pro": {ModelID: "gemini-pro-endpoint"}}, VertexKeyConfig: &schemas.VertexKeyConfig{ ProjectID: *schemas.NewEnvVar("my-project"), Region: *schemas.NewEnvVar("us-central1"), @@ -15828,7 +15914,7 @@ func TestKeyHashComparison_VertexDeploymentsChange(t *testing.T) { Name: "vertex-key", Value: *schemas.NewEnvVar("vertex-creds"), Weight: 1, - Aliases: schemas.KeyAliases{"gemini-pro": "gemini-pro-endpoint-v1"}, + Aliases: schemas.KeyAliases{"gemini-pro": {ModelID: "gemini-pro-endpoint-v1"}}, VertexKeyConfig: &schemas.VertexKeyConfig{ ProjectID: *schemas.NewEnvVar("my-project"), Region: *schemas.NewEnvVar("us-central1"), @@ -15840,7 +15926,7 @@ func TestKeyHashComparison_VertexDeploymentsChange(t *testing.T) { Name: "vertex-key", Value: *schemas.NewEnvVar("vertex-creds"), Weight: 1, - Aliases: schemas.KeyAliases{"gemini-pro": "gemini-pro-endpoint-v2"}, + Aliases: schemas.KeyAliases{"gemini-pro": {ModelID: "gemini-pro-endpoint-v2"}}, VertexKeyConfig: &schemas.VertexKeyConfig{ ProjectID: *schemas.NewEnvVar("my-project"), Region: *schemas.NewEnvVar("us-central1"), @@ -15872,7 +15958,7 @@ func TestKeyHashComparison_VertexDeploymentsChange(t *testing.T) { Name: "vertex-key", Value: *schemas.NewEnvVar("vertex-creds"), Weight: 1, - Aliases: schemas.KeyAliases{"gemini-pro": "gemini-pro-endpoint"}, + Aliases: schemas.KeyAliases{"gemini-pro": {ModelID: "gemini-pro-endpoint"}}, VertexKeyConfig: &schemas.VertexKeyConfig{ ProjectID: *schemas.NewEnvVar("my-project"), Region: *schemas.NewEnvVar("us-central1"), @@ -15940,6 +16026,9 @@ func getSchemaTypeMappings() []schemaTypeMapping { {"governance.virtual_keys.provider_configs", reflect.TypeOf(tables.TableVirtualKeyProviderConfig{}), true}, {"governance.virtual_keys.mcp_configs", reflect.TypeOf(tables.TableVirtualKeyMCPConfig{}), true}, {"governance.auth_config", reflect.TypeOf(configstore.AuthConfig{}), false}, + {"governance.complexity_analyzer_config", reflect.TypeOf(configstore.ComplexityAnalyzerConfig{}), false}, + {"governance.complexity_analyzer_config.tier_boundaries", reflect.TypeOf(configstore.ComplexityTierBoundaries{}), false}, + {"governance.complexity_analyzer_config.keywords", reflect.TypeOf(configstore.ComplexityEditableKeywordConfig{}), false}, // Plugins {"plugins", reflect.TypeOf(schemas.PluginConfig{}), true}, @@ -16074,6 +16163,12 @@ var excludedSchemaFields = map[string]map[string]bool{ "governance": { "business_units": true, // Enterprise feature; not in OSS GovernanceConfig }, + "auth_config": { + "disable_auth_on_inference": true, // Deprecated and ignored; kept in schema for backward-compatible config.json validation. Use enforce_auth_on_inference. + }, + "governance.auth_config": { + "disable_auth_on_inference": true, // Deprecated and ignored; kept in schema for backward-compatible config.json validation. Use enforce_auth_on_inference. + }, "governance.teams": { "budget_id": true, // Replaced by budgets[] relationship with team_id FK on TableBudget "business_unit_id": true, // Enterprise feature; not in OSS TableTeam @@ -16399,6 +16494,7 @@ func TestResolveFrameworkPricingConfig(t *testing.T) { defaultURL := modelcatalog.DefaultPricingURL defaultSyncSeconds := int64(modelcatalog.DefaultSyncInterval.Seconds()) defaultModelParamsURL := modelcatalog.DefaultModelParametersURL + defaultMCPLibraryURL := modelcatalog.DefaultMCPLibraryURL fileURL := "https://example.com/pricing.json" fileSyncSeconds := int64((12 * time.Hour).Seconds()) dbURL := "https://db.example.com/pricing.json" @@ -16439,11 +16535,13 @@ func TestResolveFrameworkPricingConfig(t *testing.T) { uiEditedURL := "https://ui-edited.example.com/pricing.json" uiEditedSync := int64((24 * time.Hour).Seconds()) dbConfig := &tables.TableFrameworkConfig{ - ID: 8, - PricingURL: &uiEditedURL, - PricingSyncInterval: &uiEditedSync, - ModelParametersURL: &defaultModelParamsURL, - ConfigHash: storedHash, // hash of last file-applied values + ID: 8, + PricingURL: &uiEditedURL, + PricingSyncInterval: &uiEditedSync, + ModelParametersURL: &defaultModelParamsURL, + MCPLibraryURL: &defaultMCPLibraryURL, + MCPLibrarySyncInterval: &defaultSyncSeconds, + ConfigHash: storedHash, // hash of last file-applied values } fileConfig := &framework.FrameworkConfig{ Pricing: &modelcatalog.Config{ @@ -16525,8 +16623,118 @@ func TestResolveFrameworkPricingConfig(t *testing.T) { require.False(t, needsDBUpdate) require.Equal(t, defaultURL, *normalizedTable.PricingURL) require.Equal(t, defaultSyncSeconds, *normalizedTable.PricingSyncInterval) + require.Equal(t, defaultMCPLibraryURL, *normalizedTable.MCPLibraryURL) + require.Equal(t, defaultSyncSeconds, *normalizedTable.MCPLibrarySyncInterval) require.Equal(t, defaultURL, *normalizedModelCatalog.PricingURL) require.Equal(t, defaultSyncSeconds, *normalizedModelCatalog.PricingSyncInterval) + require.Equal(t, defaultMCPLibraryURL, *normalizedModelCatalog.MCPLibraryURL) + require.Equal(t, defaultSyncSeconds, *normalizedModelCatalog.MCPLibrarySyncInterval) + }) + + t.Run("mcp library file values override defaults", func(t *testing.T) { + mcpURL := "https://example.com/mcp-library.json" + mcpSyncSeconds := int64((2 * time.Hour).Seconds()) + fileConfig := &framework.FrameworkConfig{ + Pricing: &modelcatalog.Config{ + MCPLibraryURL: &mcpURL, + MCPLibrarySyncInterval: &mcpSyncSeconds, + }, + } + + normalizedTable, normalizedModelCatalog, needsDBUpdate := ResolveFrameworkPricingConfig(nil, fileConfig) + require.False(t, needsDBUpdate) + require.Equal(t, mcpURL, *normalizedTable.MCPLibraryURL) + require.Equal(t, mcpSyncSeconds, *normalizedTable.MCPLibrarySyncInterval) + require.Equal(t, mcpURL, *normalizedModelCatalog.MCPLibraryURL) + require.Equal(t, mcpSyncSeconds, *normalizedModelCatalog.MCPLibrarySyncInterval) + }) + + t.Run("mcp library file values override db when file hash changed", func(t *testing.T) { + mcpURL := "https://file.example.com/mcp-library.json" + mcpSyncSeconds := int64((2 * time.Hour).Seconds()) + dbMCPURL := modelcatalog.DefaultMCPLibraryURL + dbMCPSyncSeconds := int64((24 * time.Hour).Seconds()) + dbConfig := &tables.TableFrameworkConfig{ + ID: 11, + MCPLibraryURL: &dbMCPURL, + MCPLibrarySyncInterval: &dbMCPSyncSeconds, + ConfigHash: "old-hash", + } + fileConfig := &framework.FrameworkConfig{ + Pricing: &modelcatalog.Config{ + MCPLibraryURL: &mcpURL, + MCPLibrarySyncInterval: &mcpSyncSeconds, + }, + } + + normalizedTable, normalizedModelCatalog, needsDBUpdate := ResolveFrameworkPricingConfig(dbConfig, fileConfig) + require.True(t, needsDBUpdate) + require.Equal(t, mcpURL, *normalizedTable.MCPLibraryURL) + require.Equal(t, mcpSyncSeconds, *normalizedTable.MCPLibrarySyncInterval) + require.Equal(t, mcpURL, *normalizedModelCatalog.MCPLibraryURL) + require.Equal(t, mcpSyncSeconds, *normalizedModelCatalog.MCPLibrarySyncInterval) + }) + + t.Run("mcp library db wins when file hash matches stored hash", func(t *testing.T) { + fileMCPURL := "https://file.example.com/mcp-library.json" + fileMCPSyncSeconds := int64((2 * time.Hour).Seconds()) + storedHash, err := configstore.GenerateFrameworkConfigHash(nil, nil, nil, configstore.FrameworkConfigHashOptions{ + MCPLibraryURL: &fileMCPURL, + MCPLibrarySyncInterval: &fileMCPSyncSeconds, + }) + require.NoError(t, err) + uiEditedMCPURL := "https://ui.example.com/mcp-library.json" + uiEditedMCPSyncSeconds := int64((3 * time.Hour).Seconds()) + dbConfig := &tables.TableFrameworkConfig{ + ID: 12, + PricingURL: &defaultURL, + PricingSyncInterval: &defaultSyncSeconds, + ModelParametersURL: &defaultModelParamsURL, + MCPLibraryURL: &uiEditedMCPURL, + MCPLibrarySyncInterval: &uiEditedMCPSyncSeconds, + ConfigHash: storedHash, + } + fileConfig := &framework.FrameworkConfig{ + Pricing: &modelcatalog.Config{ + MCPLibraryURL: &fileMCPURL, + MCPLibrarySyncInterval: &fileMCPSyncSeconds, + }, + } + + normalizedTable, normalizedModelCatalog, needsDBUpdate := ResolveFrameworkPricingConfig(dbConfig, fileConfig) + require.False(t, needsDBUpdate) + require.Equal(t, uiEditedMCPURL, *normalizedTable.MCPLibraryURL) + require.Equal(t, uiEditedMCPSyncSeconds, *normalizedTable.MCPLibrarySyncInterval) + require.Equal(t, uiEditedMCPURL, *normalizedModelCatalog.MCPLibraryURL) + require.Equal(t, uiEditedMCPSyncSeconds, *normalizedModelCatalog.MCPLibrarySyncInterval) + }) + + t.Run("mcp library file interval below minimum is clamped", func(t *testing.T) { + tooLow := int64(1800) + fileConfig := &framework.FrameworkConfig{ + Pricing: &modelcatalog.Config{ + MCPLibrarySyncInterval: &tooLow, + }, + } + + normalizedTable, normalizedModelCatalog, needsDBUpdate := ResolveFrameworkPricingConfig(nil, fileConfig) + require.False(t, needsDBUpdate) + require.Equal(t, modelcatalog.MinimumPricingSyncIntervalSec, *normalizedTable.MCPLibrarySyncInterval) + require.Equal(t, modelcatalog.MinimumPricingSyncIntervalSec, *normalizedModelCatalog.MCPLibrarySyncInterval) + }) + + t.Run("mcp library invalid db interval falls back and requests db update", func(t *testing.T) { + invalidDBSync := int64(0) + dbConfig := &tables.TableFrameworkConfig{ + ID: 10, + MCPLibraryURL: &defaultMCPLibraryURL, + MCPLibrarySyncInterval: &invalidDBSync, + } + + normalizedTable, normalizedModelCatalog, needsDBUpdate := ResolveFrameworkPricingConfig(dbConfig, nil) + require.True(t, needsDBUpdate) + require.Equal(t, defaultSyncSeconds, *normalizedTable.MCPLibrarySyncInterval) + require.Equal(t, defaultSyncSeconds, *normalizedModelCatalog.MCPLibrarySyncInterval) }) t.Run("invalid db interval (zero) falls back and requests db update", func(t *testing.T) { @@ -16671,10 +16879,14 @@ func TestResolveFrameworkPricingConfig(t *testing.T) { require.NotNil(t, tableOut.PricingURL, "PricingURL must never be nil") require.NotNil(t, tableOut.PricingSyncInterval, "PricingSyncInterval must never be nil") require.NotNil(t, tableOut.ModelParametersURL, "ModelParametersURL must never be nil") + require.NotNil(t, tableOut.MCPLibraryURL, "MCPLibraryURL must never be nil") + require.NotNil(t, tableOut.MCPLibrarySyncInterval, "MCPLibrarySyncInterval must never be nil") require.NotNil(t, catalogOut, "modelcatalog.Config must never be nil") require.NotNil(t, catalogOut.PricingURL, "Config.PricingURL must never be nil") require.NotNil(t, catalogOut.PricingSyncInterval, "Config.PricingSyncInterval must never be nil") require.NotNil(t, catalogOut.ModelParametersURL, "Config.ModelParametersURL must never be nil") + require.NotNil(t, catalogOut.MCPLibraryURL, "Config.MCPLibraryURL must never be nil") + require.NotNil(t, catalogOut.MCPLibrarySyncInterval, "Config.MCPLibrarySyncInterval must never be nil") } }) @@ -16874,21 +17086,17 @@ func TestLoadAuthConfigFromFile_PasswordHashing(t *testing.T) { require.NoError(t, err) mockStore.authConfig = &configstore.AuthConfig{ - AdminUserName: schemas.NewEnvVar("sameadmin"), - AdminPassword: schemas.NewEnvVar(hashedPassword), - IsEnabled: true, - DisableAuthOnInference: false, - } + AdminUserName: schemas.NewEnvVar("sameadmin"), + AdminPassword: schemas.NewEnvVar(hashedPassword), + IsEnabled: true} config := &Config{ ConfigStore: mockStore, } configData := &ConfigData{ AuthConfig: &configstore.AuthConfig{ - AdminUserName: schemas.NewEnvVar("sameadmin"), - AdminPassword: schemas.NewEnvVar(plainPassword), - IsEnabled: true, - DisableAuthOnInference: false, - }, + AdminUserName: schemas.NewEnvVar("sameadmin"), + AdminPassword: schemas.NewEnvVar(plainPassword), + IsEnabled: true}, } loadAuthConfig(ctx, config, configData) @@ -17090,21 +17298,17 @@ func TestLoadAuthConfigFromFile_PasswordHashing(t *testing.T) { require.NoError(t, err) mockStore.authConfig = &configstore.AuthConfig{ - AdminUserName: schemas.NewEnvVar("admin"), - AdminPassword: schemas.NewEnvVar(hashedPassword), - IsEnabled: true, - DisableAuthOnInference: false, - } + AdminUserName: schemas.NewEnvVar("admin"), + AdminPassword: schemas.NewEnvVar(hashedPassword), + IsEnabled: true} config := &Config{ ConfigStore: mockStore, } configData := &ConfigData{ AuthConfig: &configstore.AuthConfig{ - AdminUserName: schemas.NewEnvVar("admin"), - AdminPassword: schemas.NewEnvVar(plainPassword), - IsEnabled: true, - DisableAuthOnInference: false, - }, + AdminUserName: schemas.NewEnvVar("admin"), + AdminPassword: schemas.NewEnvVar(plainPassword), + IsEnabled: true}, } loadAuthConfig(ctx, config, configData) diff --git a/transports/bifrost-http/lib/ctx.go b/transports/bifrost-http/lib/ctx.go index e94bbeabe1..c86e023301 100644 --- a/transports/bifrost-http/lib/ctx.go +++ b/transports/bifrost-http/lib/ctx.go @@ -33,8 +33,9 @@ const ( // It is used by transport middleware to avoid re-buffering response bodies for post-hooks. FastHTTPUserValueLargeResponseMode = "__bifrost_large_response_mode" // FastHTTPUserValueModelCatalogResolution stores model catalog resolution metadata - // set by prepare*Request functions when a provider was auto-resolved. Picked up - // centrally in ConvertToBifrostContext to add the routing engine log. + // set by prepare*Request functions (and inline realtime catalog lookups) when a + // provider was auto-resolved. Picked up centrally in ConvertToBifrostContext to + // add the routing engine log via EmitModelCatalogRoutingLog. FastHTTPUserValueModelCatalogResolution = "__bifrost_model_catalog_resolution" ) @@ -46,6 +47,26 @@ type ModelCatalogResolution struct { AllProviders []schemas.ModelProvider } +// EmitModelCatalogRoutingLog appends a RoutingEngineModelCatalog log entry and +// engines-used marker to bifrostCtx for an inline catalog resolution. Used by +// ConvertToBifrostContext (normal HTTP path) and by realtime handlers that +// bypass it (WebRTC, realtime client_secrets) so all paths emit observability +// in the same shape regardless of which routing layer did the lookup. +func EmitModelCatalogRoutingLog(bifrostCtx *schemas.BifrostContext, res *ModelCatalogResolution) { + if bifrostCtx == nil || res == nil { + return + } + providerStrs := make([]string, len(res.AllProviders)) + for i, p := range res.AllProviders { + providerStrs[i] = string(p) + } + bifrostCtx.AppendRoutingEngineLog(schemas.RoutingEngineModelCatalog, schemas.LogLevelInfo, fmt.Sprintf( + "No provider specified for model %s, found %d options in model catalog: [%s], selected: %s", + res.Model, len(res.AllProviders), strings.Join(providerStrs, ", "), res.ResolvedProvider, + )) + schemas.AppendToContextList(bifrostCtx, schemas.BifrostContextKeyRoutingEnginesUsed, schemas.RoutingEngineModelCatalog) +} + // ParseSessionIDFromBaggage extracts the session-id baggage member value. // It supports simple W3C baggage parsing sufficient for log grouping. func ParseSessionIDFromBaggage(header string) string { @@ -208,15 +229,7 @@ func ConvertToBifrostContext(ctx *fasthttp.RequestCtx, store HandlerStore) (*sch // it stores the resolution info on the fasthttp context. Emit the routing // engine log and mark the engine as used centrally here. if res, ok := ctx.UserValue(FastHTTPUserValueModelCatalogResolution).(*ModelCatalogResolution); ok && res != nil { - providerStrs := make([]string, len(res.AllProviders)) - for i, p := range res.AllProviders { - providerStrs[i] = string(p) - } - bifrostCtx.AppendRoutingEngineLog(schemas.RoutingEngineModelCatalog, schemas.LogLevelInfo, fmt.Sprintf( - "No provider specified for model %s, found %d options in model catalog: [%s], selecting first: %s", - res.Model, len(res.AllProviders), strings.Join(providerStrs, ", "), res.ResolvedProvider, - )) - schemas.AppendToContextList(bifrostCtx, schemas.BifrostContextKeyRoutingEnginesUsed, schemas.RoutingEngineModelCatalog) + EmitModelCatalogRoutingLog(bifrostCtx, res) } // Initialize tags map for collecting maxim tags @@ -621,6 +634,18 @@ func ConvertToBifrostContext(ctx *fasthttp.RequestCtx, store HandlerStore) (*sch }) bifrostCtx.SetValue(schemas.BifrostContextKeyRequestHeaders, allHeaders) + // Collect all request query params for downstream use (e.g., governance routing CEL rules + // that read params["..."]). Keys are lowercased for case-insensitive lookup. + queryArgs := ctx.Request.URI().QueryArgs() + if queryArgs.Len() > 0 { + allQuery := make(map[string]string, queryArgs.Len()) + queryArgs.All()(func(key, value []byte) bool { + allQuery[strings.ToLower(string(key))] = string(value) + return true + }) + bifrostCtx.SetValue(schemas.BifrostContextKeyRequestQuery, allQuery) + } + // Build and set the MCP callback base URL. Used by per-user OAuth (appends // /api/oauth/callback) and per-user headers (appends the workspace submit // path) resolvers when initiating their respective auth flows. Bifrost is diff --git a/transports/bifrost-http/server/plugins.go b/transports/bifrost-http/server/plugins.go index a868964e4e..e2ec513949 100644 --- a/transports/bifrost-http/server/plugins.go +++ b/transports/bifrost-http/server/plugins.go @@ -3,6 +3,7 @@ package server import ( "context" "fmt" + "math" "slices" "github.com/maximhq/bifrost/core/schemas" @@ -10,6 +11,7 @@ import ( "github.com/maximhq/bifrost/plugins/governance" "github.com/maximhq/bifrost/plugins/logging" "github.com/maximhq/bifrost/plugins/maxim" + "github.com/maximhq/bifrost/plugins/modelcatalogresolver" "github.com/maximhq/bifrost/plugins/otel" "github.com/maximhq/bifrost/plugins/prompts" "github.com/maximhq/bifrost/plugins/semanticcache" @@ -120,6 +122,9 @@ func loadBuiltinPlugin(ctx context.Context, name string, pluginConfig any, bifro } return compat.Init(*compatConfig, logger, bifrostConfig.ModelCatalog) + case modelcatalogresolver.PluginName: + return modelcatalogresolver.Init(bifrostConfig.ModelCatalog, logger) + default: return nil, fmt.Errorf("unknown built-in plugin: %s", name) } @@ -252,6 +257,20 @@ func (s *BifrostHTTPServer) loadBuiltinPlugins(ctx context.Context) error { } s.Config.SetPluginOrderInfo(maxim.PluginName, builtinPlacement, schemas.Ptr(8)) + // 9. ModelCatalogResolver (last routing layer — fills req.Provider from catalog only when + // no earlier routing plugin (governance routing rules, governance VK LB, enterprise LB) + // already set one. CEL rules can still match on provider == "" because this runs last. + // Requires a model catalog; only register when one is configured. + if s.Config.ModelCatalog != nil { + s.registerPluginWithStatus(ctx, modelcatalogresolver.PluginName, nil, nil, false) + } else { + s.markPluginDisabled(modelcatalogresolver.PluginName) + } + // Place it in post_builtin with a max order so it runs after every other routing plugin, + // including post_builtin ones like the enterprise load balancer (which would otherwise run + // after this builtin and never get a chance to pick the provider first). + s.Config.SetPluginOrderInfo(modelcatalogresolver.PluginName, schemas.Ptr(schemas.PluginPlacementPostBuiltin), schemas.Ptr(math.MaxInt)) + return nil } diff --git a/transports/bifrost-http/server/server.go b/transports/bifrost-http/server/server.go index 56b380cf88..b097ac1769 100644 --- a/transports/bifrost-http/server/server.go +++ b/transports/bifrost-http/server/server.go @@ -9,6 +9,7 @@ import ( "net" "os" "os/signal" + "strconv" "strings" "sync" "syscall" @@ -26,7 +27,9 @@ import ( "github.com/maximhq/bifrost/framework/temptoken" "github.com/maximhq/bifrost/framework/tracing" "github.com/maximhq/bifrost/plugins/governance" + "github.com/maximhq/bifrost/plugins/governance/complexity" "github.com/maximhq/bifrost/plugins/logging" + "github.com/maximhq/bifrost/plugins/otel" "github.com/maximhq/bifrost/plugins/prompts" "github.com/maximhq/bifrost/plugins/semanticcache" "github.com/maximhq/bifrost/plugins/telemetry" @@ -64,6 +67,7 @@ type ServerCallbacks interface { ReloadPlugin(ctx context.Context, name string, path *string, pluginConfig any, placement *schemas.PluginPlacement, order *int) error RemovePlugin(ctx context.Context, name string) error GetPluginStatus(ctx context.Context) map[string]schemas.PluginStatus + GetLoadedPluginNames() []string NormalizePluginConfig(name string, config map[string]any) (map[string]any, error) ExpandPluginConfigForAPI(name string, config map[string]any) (map[string]any, error) // Auth related callbacks @@ -100,6 +104,9 @@ type ServerCallbacks interface { RemoveModelConfig(ctx context.Context, id string) error ReloadProvider(ctx context.Context, provider schemas.ModelProvider) (*tables.TableProvider, error) RemoveProvider(ctx context.Context, provider schemas.ModelProvider) error + OnKeyAdded(ctx context.Context, provider schemas.ModelProvider, key schemas.Key) error + OnKeyUpdated(ctx context.Context, provider schemas.ModelProvider, key schemas.Key) error + OnKeyDeleted(ctx context.Context, provider schemas.ModelProvider, keyID string) error ReloadRoutingRule(ctx context.Context, id string) error RemoveRoutingRule(ctx context.Context, id string) error // MCP related callbacks @@ -624,84 +631,23 @@ func (s *BifrostHTTPServer) ReloadProvider(ctx context.Context, provider schemas } } - // Read current key count from in-memory store (providerInfo.Keys is not preloaded from DB) - inMemoryKeys, _ := s.Config.GetProviderKeysRaw(provider) - isKeylessProvider := providerInfo.CustomProviderConfig != nil && providerInfo.CustomProviderConfig.IsKeyLess - hasNoKeys := len(inMemoryKeys) == 0 && !isKeylessProvider - - // Getting allowed models from all provider keys (needed before model listing) - providerKeys, err := s.Config.ConfigStore.GetKeysByProvider(ctx, string(provider)) + // In-memory store holds the latest schemas.Key slice after the most recent + // CRUD write — read from there to avoid re-fetching + re-converting from DB. + inMemoryKeys, err := s.Config.GetProviderKeysRaw(provider) if err != nil { - return nil, fmt.Errorf("failed to update provider model catalog: failed to get keys by provider: %s", err) - } - - bfCtx := schemas.NewBifrostContext(ctx, time.Now().Add(15*time.Second)) - bfCtx.SetValue(schemas.BifrostContextKeySkipPluginPipeline, true) - bfCtx.SetValue(schemas.BifrostContextKeyValidateKeys, true) // Validate keys during provider add/update - defer bfCtx.Cancel() - - // Run filtered and unfiltered model listing concurrently - var ( - allModels *schemas.BifrostListModelsResponse - bifrostErr *schemas.BifrostError - unfilteredModels *schemas.BifrostListModelsResponse - listModelsErr *schemas.BifrostError - listWg sync.WaitGroup - ) - listWg.Add(2) - go func() { - defer listWg.Done() - allModels, bifrostErr = s.Client.ListModelsRequest(bfCtx, &schemas.BifrostListModelsRequest{ - Provider: provider, - }) - }() - go func() { - defer listWg.Done() - unfilteredModels, listModelsErr = s.Client.ListModelsRequest(bfCtx, &schemas.BifrostListModelsRequest{ - Provider: provider, - Unfiltered: true, - }) - }() - listWg.Wait() - - if allModels != nil && len(allModels.KeyStatuses) > 0 && s.Config.ConfigStore != nil { - s.updateKeyStatus(ctx, allModels.KeyStatuses) + return nil, fmt.Errorf("failed to read provider keys for %s: %w", provider, err) } - if bifrostErr != nil { - if len(bifrostErr.ExtraFields.KeyStatuses) > 0 && s.Config.ConfigStore != nil { - s.updateKeyStatus(ctx, bifrostErr.ExtraFields.KeyStatuses) - } + isKeylessProvider := providerInfo.CustomProviderConfig != nil && providerInfo.CustomProviderConfig.IsKeyLess + hasNoKeys := len(inMemoryKeys) == 0 && !isKeylessProvider - if hasNoKeys { - logger.Warn("model discovery skipped for provider %s: no keys configured", provider) - } else { - logger.Warn("failed to update provider model catalog: failed to list all models: %s. We are falling back onto the static datasheet", bifrost.GetErrorMessage(bifrostErr)) - } - // In case of error, we return an empty list of models, and fallback onto the static datasheet - allModels = &schemas.BifrostListModelsResponse{ - Data: make([]schemas.Model, 0), - } - } - modelsInKeys := make([]schemas.Model, 0) - for _, key := range providerKeys { - if key.Models.IsUnrestricted() { - continue - } - for _, model := range key.Models { - modelsInKeys = append(modelsInKeys, schemas.Model{ - ID: string(provider) + "/" + model, - }) - } - } - s.Config.ModelCatalog.UpsertModelDataForProvider(provider, allModels, modelsInKeys) - if listModelsErr != nil { - if hasNoKeys { - logger.Warn("unfiltered model discovery skipped for provider %s: no keys configured", provider) - } else { - logger.Error("failed to list unfiltered models for provider %s: %v: falling back onto the static datasheet", provider, bifrost.GetErrorMessage(listModelsErr)) - } + // Refresh keyconfig from the current key list, then drop any stale live + // entries (for keys removed in this update) before refetching per-key. + s.Config.ModelCatalog.SetKeyConfigForProvider(provider, inMemoryKeys) + s.Config.ModelCatalog.InvalidateLiveProvider(provider) + if hasNoKeys { + logger.Warn("model discovery skipped for provider %s: no keys configured", provider) } else { - s.Config.ModelCatalog.UpsertUnfilteredModelDataForProvider(provider, unfilteredModels) + s.RefreshLiveModelsForProvider(ctx, provider, inMemoryKeys) } return updatedProvider, nil } @@ -726,11 +672,84 @@ func (s *BifrostHTTPServer) RemoveProvider(ctx context.Context, provider schemas if s.Config == nil || s.Config.ModelCatalog == nil { return fmt.Errorf("pricing manager not found") } - s.Config.ModelCatalog.DeleteModelDataForProvider(provider) + s.Config.ModelCatalog.InvalidateLiveProvider(provider) + s.Config.ModelCatalog.RemoveKeyConfigForProvider(provider) + + return nil +} + +// OnKeyAdded refreshes the keyconfig snapshot and fetches list-models for the +// new key only — 2 calls instead of ReloadProvider's 2×N. Called by the key +// handler after a successful AddProviderKey write. +func (s *BifrostHTTPServer) OnKeyAdded(ctx context.Context, provider schemas.ModelProvider, key schemas.Key) error { + if s.Config == nil || s.Config.ModelCatalog == nil { + return fmt.Errorf("model catalog not found") + } + keys, err := s.Config.GetProviderKeysRaw(provider) + if err != nil { + return fmt.Errorf("failed to read provider keys for %s: %w", provider, err) + } + s.Config.ModelCatalog.SetKeyConfigForProvider(provider, keys) + // Keyless providers: empty keyID sentinel. + keyID := key.ID + if isKeylessProvider(provider, s.Config) { + keyID = "" + } + s.FetchAndStoreLiveForKey(ctx, provider, keyID) + return nil +} + +// OnKeyUpdated invalidates the affected key's live entries (the gate may have +// changed even when Value didn't), refreshes the keyconfig, then refetches +// for just that key. 2 calls regardless of N keys on the provider. +func (s *BifrostHTTPServer) OnKeyUpdated(ctx context.Context, provider schemas.ModelProvider, key schemas.Key) error { + if s.Config == nil || s.Config.ModelCatalog == nil { + return fmt.Errorf("model catalog not found") + } + keys, err := s.Config.GetProviderKeysRaw(provider) + if err != nil { + return fmt.Errorf("failed to read provider keys for %s: %w", provider, err) + } + s.Config.ModelCatalog.SetKeyConfigForProvider(provider, keys) + keyID := key.ID + if isKeylessProvider(provider, s.Config) { + keyID = "" + } + s.Config.ModelCatalog.InvalidateLive(provider, keyID) + s.FetchAndStoreLiveForKey(ctx, provider, keyID) + return nil +} +// OnKeyDeleted invalidates the deleted key's live entries and refreshes the +// keyconfig. No list-models calls — the provider's remaining keys' cached +// entries stay valid. +func (s *BifrostHTTPServer) OnKeyDeleted(ctx context.Context, provider schemas.ModelProvider, keyID string) error { + if s.Config == nil || s.Config.ModelCatalog == nil { + return fmt.Errorf("model catalog not found") + } + keys, err := s.Config.GetProviderKeysRaw(provider) + if err != nil { + return fmt.Errorf("failed to read provider keys for %s: %w", provider, err) + } + s.Config.ModelCatalog.SetKeyConfigForProvider(provider, keys) + s.Config.ModelCatalog.InvalidateLive(provider, keyID) return nil } +// isKeylessProvider returns true when the provider's config marks it +// keyless. Used to pick the live-cache key for OnKey* helpers: keyless +// providers cache under the empty-string sentinel. +func isKeylessProvider(provider schemas.ModelProvider, cfg *lib.Config) bool { + if cfg == nil { + return false + } + pc, err := cfg.GetProviderConfigRaw(provider) + if err != nil || pc == nil || pc.CustomProviderConfig == nil { + return false + } + return pc.CustomProviderConfig.IsKeyLess +} + // GetGovernanceData returns the governance data func (s *BifrostHTTPServer) GetGovernanceData(ctx context.Context) *governance.GovernanceData { // Use type-safe finder from Config @@ -741,6 +760,22 @@ func (s *BifrostHTTPServer) GetGovernanceData(ctx context.Context) *governance.G return governancePlugin.GetGovernanceStore().GetGovernanceData(ctx) } +// ReloadComplexityAnalyzerConfig reloads the complexity analyzer config into the governance plugin. +func (s *BifrostHTTPServer) ReloadComplexityAnalyzerConfig(ctx context.Context, config *complexity.AnalyzerConfig) error { + governancePlugin, err := s.getGovernancePlugin() + if err != nil { + return fmt.Errorf("governance plugin not found: %w", err) + } + reloader, ok := governancePlugin.(interface { + ReloadComplexityAnalyzerConfig(config *complexity.AnalyzerConfig) + }) + if !ok { + return fmt.Errorf("governance plugin does not support complexity analyzer config reload") + } + reloader.ReloadComplexityAnalyzerConfig(config) + return nil +} + // ReloadRoutingRule reloads a routing rule from the database into the governance store func (s *BifrostHTTPServer) ReloadRoutingRule(ctx context.Context, id string) error { governancePluginName := governance.PluginName @@ -800,6 +835,7 @@ func (s *BifrostHTTPServer) ReloadClientConfigFromConfigStore(ctx context.Contex if s.AuthMiddleware != nil { s.AuthMiddleware.UpdateWhitelistedRoutes(config.WhitelistedRoutes) s.AuthMiddleware.UpdateTempTokenAuthEnabled(config.MCPEnableTempTokenAuth) + s.AuthMiddleware.UpdateEnforceAuthOnInference(config.EnforceAuthOnInference) } // Reloading config in bifrost client if s.Client != nil { @@ -895,51 +931,120 @@ func (s *BifrostHTTPServer) UpdateSyncConfig(ctx context.Context) error { return s.Config.ModelCatalog.UpdateSyncConfig(ctx, s.Config.FrameworkConfig.Pricing) } -func (s *BifrostHTTPServer) populateModelPoolWithListModels(ctx context.Context) error { - // Fetching keys for all providers and allowed models first - // Based on allowed models we will set the data in the model catalog +// RefreshLiveModelsForProvider runs filtered + unfiltered list-models for the +// provider, fanning out per key in parallel so the live cache ends up with +// per-(provider, keyID) entries. Keyless providers cache under the "" sentinel. +// +// Callers are responsible for invalidating stale entries first when keys +// have been removed from the provider's set. +func (s *BifrostHTTPServer) RefreshLiveModelsForProvider(ctx context.Context, provider schemas.ModelProvider, keys []schemas.Key) { + if len(keys) == 0 { + // Empty key slice + non-keyless provider would write under the "" sentinel + // reserved for keyless providers — colliding with the keyless namespace and + // triggering an unauthenticated fetch for a provider that requires a key. + if !isKeylessProvider(provider, s.Config) { + logger.Warn("model discovery skipped for provider %s: no keys configured", provider) + return + } + s.FetchAndStoreLiveForKey(ctx, provider, "") + return + } var wg sync.WaitGroup - for provider, providerConfig := range s.Config.Providers { + for _, key := range keys { wg.Add(1) - go func(provider schemas.ModelProvider, providerConfig configstore.ProviderConfig) { + go func(keyID string) { defer wg.Done() - bfCtx := schemas.NewBifrostContext(ctx, time.Now().Add(15*time.Second)) - bfCtx.SetValue(schemas.BifrostContextKeySkipPluginPipeline, true) - defer bfCtx.Cancel() - modelData, listModelsErr := s.Client.ListModelsRequest(bfCtx, &schemas.BifrostListModelsRequest{ - Provider: provider, - }) - if listModelsErr != nil { - logger.Error("failed to list models for provider %s: %v: falling back onto the static datasheet", provider, bifrost.GetErrorMessage(listModelsErr)) - } - allowedModels := make([]schemas.Model, 0) - for _, key := range providerConfig.Keys { - if key.Models.IsUnrestricted() { - continue - } - for _, model := range key.Models { - allowedModels = append(allowedModels, schemas.Model{ - ID: string(provider) + "/" + model, - }) - } - } - s.Config.ModelCatalog.UpsertModelDataForProvider(provider, modelData, allowedModels) - unfilteredModelData, listModelsErr := s.Client.ListModelsRequest(bfCtx, &schemas.BifrostListModelsRequest{ - Provider: provider, - Unfiltered: true, - }) - if listModelsErr != nil { - logger.Error("failed to list unfiltered models for provider %s: %v: falling back onto the static datasheet", provider, bifrost.GetErrorMessage(listModelsErr)) - } else { - s.Config.ModelCatalog.UpsertUnfilteredModelDataForProvider(provider, unfilteredModelData) - } - }(provider, providerConfig) + s.FetchAndStoreLiveForKey(ctx, provider, keyID) + }(key.ID) } wg.Wait() - return nil } -// ForceReloadPricing triggers an immediate pricing sync and resets the sync timer +// FetchAndStoreLiveForKey issues the filtered and unfiltered list-models +// calls for one (provider, keyID) in parallel and writes the results into +// the catalog. Errors are logged and surfaced via updateKeyStatus when the +// provider returns per-key statuses, but they do not abort the other call. +// keyID="" scopes to "no specific key" — used for keyless providers and as +// the legacy sentinel. Always validates keys for the providers that opt into +// the check (today: OpenRouter, whose /v1/models is unauthenticated) so the +// routing graph is the same at boot, after a key add, and after a reload — +// stale-but-routable behavior would diverge otherwise. +func (s *BifrostHTTPServer) FetchAndStoreLiveForKey(ctx context.Context, provider schemas.ModelProvider, keyID string) { + // Skip the fetch entirely when the provider has disabled list_models via + // allowed_requests — every per-(provider,keyID) call would just bounce with + // "operation not allowed", wasting two goroutines and one bfCtx per attempt. + if s.Config != nil { + if pc, err := s.Config.GetProviderConfigRaw(provider); err == nil && pc != nil && + pc.CustomProviderConfig != nil && + !pc.CustomProviderConfig.IsOperationAllowed(schemas.ListModelsRequest) { + return + } + } + // One BifrostContext per goroutine. BifrostContext.SetValue mutates state + // in place, so the request-scoped metadata core sets during a routing pass + // (RequestID, FallbackIndex, span IDs, ...) would otherwise bleed between + // the filtered and unfiltered calls and conflate them in logs/billing. + newListModelsCtx := func() *schemas.BifrostContext { + c := schemas.NewBifrostContext(ctx, time.Now().Add(15*time.Second)) + c.SetValue(schemas.BifrostContextKeySkipPluginPipeline, true) + c.SetValue(schemas.BifrostContextKeyValidateKeys, true) + return c + } + + var keyIDPtr *string + if keyID != "" { + keyIDPtr = &keyID + } + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + bfCtx := newListModelsCtx() + defer bfCtx.Cancel() + resp, bfErr := s.Client.ListModelsRequest(bfCtx, &schemas.BifrostListModelsRequest{ + Provider: provider, + KeyID: keyIDPtr, + }) + if bfErr != nil { + logger.Warn("filtered list-models failed for provider %s key %s: %v: falling back onto the static datasheet", provider, keyID, bifrost.GetErrorMessage(bfErr)) + if len(bfErr.ExtraFields.KeyStatuses) > 0 && s.Config.ConfigStore != nil { + s.updateKeyStatus(ctx, bfErr.ExtraFields.KeyStatuses) + } + return + } + if resp == nil { + return + } + s.Config.ModelCatalog.UpsertLiveFromResponse(provider, keyID, false, resp) + if len(resp.KeyStatuses) > 0 && s.Config.ConfigStore != nil { + s.updateKeyStatus(ctx, resp.KeyStatuses) + } + }() + go func() { + defer wg.Done() + bfCtx := newListModelsCtx() + defer bfCtx.Cancel() + resp, bfErr := s.Client.ListModelsRequest(bfCtx, &schemas.BifrostListModelsRequest{ + Provider: provider, + KeyID: keyIDPtr, + Unfiltered: true, + }) + if bfErr != nil { + logger.Warn("unfiltered list-models failed for provider %s key %s: %v: falling back onto the static datasheet", provider, keyID, bifrost.GetErrorMessage(bfErr)) + return + } + if resp == nil { + return + } + s.Config.ModelCatalog.UpsertLiveFromResponse(provider, keyID, true, resp) + }() + wg.Wait() +} + +// ForceReloadPricing triggers an immediate pricing sync and resets the sync +// timer. No longer triggers a list-models refresh — pricing reload is now +// pricing-only. func (s *BifrostHTTPServer) ForceReloadPricing(ctx context.Context) error { if s.Config == nil { return fmt.Errorf("server config not initialized") @@ -948,12 +1053,13 @@ func (s *BifrostHTTPServer) ForceReloadPricing(ctx context.Context) error { if err := s.Config.ModelCatalog.ForceReloadPricing(ctx); err != nil { return fmt.Errorf("failed to force reload pricing: %w", err) } - return s.populateModelPoolWithListModels(ctx) } return nil } -// ReloadPricingFromDBAndPopulateModelPool reloads the pricing from DB and populates the model pool +// ReloadPricingFromDBAndPopulateModelPool reloads the pricing from DB. The +// list-models refresh that used to follow is gone — pricing reload is now +// pricing-only. func (s *BifrostHTTPServer) ReloadPricingFromDBAndPopulateModelPool(ctx context.Context) error { if s.Config == nil { return fmt.Errorf("server config not initialized") @@ -962,7 +1068,6 @@ func (s *BifrostHTTPServer) ReloadPricingFromDBAndPopulateModelPool(ctx context. if err := s.Config.ModelCatalog.ReloadFromDB(ctx); err != nil { return fmt.Errorf("failed to reload pricing from DB: %w", err) } - return s.populateModelPoolWithListModels(ctx) } return nil } @@ -1074,6 +1179,15 @@ func (s *BifrostHTTPServer) GetPluginStatus(ctx context.Context) map[string]sche return s.Config.GetPluginStatus() } +// GetLoadedPluginNames returns the sanitized names of all currently loaded plugins, +// matching the names embedded in their trace span names. +func (s *BifrostHTTPServer) GetLoadedPluginNames() []string { + if s.Config == nil { + return []string{} + } + return s.Config.GetLoadedPluginNames() +} + // NormalizePluginConfig implements handlers.PluginsLoader. It looks up the plugin // by name in the ConfigMarshallers cache and calls MarshalConfigForStorage if found. // Returns nil, nil when the plugin is not loaded or does not implement ConfigMarshallerPlugin. @@ -1226,8 +1340,10 @@ func (s *BifrostHTTPServer) RegisterAPIRoutes(ctx context.Context, callbacks Ser // Initializing plugin specific handlers var loggingHandler *handlers.LoggingHandler loggerPlugin, _ := lib.FindPluginAs[*logging.LoggerPlugin](s.Config, logging.PluginName) + var govLogManager logging.LogManager if loggerPlugin != nil { loggingHandler = handlers.NewLoggingHandler(loggerPlugin.GetPluginLogManager(), s, s.Config) + govLogManager = loggerPlugin.GetPluginLogManager() } var governanceHandler *handlers.GovernanceHandler governancePluginName := governance.PluginName @@ -1236,7 +1352,7 @@ func (s *BifrostHTTPServer) RegisterAPIRoutes(ctx context.Context, callbacks Ser } governancePlugin, _ := lib.FindPluginAs[schemas.LLMPlugin](s.Config, governancePluginName) if governancePlugin != nil { - governanceHandler, err = handlers.NewGovernanceHandler(callbacks, s.Config.ConfigStore) + governanceHandler, err = handlers.NewGovernanceHandler(callbacks, s.Config.ConfigStore, govLogManager) if err != nil { return fmt.Errorf("failed to initialize governance handler: %v", err) } @@ -1394,6 +1510,29 @@ func (s *BifrostHTTPServer) PrepareCommonMiddlewares() []schemas.BifrostHTTPMidd } else { logger.Warn("prometheus plugin not found, skipping telemetry middleware") } + // OTel HTTP metrics (http_requests_total etc., pushed via OTLP). The otel plugin is + // resolved per request rather than captured here: a config reload swaps in a freshly + // constructed plugin instance, and a pointer captured at startup would keep recording + // against exporters whose meter provider has been shut down. + commonMiddlewares = append(commonMiddlewares, func(next fasthttp.RequestHandler) fasthttp.RequestHandler { + return func(ctx *fasthttp.RequestCtx) { + start := time.Now() + reqSize := float64(ctx.Request.Header.ContentLength()) + next(ctx) + otelPlugin, err := lib.FindPluginAs[*otel.OtelPlugin](s.Config, otel.PluginName) + if err != nil { + return + } + otelPlugin.RecordHTTPMetrics(ctx, + string(ctx.Path()), + string(ctx.Method()), + strconv.Itoa(ctx.Response.StatusCode()), + time.Since(start).Seconds(), + reqSize, + float64(ctx.Response.Header.ContentLength()), + ) + } + }) return commonMiddlewares } @@ -1516,54 +1655,23 @@ func (s *BifrostHTTPServer) Bootstrap(ctx context.Context) error { // Sync plugin execution order from config to core (defensive — Init receives sorted list, // but this ensures order consistency if the loading path changes in the future) s.Client.ReorderPlugins(s.Config.GetPluginOrder()) - // List all models and add to model catalog with per-provider status tracking + // Seed the catalog: push the initial keyconfig snapshot and fetch per-key + // live models for every provider concurrently. logger.Info("listing all models and adding to model catalog") if s.Config.ModelCatalog != nil { - // Fetching keys for all providers and allowed models first - // Based on allowed models we will set the data in the model catalog + snapshot := make(map[schemas.ModelProvider][]schemas.Key, len(s.Config.Providers)) + for provider, providerConfig := range s.Config.Providers { + snapshot[provider] = providerConfig.Keys + } + s.Config.ModelCatalog.ReplaceKeyConfig(snapshot) + var wg sync.WaitGroup for provider, providerConfig := range s.Config.Providers { wg.Add(1) - go func(provider schemas.ModelProvider, providerConfig configstore.ProviderConfig) { + go func(p schemas.ModelProvider, keys []schemas.Key) { defer wg.Done() - bfCtx := schemas.NewBifrostContext(ctx, time.Now().Add(15*time.Second)) - bfCtx.SetValue(schemas.BifrostContextKeySkipPluginPipeline, true) - defer bfCtx.Cancel() - - modelData, listModelsErr := s.Client.ListModelsRequest(bfCtx, &schemas.BifrostListModelsRequest{ - Provider: provider, - }) - if modelData != nil && len(modelData.KeyStatuses) > 0 && s.Config.ConfigStore != nil { - s.updateKeyStatus(ctx, modelData.KeyStatuses) - } - if listModelsErr != nil { - if len(listModelsErr.ExtraFields.KeyStatuses) > 0 && s.Config.ConfigStore != nil { - s.updateKeyStatus(ctx, listModelsErr.ExtraFields.KeyStatuses) - } - logger.Error("failed to list models for provider %s: %v: falling back onto the static datasheet", provider, bifrost.GetErrorMessage(listModelsErr)) - } - allowedModels := make([]schemas.Model, 0) - for _, key := range providerConfig.Keys { - if key.Models.IsUnrestricted() { - continue - } - for _, model := range key.Models { - allowedModels = append(allowedModels, schemas.Model{ - ID: string(provider) + "/" + model, - }) - } - } - s.Config.ModelCatalog.UpsertModelDataForProvider(provider, modelData, allowedModels) - unfilteredModelData, listModelsErr := s.Client.ListModelsRequest(bfCtx, &schemas.BifrostListModelsRequest{ - Provider: provider, - Unfiltered: true, - }) - if listModelsErr != nil { - logger.Error("failed to list unfiltered models for provider %s: %v: falling back onto the static datasheet", provider, bifrost.GetErrorMessage(listModelsErr)) - } else { - s.Config.ModelCatalog.UpsertUnfilteredModelDataForProvider(provider, unfilteredModelData) - } - }(provider, providerConfig) + s.RefreshLiveModelsForProvider(ctx, p, keys) + }(provider, providerConfig.Keys) } wg.Wait() } diff --git a/transports/changelog.md b/transports/changelog.md index 64cd4aea4c..9392b54dce 100644 --- a/transports/changelog.md +++ b/transports/changelog.md @@ -1,3 +1,4 @@ ## 🐞 Fixed -- **VK Budget Quota & Reload APIs** — The virtual key quota and reload (rotate) APIs now hydrate governance data (model configs and budgets) before returning, so budget information is accurate instead of missing or stale. Also added proper error handling when fetching model config during hydration. \ No newline at end of file +- **List Models Metadata Passthrough** — OpenAI-compatible `/v1/models` responses now preserve rich upstream model metadata at the top level while keeping Bifrost's normalized IDs, response envelope, and non-destructive pricing enrichment. +- **VK Budget Quota & Reload APIs** — The virtual key quota and reload (rotate) APIs now hydrate governance data (model configs and budgets) before returning, so budget information is accurate instead of missing or stale. Also added proper error handling when fetching model config during hydration. diff --git a/transports/config.schema.json b/transports/config.schema.json index a28081efb9..0f88aeb60a 100644 --- a/transports/config.schema.json +++ b/transports/config.schema.json @@ -198,7 +198,7 @@ }, "allow_direct_keys": { "type": "boolean", - "description": "Allow callers to bypass the registered key pool by supplying x-bf-direct-key: true and an Authorization header carrying the provider's raw API key.", + "description": "Allow callers to bypass the registered key pool by supplying x-bf-direct-key: true and the provider's raw API key in an Authorization (Bearer), x-api-key, or x-goog-api-key header.", "default": false }, "mcp_agent_depth": { @@ -746,6 +746,9 @@ "auth_config": { "$ref": "#/$defs/auth_config" }, + "complexity_analyzer_config": { + "$ref": "#/$defs/complexity_analyzer_config" + }, "model_configs": { "type": "array", "description": "Per-model rate limit and budget configurations", @@ -1679,11 +1682,20 @@ "description": "Name of the service to report to Datadog", "default": "bifrost" }, + "ml_app": { + "type": "string", + "description": "ML application name for LLM Observability grouping (defaults to service_name)" + }, "agent_addr": { "type": "string", - "description": "Address of the Datadog Agent for APM traces", + "description": "Address of the Datadog Agent for APM traces (agent mode only, can use env. prefix)", "default": "localhost:8126" }, + "dogstatsd_addr": { + "type": "string", + "description": "Address of the DogStatsD server for metrics (agent mode only, can use env. prefix)", + "default": "localhost:8125" + }, "env": { "type": "string", "description": "Environment tag (e.g., production, staging)" @@ -1697,14 +1709,170 @@ "additionalProperties": { "type": "string" }, - "description": "Additional tags to add to all traces and metrics" + "description": "Additional tags to add to all traces and metrics (values can use env. prefix)" + }, + "enable_metrics": { + "type": "boolean", + "description": "Enable metrics emission (default: true)", + "default": true }, "enable_traces": { "type": "boolean", "description": "Enable APM traces (default: true)", "default": true + }, + "enable_llm_obs": { + "type": "boolean", + "description": "Enable LLM Observability (default: true)", + "default": true + }, + "disable_content_logging": { + "type": "boolean", + "description": "Disable logging of message content to Datadog (default: false)", + "default": false + }, + "agentless": { + "type": "boolean", + "description": "Use agentless mode to send data directly to Datadog APIs (default: false)", + "default": false + }, + "api_key": { + "type": "string", + "description": "Datadog API key, required for agentless mode (can use env. prefix)" + }, + "site": { + "type": "string", + "description": "Datadog site/region (e.g., datadoghq.com, datadoghq.eu)", + "default": "datadoghq.com" + }, + "request_headers": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Request header names to capture as span tags" + }, + "plugin_span_filter": { + "$ref": "#/$defs/plugin_span_filter" } }, + "allOf": [ + { + "if": { + "properties": { + "agentless": { + "const": true + } + }, + "required": ["agentless"] + }, + "then": { + "required": ["api_key"] + } + } + ], + "additionalProperties": false + } + } + } + }, + { + "if": { + "properties": { + "name": { + "const": "bigquery" + } + } + }, + "then": { + "required": ["config"], + "properties": { + "config": { + "type": "object", + "description": "Configuration for the BigQuery traces plugin", + "properties": { + "project_id": { + "type": "string", + "description": "GCP project ID (required)" + }, + "dataset_id": { + "type": "string", + "description": "BigQuery dataset name", + "default": "bifrost_traces" + }, + "table_id": { + "type": "string", + "description": "BigQuery table name", + "default": "traces" + }, + "location": { + "type": "string", + "description": "BigQuery dataset location", + "default": "US" + }, + "service_account_key": { + "anyOf": [ + { "type": "string" }, + { + "type": "object", + "properties": { + "value": { "type": "string" }, + "env_var": { "type": "string" }, + "from_env": { "type": "boolean" } + }, + "additionalProperties": false + } + ], + "description": "Service account key JSON for authentication. Omit to use Application Default Credentials (ADC). Supports env var syntax: \"env.MY_VAR\"." + }, + "create_table_if_not_exists": { + "type": "boolean", + "description": "Auto-create the table if it does not exist", + "default": true + }, + "flush_interval_seconds": { + "type": "integer", + "description": "Interval between buffer flushes, in seconds", + "default": 5 + }, + "buffer_size": { + "type": "integer", + "description": "Max rows to buffer before flushing", + "default": 500 + }, + "custom_labels": { + "type": "object", + "additionalProperties": { + "anyOf": [ + { "type": "string" }, + { + "type": "object", + "properties": { + "value": { "type": "string" }, + "env_var": { "type": "string" }, + "from_env": { "type": "boolean" } + }, + "additionalProperties": false + } + ] + }, + "description": "Arbitrary key-value pairs stored as JSON in the labels column" + }, + "disable_content_logging": { + "type": "boolean", + "description": "Omit conversation history (input/output) columns", + "default": false + }, + "request_headers": { + "type": "array", + "items": { "type": "string" }, + "description": "Request-header name patterns (exact or wildcard like \"x-custom-*\") whose values are stored in the request_headers column" + }, + "plugin_span_filter": { + "$ref": "#/$defs/plugin_span_filter" + } + }, + "required": ["project_id"], "additionalProperties": false } } @@ -1758,9 +1926,9 @@ } ] }, - "otel_plugin_span_filter": { + "plugin_span_filter": { "type": "object", - "description": "Controls which plugin hook spans are exported to the OTEL collector. Omit to export all plugin spans.", + "description": "Controls which plugin hook spans this observability connector exports. Omit to export all plugin spans. Mode \"include\" exports only the listed plugins; mode \"exclude\" exports everything except them. Plugin names match the in the span name \"plugin..\".", "properties": { "mode": { "type": "string", @@ -1849,7 +2017,7 @@ "default": true }, "plugin_span_filter": { - "$ref": "#/$defs/otel_plugin_span_filter" + "$ref": "#/$defs/plugin_span_filter" } }, "allOf": [ @@ -1897,7 +2065,7 @@ "minItems": 1 }, "plugin_span_filter": { - "$ref": "#/$defs/otel_plugin_span_filter" + "$ref": "#/$defs/plugin_span_filter" } }, "required": ["profiles"], @@ -2109,7 +2277,7 @@ }, "key_ids": { "type": "array", - "description": "Key identifiers allowed for this provider config. Use [\"*\"] to allow all keys; empty array denies all (deny-by-default). In config.json, values are key names. Via the API, values are key UUIDs.", + "description": "Key identifiers allowed for this provider config. Use [\"*\"] to allow all keys. Semantics depend on version: v2 (default) empty = deny all; v1 empty = allow all. Values are the key IDs.", "items": { "type": "string" } @@ -2148,6 +2316,86 @@ }, "additionalProperties": false }, + "complexity_tier_boundaries": { + "type": "object", + "description": "Score thresholds used to classify complexity_tier values", + "properties": { + "simple_medium": { + "type": "number", + "exclusiveMinimum": 0, + "exclusiveMaximum": 1, + "description": "Scores below this threshold are SIMPLE" + }, + "medium_complex": { + "type": "number", + "exclusiveMinimum": 0, + "exclusiveMaximum": 1, + "description": "Scores below this threshold and at or above simple_medium are MEDIUM" + }, + "complex_reasoning": { + "type": "number", + "exclusiveMinimum": 0, + "exclusiveMaximum": 1, + "description": "Scores below this threshold and at or above medium_complex are COMPLEX" + } + }, + "required": ["simple_medium", "medium_complex", "complex_reasoning"], + "additionalProperties": false + }, + "complexity_analyzer_keywords": { + "type": "object", + "description": "User-editable keyword lists for complexity analysis", + "properties": { + "code_keywords": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + }, + "reasoning_keywords": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + }, + "technical_keywords": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + }, + "simple_keywords": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + } + }, + "required": ["code_keywords", "reasoning_keywords", "technical_keywords", "simple_keywords"], + "additionalProperties": false + }, + "complexity_analyzer_config": { + "type": "object", + "description": "Runtime configuration for complexity_tier CEL routing", + "properties": { + "tier_boundaries": { + "$ref": "#/$defs/complexity_tier_boundaries" + }, + "keywords": { + "$ref": "#/$defs/complexity_analyzer_keywords" + } + }, + "required": ["tier_boundaries", "keywords"], + "additionalProperties": false + }, "auth_config": { "type": "object", "description": "Authentication configuration. Deprecated: Use governance.auth_config instead.", @@ -2166,7 +2414,8 @@ }, "disable_auth_on_inference": { "type": "boolean", - "description": "Whether authentication is disabled on inference" + "deprecated": true, + "description": "Deprecated and ignored. Use client_config.enforce_auth_on_inference instead." } }, "additionalProperties": false @@ -2192,6 +2441,19 @@ "description": "Model parameters URL", "optional": true, "format": "uri" + }, + "mcp_library_url": { + "type": "string", + "description": "URL to a custom MCP server catalog. Leave empty to use the default Bifrost catalog.", + "optional": true, + "format": "uri" + }, + "mcp_library_sync_interval": { + "type": "integer", + "description": "MCP library sync interval in seconds. Default is 24 hours. Minimum is 3600 seconds (1 hour).", + "default": 86400, + "optional": true, + "minimum": 3600 } }, "additionalProperties": false @@ -2398,13 +2660,74 @@ "aliases": { "type": "object", "additionalProperties": { - "type": "string", - "minLength": 1 + "oneOf": [ + { + "type": "string", + "minLength": 1, + "description": "Legacy shape: a bare provider-specific identifier. Equivalent to {\"model_id\": \"\"}." + }, + { + "type": "object", + "properties": { + "model_id": { + "type": "string", + "minLength": 1, + "description": "Provider-specific identifier sent on the wire (deployment name, inference profile ID, fine-tuned model ID, etc.)." + }, + "model_name": { + "type": "string", + "description": "Canonical model name used for pricing, logging, and family inference." + }, + "model_family": { + "type": "string", + "enum": ["anthropic", "openai", "mistral", "cohere", "gemini", "nova", "titan"], + "description": "Underlying model family. Used by provider routing without substring-sniffing the wire model ID." + }, + "description": { + "type": "string" + }, + "region": { + "type": "string", + "description": "Per-alias region override (can use env. prefix)." + }, + "api_version": { + "type": "string", + "description": "Azure OpenAI api-version override for this alias." + }, + "anthropic_version": { + "type": "string", + "description": "Azure anthropic-version header override for Claude-on-Azure deployments." + }, + "endpoint": { + "type": "string", + "description": "Per-alias Azure endpoint override (can use env. prefix)." + }, + "project_id": { + "type": "string", + "description": "Per-alias Vertex project ID override (can use env. prefix)." + }, + "project_number": { + "type": "string", + "description": "Per-alias Vertex project number override (can use env. prefix)." + }, + "inference_profile_arn": { + "type": "string", + "description": "Per-alias Bedrock inference profile ARN (can use env. prefix)." + }, + "use_deployments_endpoint": { + "type": "boolean", + "description": "Replicate: use the deployments endpoint instead of the predictions endpoint for this alias." + } + }, + "required": ["model_id"], + "additionalProperties": false + } + ] }, "propertyNames": { "minLength": 1 }, - "description": "Model alias mappings: maps a model name to a provider-specific identifier (deployment name, inference profile ID, fine-tuned model ID, etc.)" + "description": "Model alias mappings: each entry maps a user-facing model name to either a bare provider identifier (legacy string shape) or an AliasConfig object carrying the wire identifier plus optional canonical name, family, and provider-specific overrides." } }, "required": ["name", "weight"] @@ -4393,6 +4716,13 @@ }, "rate_limit": { "$ref": "#/$defs/rate_limit_line" + }, + "key_ids": { + "type": "array", + "description": "Key IDs allowed for this provider config. Use [\"*\"] to allow all keys; empty array or omitted denies all keys. Specific IDs restrict access to those keys only.", + "items": { + "type": "string" + } } }, "required": ["provider_name"], diff --git a/transports/go.mod b/transports/go.mod index 1cd2004995..410fc40297 100644 --- a/transports/go.mod +++ b/transports/go.mod @@ -8,6 +8,8 @@ require ( github.com/bytedance/sonic v1.15.1 github.com/fasthttp/router v1.5.4 github.com/fasthttp/websocket v1.5.12 + github.com/go-git/go-billy/v5 v5.9.0 + github.com/go-git/go-git/v5 v5.19.1 github.com/google/pprof v0.0.0-20251213031049-b05bdaca462f github.com/google/uuid v1.6.0 github.com/klauspost/compress v1.18.6 @@ -18,6 +20,7 @@ require ( github.com/maximhq/bifrost/plugins/governance v1.5.18 github.com/maximhq/bifrost/plugins/logging v1.5.18 github.com/maximhq/bifrost/plugins/maxim v1.6.18 + github.com/maximhq/bifrost/plugins/modelcatalogresolver v0.0.0-20260531215024-856c9963e662 github.com/maximhq/bifrost/plugins/otel v1.2.18 github.com/maximhq/bifrost/plugins/prompts v1.0.18 github.com/maximhq/bifrost/plugins/semanticcache v1.5.18 @@ -36,13 +39,31 @@ require ( gorm.io/gorm v1.31.1 ) +require ( + dario.cat/mergo v1.0.2 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/ProtonMail/go-crypto v1.1.6 // indirect + github.com/cloudflare/circl v1.6.3 // indirect + github.com/cyphar/filepath-securejoin v0.6.1 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/pjbgf/sha1cd v0.6.0 // indirect + github.com/sergi/go-diff v1.4.0 // indirect + github.com/skeema/knownhosts v1.3.1 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect +) + require ( cel.dev/expr v0.25.1 // indirect cloud.google.com/go v0.123.0 // indirect cloud.google.com/go/auth v0.20.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect - cloud.google.com/go/iam v1.5.3 // indirect + cloud.google.com/go/iam v1.7.0 // indirect cloud.google.com/go/monitoring v1.24.3 // indirect cloud.google.com/go/storage v1.61.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect @@ -54,12 +75,12 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect - github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect + github.com/aws/aws-sdk-go-v2 v1.41.12 // indirect github.com/aws/aws-sdk-go-v2/config v1.32.11 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect @@ -71,7 +92,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect - github.com/aws/smithy-go v1.25.1 // indirect + github.com/aws/smithy-go v1.27.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/buger/jsonparser v1.1.2 // indirect @@ -131,7 +152,7 @@ require ( github.com/mailru/easyjson v0.9.1 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-sqlite3 v1.14.32 // indirect; indirect<<<<<<< HEAD + github.com/mattn/go-sqlite3 v1.14.32 // indirect github.com/maximhq/bifrost/plugins/mocker v1.5.18 // indirect github.com/maximhq/maxim-go v0.2.1 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect @@ -192,11 +213,11 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.23.0 // indirect golang.org/x/crypto v0.52.0 // indirect - golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect + golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect golang.org/x/net v0.55.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sys v0.45.0 // indirect - golang.org/x/text v0.37.0 // indirect + golang.org/x/text v0.37.0 golang.org/x/time v0.15.0 // indirect google.golang.org/api v0.282.0 // indirect google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect diff --git a/transports/go.sum b/transports/go.sum index a8f0c0f333..8a2a54e806 100644 --- a/transports/go.sum +++ b/transports/go.sum @@ -8,8 +8,8 @@ cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIi cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= -cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= +cloud.google.com/go/iam v1.7.0 h1:JD3zh0C6LHl16aCn5Akff0+GELdp1+4hmh6ndoFLl8U= +cloud.google.com/go/iam v1.7.0/go.mod h1:tetWZW1PD/m6vcuY2Zj/aU0eCHNPuxedbnbRTyKXvdY= cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHAjA= cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZCn6hSBtKak= cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8= @@ -20,6 +20,8 @@ cloud.google.com/go/storage v1.61.3 h1:VS//ZfBuPGDvakfD9xyPW1RGF1Vy3BWUoVZXgW1KM cloud.google.com/go/storage v1.61.3/go.mod h1:JtqK8BBB7TWv0HVGHubtUdzYYrakOQIsMLffZ2Z/HWk= cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U= cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 h1:JXg2dwJUmPB9JmtVmdEB16APJ7jurfbY5jnfXpJoRMc= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= @@ -40,15 +42,24 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw= +github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro= github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= -github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= -github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/aws/aws-sdk-go-v2 v1.41.12 h1:DIKX2c31ekm9RA2D9FBj1EWXx++9AdAqRw+e78Tq2Ck= +github.com/aws/aws-sdk-go-v2 v1.41.12/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= github.com/aws/aws-sdk-go-v2/config v1.32.11 h1:ftxI5sgz8jZkckuUHXfC/wMUc8u3fG1vQS0plr2F2Zs= @@ -57,10 +68,10 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8TH github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 h1:Xf2j7NdVcUKomlZ4iihOP4AZ3Fzlr8h4yKpXeP+OFPg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28/go.mod h1:O8cDo1dW63jU7ki//kRe1z+tLGcpnD1jrouitsQddDw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 h1:KqIfN9kpkKkcBqBbNpNGTIrXO6ExTUvFKvXkC+YAzVo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28/go.mod h1:uxtQiKvLtNS4iXVsH2McVD/ls8FKN/uUhe1hGxPjrw0= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 h1:clHU5fm//kWS1C2HgtgWxfQbFbx4b6rx+5jzhgX9HrI= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= @@ -83,8 +94,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6f github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= -github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= -github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8= +github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -106,11 +117,15 @@ github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1x github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= +github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= +github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -119,6 +134,10 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo= github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= +github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= @@ -135,6 +154,16 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= +github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.9.0 h1:jItGXszUDRtR/AlferWPTMN4j38BQ88XnXKbilmmBPA= +github.com/go-git/go-billy/v5 v5.9.0/go.mod h1:jCnQMLj9eUgGU7+ludSTYoZL/GGmii14RxKFj7ROgHw= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= +github.com/go-git/go-git/v5 v5.19.1 h1:nX27AnaU43/K5bKktKwgBmR9lawoYVe1Ckg0rgzzN00= +github.com/go-git/go-git/v5 v5.19.1/go.mod h1:Pb1v0c7/g8aGQJwx9Us09W85yGoyvSwuhEGMH7zjDKQ= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -195,6 +224,8 @@ github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlnd github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/cel-go v0.28.1 h1:YWIwi77J4xIsYUwAF/iIuS6haffzIHS8yWI8glSbLWM= @@ -229,19 +260,26 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jaswdr/faker/v2 v2.8.0 h1:3AxdXW9U7dJmWckh/P0YgRbNlCcVsTyrUNUnLVP9b3Q= github.com/jaswdr/faker/v2 v2.8.0/go.mod h1:jZq+qzNQr8/P+5fHd9t3txe2GNPnthrTfohtnJ7B+68= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= @@ -273,6 +311,8 @@ github.com/maximhq/bifrost/plugins/maxim v1.6.18 h1:0EfHmwBLbmrG9hwofdU41x5x+SCw github.com/maximhq/bifrost/plugins/maxim v1.6.18/go.mod h1:L5gE+GCGWLiSi1UljV6ZV5v5sf0YhxV1j+EvHQNhqts= github.com/maximhq/bifrost/plugins/mocker v1.5.18 h1:4HCqMfTcxzjO2nicAxWzIKyyfOKwiGAJqT3lVX32oKI= github.com/maximhq/bifrost/plugins/mocker v1.5.18/go.mod h1:zr9x3vsPDYmdOPnbQlqC3a6TEFLnwcCa2hb1Z9zq5SE= +github.com/maximhq/bifrost/plugins/modelcatalogresolver v0.0.0-20260531215024-856c9963e662 h1:RMa9QlP7IIPjCJ6w+XTI+68HzV303XiPFBWVf1risPQ= +github.com/maximhq/bifrost/plugins/modelcatalogresolver v0.0.0-20260531215024-856c9963e662/go.mod h1:AS5NErs/iuEUfBaeNjHbvdr3rRI5GezWmLh3xG52dY0= github.com/maximhq/bifrost/plugins/otel v1.2.18 h1:dBrB0P9RCpJ71p+z+JBNSJAF/ZskqC7GkvFa+TlFnu0= github.com/maximhq/bifrost/plugins/otel v1.2.18/go.mod h1:G/wM8Ks+tv6QkRd0QXK/lwveKYTLNGTCsykLLVy3xTc= github.com/maximhq/bifrost/plugins/prompts v1.0.18 h1:BBtD2h4nQvZ2ewKurE3mu+I9l5VO94ZpfitYceaT4r0= @@ -289,6 +329,8 @@ github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmt github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/pinecone-io/go-pinecone/v5 v5.3.0 h1:0YQlEtmXGWK/I8ztkOVM6PuBYgFJZhjSdb0ddU+bHPE= github.com/pinecone-io/go-pinecone/v5 v5.3.0/go.mod h1:6Fg85fcyvMUQFf9KW7zniN81kelSYvsjF+KPLdc1MGA= github.com/pion/datachannel v1.6.0 h1:XecBlj+cvsxhAMZWFfFcPyUaDZtd7IJvrXqlXD/53i0= @@ -325,6 +367,8 @@ github.com/pion/turn/v4 v4.1.4 h1:EU11yMXKIsK43FhcUnjLlrhE4nboHZq+TXBIi3QpcxQ= github.com/pion/turn/v4 v4.1.4/go.mod h1:ES1DXVFKnOhuDkqn9hn5VJlSWmZPaRJLyBXoOeO/BmQ= github.com/pion/webrtc/v4 v4.2.9 h1:DZIh1HAhPIL3RvwEDFsmL5hfPSLEpxsQk9/Jir2vkJE= github.com/pion/webrtc/v4 v4.2.9/go.mod h1:9EmLZve0H76eTzf8v2FmchZ6tcBXtDgpfTEu+drW6SY= +github.com/pjbgf/sha1cd v0.6.0 h1:3WJ8Wz8gvDz29quX1OcEmkAlUg9diU4GxJHqs0/XiwU= +github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -357,6 +401,11 @@ github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEV github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/savsgio/gotils v0.0.0-20250408102913-196191ec6287 h1:qIQ0tWF9vxGtkJa24bR+2i53WBCz1nW/Pc47oVYauC4= github.com/savsgio/gotils v0.0.0-20250408102913-196191ec6287/go.mod h1:sM7Mt7uEoCeFSCBM+qBrqvEo+/9vdmj19wzp3yzUhmg= +github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= +github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= +github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= @@ -368,7 +417,9 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -399,6 +450,8 @@ github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/ github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU= github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= @@ -443,26 +496,39 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/arch v0.23.0 h1:lKF64A2jF6Zd8L0knGltUnegD62JMFBiCPBmQpToHhg= golang.org/x/arch v0.23.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= -golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0= -golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= +golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.282.0 h1:WmJiSVqUnKqJCpJOx7YADbXaC+9DDsnGSfllFSj7R2I= @@ -478,8 +544,13 @@ google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zN google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/ui/app/_fallbacks/enterprise/components/api-keys/apiKeysIndexView.tsx b/ui/app/_fallbacks/enterprise/components/api-keys/apiKeysIndexView.tsx index c413b3dcdd..48300f1076 100644 --- a/ui/app/_fallbacks/enterprise/components/api-keys/apiKeysIndexView.tsx +++ b/ui/app/_fallbacks/enterprise/components/api-keys/apiKeysIndexView.tsx @@ -53,7 +53,7 @@ curl --location 'http://localhost:8080/v1/chat/completions' ); } - const isInferenceAuthDisabled = bifrostConfig?.auth_config?.disable_auth_on_inference ?? false; + const isInferenceAuthDisabled = !(bifrostConfig?.client_config?.enforce_auth_on_inference ?? false); return (

diff --git a/ui/app/_fallbacks/enterprise/components/load-balancer/loadBalancerSettingsView.tsx b/ui/app/_fallbacks/enterprise/components/load-balancer/loadBalancerSettingsView.tsx new file mode 100644 index 0000000000..c3760a460f --- /dev/null +++ b/ui/app/_fallbacks/enterprise/components/load-balancer/loadBalancerSettingsView.tsx @@ -0,0 +1,3 @@ +// On OSS, the adaptive routing settings page shows the same enterprise upsell as the +// adaptive routing dashboard — reuse that fallback rather than duplicating it. +export { default } from "../adaptive-routing/adaptiveRoutingView"; diff --git a/ui/app/workspace/adaptive-routing/layout.tsx b/ui/app/workspace/adaptive-routing/layout.tsx index 5a3ac61263..8fe95f9298 100644 --- a/ui/app/workspace/adaptive-routing/layout.tsx +++ b/ui/app/workspace/adaptive-routing/layout.tsx @@ -1,14 +1,16 @@ -import { createFileRoute } from "@tanstack/react-router"; +import { createFileRoute, Outlet, useChildMatches } from "@tanstack/react-router"; import { NoPermissionView } from "@/components/noPermissionView"; import { RbacOperation, RbacResource, useRbac } from "@enterprise/lib"; import AdaptiveRoutingPage from "./page"; function RouteComponent() { const hasAdaptiveRouterAccess = useRbac(RbacResource.AdaptiveRouter, RbacOperation.View); + const childMatches = useChildMatches(); if (!hasAdaptiveRouterAccess) { return ; } - return ; + // Render the dashboard at the base path; defer to child routes (e.g. /settings). + return childMatches.length === 0 ? : ; } export const Route = createFileRoute("/workspace/adaptive-routing")({ diff --git a/ui/app/workspace/adaptive-routing/settings/layout.tsx b/ui/app/workspace/adaptive-routing/settings/layout.tsx new file mode 100644 index 0000000000..1fcf9b9f7c --- /dev/null +++ b/ui/app/workspace/adaptive-routing/settings/layout.tsx @@ -0,0 +1,6 @@ +import { createFileRoute } from "@tanstack/react-router"; +import AdaptiveRoutingSettingsPage from "./page"; + +export const Route = createFileRoute("/workspace/adaptive-routing/settings")({ + component: AdaptiveRoutingSettingsPage, +}); diff --git a/ui/app/workspace/adaptive-routing/settings/page.tsx b/ui/app/workspace/adaptive-routing/settings/page.tsx new file mode 100644 index 0000000000..99731e4a05 --- /dev/null +++ b/ui/app/workspace/adaptive-routing/settings/page.tsx @@ -0,0 +1,9 @@ +import LoadBalancerSettingsView from "@enterprise/components/load-balancer/loadBalancerSettingsView"; + +export default function AdaptiveRoutingSettingsPage() { + return ( +
+ +
+ ); +} diff --git a/ui/app/workspace/complexity-router/layout.tsx b/ui/app/workspace/complexity-router/layout.tsx new file mode 100644 index 0000000000..bc11a82657 --- /dev/null +++ b/ui/app/workspace/complexity-router/layout.tsx @@ -0,0 +1,16 @@ +import { NoPermissionView } from "@/components/noPermissionView"; +import { RbacOperation, RbacResource, useRbac } from "@enterprise/lib"; +import { createFileRoute } from "@tanstack/react-router"; +import ComplexityRouterPage from "./page"; + +function RouteComponent() { + const hasRoutingRulesAccess = useRbac(RbacResource.RoutingRules, RbacOperation.View); + if (!hasRoutingRulesAccess) { + return ; + } + return ; +} + +export const Route = createFileRoute("/workspace/complexity-router")({ + component: RouteComponent, +}); diff --git a/ui/app/workspace/complexity-router/page.tsx b/ui/app/workspace/complexity-router/page.tsx new file mode 100644 index 0000000000..42025e8d4a --- /dev/null +++ b/ui/app/workspace/complexity-router/page.tsx @@ -0,0 +1,587 @@ +import FullPageLoader from "@/components/fullPageLoader"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alertDialog"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { ScrollArea } from "@/components/ui/scrollArea"; +import { TagInput } from "@/components/ui/tagInput"; +import { getErrorMessage } from "@/lib/store"; +import { + useGetComplexityAnalyzerConfigQuery, + useResetComplexityAnalyzerConfigMutation, + useUpdateComplexityAnalyzerConfigMutation, +} from "@/lib/store/apis/governanceApi"; +import { + AnalyzerConfig, + DEFAULT_TIER_BOUNDARIES, + KEYWORD_LIST_DEFINITIONS, + KeywordListKey, + TierBoundaries, +} from "@/lib/types/complexityRouter"; +import { cn } from "@/lib/utils"; +import { RbacOperation, RbacResource, useRbac } from "@enterprise/lib"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { ExternalLink, LoaderCircle, RotateCcw, Save } from "lucide-react"; +import { type ChangeEvent, type ClipboardEvent, type DragEvent, type KeyboardEvent, useEffect, useState } from "react"; +import { Controller, useForm } from "react-hook-form"; +import { toast } from "sonner"; +import { z } from "zod"; + +type TierBoundaryKey = keyof TierBoundaries; + +const COMPLEXITY_ROUTER_DOCS_URL = "https://docs.getbifrost.ai/features/governance/complexity-router"; + +// Four progressive shades of --primary: faintest → full +const P1 = "color-mix(in oklch, var(--primary) 30%, transparent)"; +const P2 = "color-mix(in oklch, var(--primary) 55%, transparent)"; +const P3 = "color-mix(in oklch, var(--primary) 75%, transparent)"; +const P4 = "var(--primary)"; + +const TIER_PALETTE = { + simple: { color: P1, name: "SIMPLE" }, + medium: { color: P2, name: "MEDIUM" }, + complex: { color: P3, name: "COMPLEX" }, + reasoning: { color: P4, name: "REASONING" }, +} as const; + +interface BoundaryFieldConfig { + key: TierBoundaryKey; + label: string; + description: string; + fromTier: string; + toTier: string; + fromColor: string; + toColor: string; +} + +const BOUNDARY_FIELDS: BoundaryFieldConfig[] = [ + { + key: "simple_medium", + label: "Simple → Medium", + description: "Scores at or below this are classified as SIMPLE.", + fromTier: "SIMPLE", + toTier: "MEDIUM", + fromColor: P1, + toColor: P2, + }, + { + key: "medium_complex", + label: "Medium → Complex", + description: "Scores above simple_medium and at or below this are MEDIUM.", + fromTier: "MEDIUM", + toTier: "COMPLEX", + fromColor: P2, + toColor: P3, + }, + { + key: "complex_reasoning", + label: "Complex → Reasoning", + description: "Scores above this are REASONING. Everything in between is COMPLEX.", + fromTier: "COMPLEX", + toTier: "REASONING", + fromColor: P3, + toColor: P4, + }, +]; + +const boundaryField = z.number({ error: "Enter a number between 0 and 1" }).gt(0, "Must be greater than 0").lt(1, "Must be less than 1"); + +const analyzerConfigSchema = z.object({ + tier_boundaries: z + .object({ + simple_medium: boundaryField, + medium_complex: boundaryField, + complex_reasoning: boundaryField, + }) + .superRefine((data, ctx) => { + if (Number.isFinite(data.medium_complex) && Number.isFinite(data.simple_medium) && data.medium_complex <= data.simple_medium) { + ctx.addIssue({ code: "custom", message: "Must be greater than Simple → Medium", path: ["medium_complex"] }); + } + if ( + Number.isFinite(data.complex_reasoning) && + Number.isFinite(data.medium_complex) && + data.complex_reasoning <= data.medium_complex + ) { + ctx.addIssue({ code: "custom", message: "Must be greater than Medium → Complex", path: ["complex_reasoning"] }); + } + }), + keywords: z.object({ + simple_keywords: z.array(z.string()).min(1, "Simple keywords cannot be empty"), + code_keywords: z.array(z.string()).min(1, "Code keywords cannot be empty"), + technical_keywords: z.array(z.string()).min(1, "Technical keywords cannot be empty"), + reasoning_keywords: z.array(z.string()).min(1, "Reasoning keywords cannot be empty"), + }), +}); + +const DEFAULT_FORM_VALUES: AnalyzerConfig = { + tier_boundaries: { ...DEFAULT_TIER_BOUNDARIES }, + keywords: { + code_keywords: [], + reasoning_keywords: [], + technical_keywords: [], + simple_keywords: [], + }, +}; + +function boundaryValueAsNumber(value: unknown): number { + let numericValue = Number.NaN; + if (typeof value === "number") { + numericValue = value; + } else if (typeof value === "string" && value.trim() !== "") { + numericValue = Number(value); + } + return Number.isFinite(numericValue) ? Math.max(0, numericValue) : Number.NaN; +} + +function finiteBoundaryValue(value: number | undefined, fallback: number) { + return typeof value === "number" && Number.isFinite(value) ? value : fallback; +} + +function clampUnit(value: number) { + return Math.min(1, Math.max(0, value)); +} + +function testIdPart(value: string) { + return value.replace(/_/g, "-"); +} + +function preventNegativeBoundaryKey(event: KeyboardEvent) { + if (event.key === "-") { + event.preventDefault(); + } +} + +function preventNegativeBoundaryPaste(event: ClipboardEvent) { + if (/^\s*-/.test(event.clipboardData.getData("text"))) { + event.preventDefault(); + } +} + +function preventNegativeBoundaryDrop(event: DragEvent) { + if (/^\s*-/.test(event.dataTransfer.getData("text"))) { + event.preventDefault(); + } +} + +function normalizeBoundaryInput(event: ChangeEvent) { + const { value } = event.currentTarget; + if (!/^\s*-/.test(value)) return; + + const numericValue = Number(value); + event.currentTarget.value = Number.isFinite(numericValue) ? "0" : ""; +} + +function TierSpectrumBar({ boundaries }: { boundaries: TierBoundaries }) { + const sm = clampUnit(finiteBoundaryValue(boundaries?.simple_medium, DEFAULT_TIER_BOUNDARIES.simple_medium)); + const mc = clampUnit(finiteBoundaryValue(boundaries?.medium_complex, DEFAULT_TIER_BOUNDARIES.medium_complex)); + const cr = clampUnit(finiteBoundaryValue(boundaries?.complex_reasoning, DEFAULT_TIER_BOUNDARIES.complex_reasoning)); + + const segments = [ + { tier: "SIMPLE", width: Math.max(0, sm * 100), color: TIER_PALETTE.simple.color }, + { tier: "MEDIUM", width: Math.max(0, (mc - sm) * 100), color: TIER_PALETTE.medium.color }, + { tier: "COMPLEX", width: Math.max(0, (cr - mc) * 100), color: TIER_PALETTE.complex.color }, + { tier: "REASONING", width: Math.max(0, (1 - cr) * 100), color: TIER_PALETTE.reasoning.color }, + ]; + + const markers = [ + { key: "simple-medium", pos: sm, value: sm.toFixed(2) }, + { key: "medium-complex", pos: mc, value: mc.toFixed(2) }, + { key: "complex-reasoning", pos: cr, value: cr.toFixed(2) }, + ]; + + return ( +
+
+ {segments.map(({ tier, width, color }) => ( +
+ {width > 7 && ( + + {tier} + + )} +
+ ))} + {/* Boundary dividers */} + {markers.map(({ key, pos }) => ( +
+ ))} +
+ {/* Axis labels */} +
+ 0 + {markers.map(({ key, pos, value }) => ( + + {value} + + ))} + 1 +
+
+ ); +} + +export default function ComplexityRouterPage() { + const canUpdate = useRbac(RbacResource.RoutingRules, RbacOperation.Update); + const { data, isLoading, isFetching, error, refetch } = useGetComplexityAnalyzerConfigQuery(); + const [updateConfig, { isLoading: isSaving }] = useUpdateComplexityAnalyzerConfigMutation(); + const [resetConfig, { isLoading: isResetting }] = useResetComplexityAnalyzerConfigMutation(); + + const [submitError, setSubmitError] = useState(null); + const [restoreDialogOpen, setRestoreDialogOpen] = useState(false); + + const { + register, + handleSubmit, + reset, + control, + watch, + formState: { errors, isDirty, isSubmitted }, + } = useForm({ + resolver: zodResolver(analyzerConfigSchema), + defaultValues: DEFAULT_FORM_VALUES, + mode: "onSubmit", + reValidateMode: "onChange", + }); + + const liveBoundaries = watch("tier_boundaries"); + + useEffect(() => { + if (!data || isDirty) return; + reset(data); + setSubmitError(null); + }, [data, isDirty, reset]); + + const handleDiscard = () => { + if (data) reset(data); + setSubmitError(null); + }; + + const handleRestoreDefaults = () => { + if (!canUpdate) return; + setSubmitError(null); + resetConfig() + .unwrap() + .then((defaults) => { + reset(defaults); + toast.success("Reset to defaults", { position: "top-right" }); + }) + .catch((err) => { + setSubmitError(getErrorMessage(err)); + }); + }; + + const onValid = (values: AnalyzerConfig) => { + if (!canUpdate) return; + setSubmitError(null); + updateConfig(values) + .unwrap() + .then((res) => { + reset(res); + toast.success("Configuration saved", { position: "top-right" }); + }) + .catch((err) => { + setSubmitError(getErrorMessage(err)); + }); + }; + + if (isLoading && !data) { + return ; + } + + if (error && !data) { + return ( +
+

{getErrorMessage(error)}

+ +
+ ); + } + + if (!data) { + return ( +
+

No complexity router configuration is available.

+ +
+ ); + } + + const boundaryErrors = errors.tier_boundaries; + const keywordErrors = errors.keywords; + const hasErrors = Boolean(boundaryErrors || keywordErrors); + + return ( + +
+ {/* ── Page header ── */} +
+
+

Complexity Router

+

+ Tune how incoming requests are classified into four tiers. Thresholds and keyword lists feed the{" "} + complexity_tier field that routing rules can target. +

+
+ +
+ + {/* ── Complexity Spectrum ── */} +
+
+

Complexity Spectrum

+
+ {Object.values(TIER_PALETTE).map(({ color, name }) => ( +
+
+ {name} +
+ ))} +
+
+ +
+ + {/* ── Tier Boundaries ── */} +
+

Tier Boundaries

+ +
+ {BOUNDARY_FIELDS.map(({ key, label, description, fromTier, toTier, fromColor, toColor }) => { + const fieldError = boundaryErrors?.[key]; + const inputId = `boundary-${key}`; + const errorId = `${inputId}-error`; + const { onChange, ...boundaryInputProps } = register(`tier_boundaries.${key}`, { + required: "Enter a number between 0 and 1", + setValueAs: boundaryValueAsNumber, + validate: (value) => { + if (!Number.isFinite(value)) return "Enter a number between 0 and 1"; + if (value <= 0) return "Must be greater than 0"; + if (value >= 1) return "Must be less than 1"; + const { simple_medium, medium_complex } = liveBoundaries; + if (key === "medium_complex" && Number.isFinite(simple_medium) && value <= simple_medium) { + return "Must be greater than Simple → Medium"; + } + if (key === "complex_reasoning" && Number.isFinite(medium_complex) && value <= medium_complex) { + return "Must be greater than Medium → Complex"; + } + return true; + }, + deps: + key === "simple_medium" + ? ["tier_boundaries.medium_complex"] + : key === "medium_complex" + ? ["tier_boundaries.complex_reasoning"] + : undefined, + }); + + return ( +
+ {/* Tier transition label */} +
+ + {fromTier} + + + + {toTier} + +
+ + + { + normalizeBoundaryInput(event); + onChange(event); + }} + aria-invalid={fieldError ? true : undefined} + aria-describedby={fieldError ? errorId : undefined} + className={cn( + "h-11 text-center text-lg font-mono font-medium", + fieldError && "border-destructive focus-visible:ring-destructive", + )} + {...boundaryInputProps} + /> + + {fieldError ? ( +

+ {fieldError.message} +

+ ) : ( +

{description}

+ )} +
+ ); + })} +
+
+ + {/* ── Keyword Lists ── */} +
+
+

Keyword Lists

+ + Lowercased and deduplicated on save. Each list requires at least one entry. + +
+ +
+ {KEYWORD_LIST_DEFINITIONS.map(({ key, label, description }) => { + const fieldError = keywordErrors?.[key as KeywordListKey]; + const errorId = `keywords-${key}-error`; + return ( +
+ (value.length > 0 ? true : `${label} cannot be empty`) }} + render={({ field }) => ( +
+
+ {label} + + {field.value.length} {field.value.length === 1 ? "entry" : "entries"} + +
+

{description}

+ + {fieldError && ( +

+ {fieldError.message} +

+ )} +
+ )} + /> +
+ ); + })} +
+
+ + {/* ── Submit error ── */} + {submitError && ( +
+ {submitError} +
+ )} + + {/* ── Action footer ── */} +
+ + + +
+ + + + + + Restore defaults + + This will reset all tier boundaries and keyword lists to the factory defaults. Your current configuration will be lost. This + action cannot be undone. + + + + setRestoreDialogOpen(false)} + disabled={isResetting} + > + Cancel + + { + setRestoreDialogOpen(false); + handleRestoreDefaults(); + }} + disabled={!canUpdate || isResetting} + > + Restore defaults + + + + + + ); +} diff --git a/ui/app/workspace/config/views/securityView.tsx b/ui/app/workspace/config/views/securityView.tsx index 648692e137..a7cbd2822d 100644 --- a/ui/app/workspace/config/views/securityView.tsx +++ b/ui/app/workspace/config/views/securityView.tsx @@ -13,8 +13,7 @@ import { parseArrayFromText } from "@/lib/utils/array"; import { validateOrigins } from "@/lib/utils/validation"; import { RbacOperation, RbacResource, useRbac } from "@enterprise/lib"; import { useGetAuthTypeQuery } from "@enterprise/lib/store/apis/scimApi"; -import { Link } from "@tanstack/react-router"; -import { AlertTriangle, Info, Loader2 } from "lucide-react"; +import { AlertTriangle, Loader2 } from "lucide-react"; import { useCallback, useEffect, useMemo, useState } from "react"; import { toast } from "sonner"; @@ -43,7 +42,6 @@ export default function SecurityView() { admin_username: { value: "", env_var: "", from_env: false }, admin_password: { value: "", env_var: "", from_env: false }, is_enabled: false, - disable_auth_on_inference: true, }); useEffect(() => { @@ -80,10 +78,7 @@ export default function SecurityView() { authConfig.admin_password?.env_var !== bifrostConfig?.auth_config?.admin_password?.env_var || authConfig.admin_password?.from_env !== bifrostConfig?.auth_config?.admin_password?.from_env; const authChanged = showPasswordSection - ? authConfig.is_enabled !== bifrostConfig?.auth_config?.is_enabled || - usernameChanged || - passwordChanged || - authConfig.disable_auth_on_inference !== bifrostConfig?.auth_config?.disable_auth_on_inference + ? authConfig.is_enabled !== bifrostConfig?.auth_config?.is_enabled || usernameChanged || passwordChanged : false; const localRequired = localConfig.required_headers?.slice().sort().join(","); @@ -144,10 +139,6 @@ export default function SecurityView() { setAuthConfig((prev) => ({ ...prev, is_enabled: checked })); }, []); - const handleDisableAuthOnInferenceToggle = useCallback((checked: boolean) => { - setAuthConfig((prev) => ({ ...prev, disable_auth_on_inference: checked })); - }, []); - const handleAuthFieldChange = useCallback((field: "admin_username" | "admin_password", value: EnvVar) => { setAuthConfig((prev) => ({ ...prev, [field]: value })); }, []); @@ -187,25 +178,6 @@ export default function SecurityView() {
- {authConfig.is_enabled && !authConfig.disable_auth_on_inference && ( - - - - You will need to use Basic Auth for all your inference calls (including MCP tool execution). You can disable it below. Check{" "} - - API Keys - - - - )} - {authConfig.is_enabled && (authConfig.disable_auth_on_inference ?? true) && ( - - - - Authentication is disabled for inference calls. Only dashboard, admin API and MCP tool execution calls require authentication. - - - )} {/* Password Protect the Dashboard */} {IS_ENTERPRISE && authTypeLoading ? (
@@ -260,26 +232,6 @@ export default function SecurityView() { onChange={(value) => handleAuthFieldChange("admin_password", value)} />
- {authConfig.is_enabled && ( -
-
- -

- When enabled, inference API calls (chat completions, embeddings, etc.) will not require authentication. Dashboard - and admin API calls will still require authentication. -

-
- -
- )}
diff --git a/ui/app/workspace/custom-pricing/overrides/pricingOverrideSheet.tsx b/ui/app/workspace/custom-pricing/overrides/pricingOverrideSheet.tsx index 1f0801ade1..4e4fb5ee70 100644 --- a/ui/app/workspace/custom-pricing/overrides/pricingOverrideSheet.tsx +++ b/ui/app/workspace/custom-pricing/overrides/pricingOverrideSheet.tsx @@ -90,6 +90,8 @@ export const PRICING_FIELDS = [ { key: "output_cost_per_token_priority", label: "Output / token (priority)", group: "chat", requestTypeGroups: ["chat"] }, { key: "input_cost_per_token_flex", label: "Input / token (flex)", group: "chat", requestTypeGroups: ["chat"] }, { key: "output_cost_per_token_flex", label: "Output / token (flex)", group: "chat", requestTypeGroups: ["chat"] }, + { key: "input_cost_per_token_fast", label: "Input / token (fast)", group: "chat", requestTypeGroups: ["chat"] }, + { key: "output_cost_per_token_fast", label: "Output / token (fast)", group: "chat", requestTypeGroups: ["chat"] }, { key: "input_cost_per_token_above_128k_tokens", label: "Input / token (>128k)", diff --git a/ui/app/workspace/logs/sheets/logDetailView.tsx b/ui/app/workspace/logs/sheets/logDetailView.tsx index 67caf74244..60a2df2acc 100644 --- a/ui/app/workspace/logs/sheets/logDetailView.tsx +++ b/ui/app/workspace/logs/sheets/logDetailView.tsx @@ -1092,10 +1092,10 @@ export function LogDetailView({ {log.routing_engines_used.map((engine) => (
- {RoutingEngineUsedIcons[engine as keyof typeof RoutingEngineUsedIcons]?.()} + {RoutingEngineUsedIcons[engine as keyof typeof RoutingEngineUsedIcons]?.({ className: "h-3.5 w-3.5" })} {RoutingEngineUsedLabels[engine as keyof typeof RoutingEngineUsedLabels] ?? engine}
diff --git a/ui/app/workspace/mcp-registry/layout.tsx b/ui/app/workspace/mcp-registry/layout.tsx index 13ca9d881e..cc3f45cef8 100644 --- a/ui/app/workspace/mcp-registry/layout.tsx +++ b/ui/app/workspace/mcp-registry/layout.tsx @@ -1,16 +1,17 @@ -import { createFileRoute } from "@tanstack/react-router"; +import { createFileRoute, Outlet, useChildMatches } from "@tanstack/react-router"; import { NoPermissionView } from "@/components/noPermissionView"; import { RbacOperation, RbacResource, useRbac } from "@enterprise/lib"; import MCPServersPage from "./page"; function RouteComponent() { const hasMCPGatewayAccess = useRbac(RbacResource.MCPGateway, RbacOperation.View); + const childMatches = useChildMatches(); if (!hasMCPGatewayAccess) { return ; } - return ; + return childMatches.length === 0 ? : ; } export const Route = createFileRoute("/workspace/mcp-registry")({ component: RouteComponent, -}); \ No newline at end of file +}); diff --git a/ui/app/workspace/mcp-registry/library/layout.tsx b/ui/app/workspace/mcp-registry/library/layout.tsx new file mode 100644 index 0000000000..2ac25465c2 --- /dev/null +++ b/ui/app/workspace/mcp-registry/library/layout.tsx @@ -0,0 +1,16 @@ +import { NoPermissionView } from "@/components/noPermissionView"; +import { RbacOperation, RbacResource, useRbac } from "@enterprise/lib"; +import { createFileRoute } from "@tanstack/react-router"; +import MCPLibraryPage from "./page"; + +function RouteComponent() { + const hasMCPGatewayAccess = useRbac(RbacResource.MCPGateway, RbacOperation.View); + if (!hasMCPGatewayAccess) { + return ; + } + return ; +} + +export const Route = createFileRoute("/workspace/mcp-registry/library")({ + component: RouteComponent, +}); \ No newline at end of file diff --git a/ui/app/workspace/mcp-registry/library/page.tsx b/ui/app/workspace/mcp-registry/library/page.tsx new file mode 100644 index 0000000000..9c630c6986 --- /dev/null +++ b/ui/app/workspace/mcp-registry/library/page.tsx @@ -0,0 +1,374 @@ +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { ScrollArea } from "@/components/ui/scrollArea"; +import { useToast } from "@/hooks/use-toast"; +import { useDebouncedValue } from "@/hooks/useDebounce"; +import { parseAsSafeString } from "@/lib/queryParamsParser"; +import { getErrorMessage, useGetMCPClientsQuery, useGetMCPLibraryQuery } from "@/lib/store"; +import type { MCPLibraryEntry } from "@/lib/types/mcp"; +import { cn } from "@/lib/utils"; +import { RbacOperation, RbacResource, useRbac } from "@enterprise/lib"; +import { ChevronLeft, ChevronRight, LayoutGrid, Library, List, Plus, Search, Settings } from "lucide-react"; +import { parseAsArrayOf, parseAsInteger, parseAsString, useQueryStates } from "nuqs"; +import { useCallback, useEffect, useMemo, useState } from "react"; +import { MCPLibraryAddServerSheet } from "./views/mcpLibraryAddServerSheet"; +import { MCPLibraryFilterSidebar, type MCPLibraryFilters } from "./views/mcpLibraryFilterSidebar"; +import { MCPLibraryInstallSheet, sanitizeServerName } from "./views/mcpLibraryInstallSheet"; +import { MCPLibraryServerCard, MCPLibraryServerCardSkeleton } from "./views/mcpLibraryServerCard"; +import { MCPLibraryServersTable, MCPLibraryServersTableSkeleton } from "./views/mcpLibraryServersTable"; +import { MCPLibrarySettingsSheet } from "./views/mcpLibrarySettingsSheet"; + +const PAGE_SIZE = 24; +const VIEW_MODE_STORAGE_KEY = "mcp-library-view-mode"; +type MCPLibraryViewMode = "grid" | "table"; + +function getInitialViewMode(): MCPLibraryViewMode { + if (typeof window === "undefined") return "table"; + try { + const savedViewMode = window.localStorage.getItem(VIEW_MODE_STORAGE_KEY); + return savedViewMode === "grid" || savedViewMode === "table" ? savedViewMode : "table"; + } catch { + return "table"; + } +} + +export default function MCPLibraryPage() { + const hasCreateMCPClientAccess = useRbac(RbacResource.MCPGateway, RbacOperation.Create); + const hasDeleteMCPLibraryAccess = useRbac(RbacResource.MCPGateway, RbacOperation.Delete); + const hasSettingsAccess = useRbac(RbacResource.Settings, RbacOperation.Update); + const [selectedServer, setSelectedServer] = useState(null); + const [settingsOpen, setSettingsOpen] = useState(false); + const [addServerOpen, setAddServerOpen] = useState(false); + const [viewMode, setViewMode] = useState(getInitialViewMode); + const { toast } = useToast(); + + // URL state management with nuqs — search, filters, pagination all in query params + const [urlState, setUrlState] = useQueryStates( + { + search: parseAsSafeString.withDefault(""), + categories: parseAsArrayOf(parseAsString).withDefault([]), + connection_types: parseAsArrayOf(parseAsString).withDefault([]), + auth_types: parseAsArrayOf(parseAsString).withDefault([]), + tags: parseAsArrayOf(parseAsString).withDefault([]), + offset: parseAsInteger.withDefault(0), + }, + // Live search/filter changes use replace (don't pollute history per keystroke); + // pagination opts into push per-call so back/forward steps by page. + { history: "replace" }, + ); + + const debouncedSearch = useDebouncedValue(urlState.search, 300); + + // Derive filters object for the sidebar + const filters: MCPLibraryFilters = useMemo( + () => ({ + categories: urlState.categories, + connection_types: urlState.connection_types, + auth_types: urlState.auth_types, + tags: urlState.tags, + }), + [urlState.categories, urlState.connection_types, urlState.auth_types, urlState.tags], + ); + + const setFilters = useCallback( + (newFilters: MCPLibraryFilters) => { + setUrlState({ + categories: newFilters.categories, + connection_types: newFilters.connection_types, + auth_types: newFilters.auth_types, + tags: newFilters.tags, + offset: 0, + }); + }, + [setUrlState], + ); + + const queryParams = useMemo( + () => ({ + search: debouncedSearch || undefined, + category: filters.categories.length > 0 ? filters.categories.join(",") : undefined, + connection_type: filters.connection_types.length > 0 ? filters.connection_types.join(",") : undefined, + auth_type: filters.auth_types.length > 0 ? filters.auth_types.join(",") : undefined, + tags: filters.tags.length > 0 ? filters.tags.join(",") : undefined, + limit: PAGE_SIZE, + offset: urlState.offset, + }), + [debouncedSearch, filters, urlState.offset], + ); + + const { data: libraryData, error: libraryError, isFetching, refetch } = useGetMCPLibraryQuery(queryParams); + + const servers = useMemo(() => libraryData?.servers || [], [libraryData?.servers]); + const totalCount = libraryData?.total_count || 0; + + // Installed-detection: match on connection_url or name (case-insensitive) + const { data: mcpClientsData, error: mcpClientsError } = useGetMCPClientsQuery({ limit: 100, offset: 0 }); + + useEffect(() => { + if (!libraryError && !mcpClientsError) return; + const err = libraryError || mcpClientsError; + if (!err) return; + const message = getErrorMessage(err); + if (message.toLowerCase().includes("mcp is not configured in this bifrost instance")) return; + toast({ title: "Error", description: message, variant: "destructive" }); + }, [libraryError, mcpClientsError, toast]); + + const installedServerSlugs = useMemo(() => { + const clients = mcpClientsData?.clients || []; + return new Set( + servers + .filter((server) => + clients.some((client) => { + const connectionString = client.config.connection_string; + const connectionUrl = connectionString?.from_env ? connectionString.env_var : connectionString?.value; + return ( + (server.connection_url && connectionUrl === server.connection_url) || + client.config.name.toLowerCase() === sanitizeServerName(server.name).toLowerCase() + ); + }), + ) + .map((server) => server.slug), + ); + }, [mcpClientsData?.clients, servers]); + + const handleInstalled = useCallback(async () => { + await refetch(); + }, [refetch]); + + const handleViewModeChange = useCallback((mode: MCPLibraryViewMode) => { + setViewMode(mode); + try { + window.localStorage.setItem(VIEW_MODE_STORAGE_KEY, mode); + } catch { + // Keep the in-memory preference when browser storage is unavailable. + } + }, []); + + // Pagination + const totalPages = Math.max(1, Math.ceil(totalCount / PAGE_SIZE)); + const currentPage = Math.floor(urlState.offset / PAGE_SIZE) + 1; + + const hasActiveFilters = + filters.categories.length > 0 || filters.connection_types.length > 0 || filters.auth_types.length > 0 || filters.tags.length > 0; + const isCatalogEmpty = !isFetching && totalCount === 0 && !debouncedSearch && !hasActiveFilters; + + return ( +
+
+ {/* Sidebar Filters */} + + + {/* Main Content */} +
+
+ {/* Header */} +
+
+

MCP Server Library

+

Browse and install MCP servers from the synced catalog.

+
+
+ {hasCreateMCPClientAccess && ( + + )} + {hasSettingsAccess && ( + + )} +
+
+ + {/* Search */} + {!isCatalogEmpty && ( +
+
+ + setUrlState({ search: e.target.value, offset: 0 })} + placeholder="Search servers..." + className="h-9 pl-9" + data-testid="mcp-library-search-input" + /> +
+
+ + +
+
+ )} +
+ + {/* Loading skeletons */} + {isFetching && servers.length === 0 ? ( + viewMode === "grid" ? ( + +
+ {Array.from({ length: 6 }).map((_, i) => ( + // biome-ignore lint/suspicious/noArrayIndexKey: static skeleton placeholders have no stable id + + ))} +
+
+ ) : ( + + ) + ) : servers.length === 0 ? ( +
+
+ +
+
+

+ {isCatalogEmpty ? "No synced servers yet" : "No servers found"} +

+
+ {isCatalogEmpty + ? "Configure the library sync source in Settings to populate this catalog." + : "Try adjusting your search or filters."} +
+ {isCatalogEmpty && hasSettingsAccess && ( +
+ +
+ )} +
+
+ ) : ( + <> + {viewMode === "grid" ? ( + +
+ {servers.map((server) => { + const isInstalled = installedServerSlugs.has(server.slug); + return ( + + ); + })} +
+
+ ) : ( + + )} + + {/* Pagination */} + {totalCount > 0 && ( +
+
+ {(urlState.offset + 1).toLocaleString()}-{Math.min(urlState.offset + PAGE_SIZE, totalCount).toLocaleString()} of{" "} + {totalCount.toLocaleString()} entries +
+ +
+ + +
+ Page + {currentPage} + of {totalPages} +
+ + +
+
+ )} + + )} +
+
+
+
+ + {/* Install sheet */} + {selectedServer && ( + setSelectedServer(null)} + onInstalled={handleInstalled} + /> + )} + + {/* Settings sheet */} + setSettingsOpen(false)} /> + + {/* Add Server sheet */} + setAddServerOpen(false)} /> +
+ ); +} \ No newline at end of file diff --git a/ui/app/workspace/mcp-registry/library/views/mcpLibraryAddServerSheet.tsx b/ui/app/workspace/mcp-registry/library/views/mcpLibraryAddServerSheet.tsx new file mode 100644 index 0000000000..8a4a497141 --- /dev/null +++ b/ui/app/workspace/mcp-registry/library/views/mcpLibraryAddServerSheet.tsx @@ -0,0 +1,293 @@ +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Sheet, SheetContent, SheetDescription, SheetHeader, SheetTitle } from "@/components/ui/sheet"; +import { Textarea } from "@/components/ui/textarea"; +import { getErrorMessage, useCreateMCPLibraryEntryMutation } from "@/lib/store"; +import type { CreateMCPLibraryEntryRequest, MCPAuthType, MCPConnectionType } from "@/lib/types/mcp"; +import { useEffect } from "react"; +import { useForm } from "react-hook-form"; +import { toast } from "sonner"; + +interface MCPLibraryAddServerFormData { + name: string; + description: string; + category: string; + connection_type: MCPConnectionType; + connection_url: string; + command: string; + args: string; + envs: string; + auth_type: MCPAuthType; + required_header_keys: string; + icon_url: string; + docs_url: string; + tags: string; +} + +interface MCPLibraryAddServerSheetProps { + open: boolean; + onClose: () => void; +} + +const DEFAULTS: MCPLibraryAddServerFormData = { + name: "", + description: "", + category: "", + connection_type: "http", + connection_url: "", + command: "", + args: "", + envs: "", + auth_type: "none", + required_header_keys: "", + icon_url: "", + docs_url: "", + tags: "", +}; + +// Split a comma/newline-separated string into a trimmed, non-empty list. +function parseList(text: string): string[] { + return text + .split(/[\n,]/) + .map((s) => s.trim()) + .filter(Boolean); +} + +export function MCPLibraryAddServerSheet({ open, onClose }: MCPLibraryAddServerSheetProps) { + const [createEntry, { isLoading }] = useCreateMCPLibraryEntryMutation(); + + const { + register, + handleSubmit, + watch, + setValue, + reset, + formState: { errors }, + } = useForm({ defaultValues: DEFAULTS }); + + useEffect(() => { + if (open) reset(DEFAULTS); + }, [open, reset]); + + const connectionType = watch("connection_type"); + const authType = watch("auth_type"); + const isStdio = connectionType === "stdio"; + const needsHeaderKeys = authType === "headers" || authType === "per_user_headers"; + + const onSubmit = async (data: MCPLibraryAddServerFormData) => { + const tags = parseList(data.tags); + const payload: CreateMCPLibraryEntryRequest = { + name: data.name.trim(), + description: data.description.trim() || undefined, + category: data.category.trim() || undefined, + connection_type: data.connection_type, + auth_type: data.auth_type, + icon_url: data.icon_url.trim() || undefined, + docs_url: data.docs_url.trim() || undefined, + tags: tags.length ? tags : undefined, + }; + + if (isStdio) { + payload.stdio_config = { + command: data.command.trim(), + args: parseList(data.args), + envs: parseList(data.envs), + }; + } else { + payload.connection_url = data.connection_url.trim(); + } + + if (needsHeaderKeys) { + payload.required_header_keys = parseList(data.required_header_keys); + } + + try { + await createEntry(payload).unwrap(); + toast.success("MCP server published to the library."); + onClose(); + } catch (error) { + toast.error(getErrorMessage(error)); + } + }; + + return ( + !sheetOpen && onClose()}> + + + Add MCP Server + This MCP server will be available org-wide for members to discover, install, and use. + + +
+
+ {/* Name */} +
+ + v.trim().length > 0 || "Name is required", + })} + /> + {errors.name &&

{errors.name.message}

} +
+ + {/* Description */} +
+ +