From 5e6be08f3850425cae2ddfabee0bd534d6604b7e Mon Sep 17 00:00:00 2001 From: nachiketb Date: Mon, 10 Aug 2026 15:29:32 -0700 Subject: [PATCH] refactor(python): remove deprecated server stack Signed-off-by: nachiketb --- .../switchyard-stage-router-scorer/SKILL.md | 97 - .github/ISSUE_TEMPLATE/bug_report.md | 2 +- .github/workflows/ci.yml | 7 +- .github/workflows/perf.yml | 37 +- AGENTS.md | 156 +- CHANGELOG.md | 9 + Cargo.lock | 26 - Cargo.toml | 2 - INSTALLATION.md | 197 +- README.md | 2 +- benchmark/README.md | 17 - benchmark/score_staged_run.py | 283 --- crates/libsy-llm-client/src/client.rs | 6 +- crates/libsy-llm-client/src/error.rs | 6 +- crates/switchyard-components/Cargo.toml | 32 - .../src/backends/anthropic.rs | 684 ------ .../src/backends/common.rs | 192 -- .../src/backends/context_overflow.rs | 92 - .../switchyard-components/src/backends/mod.rs | 18 - .../src/backends/multi.rs | 330 --- .../src/backends/openai.rs | 778 ------- .../src/backends/selection.rs | 92 - .../src/backends/stats.rs | 88 - .../src/contracts/backend.rs | 145 -- .../src/contracts/context.rs | 216 -- .../src/contracts/error.rs | 103 - .../src/contracts/ids.rs | 111 - .../src/contracts/mod.rs | 18 - .../src/contracts/roles.rs | 30 - .../src/contracts/types.rs | 326 --- .../src/dimension_collector/mod.rs | 18 - .../dimension_collector/response/checks.rs | 361 --- .../src/dimension_collector/response/mod.rs | 128 - .../src/dimension_collector/tool_signals.rs | 52 - crates/switchyard-components/src/lib.rs | 36 - .../request_processors/dimension_collector.rs | 78 - .../src/request_processors/mod.rs | 12 - .../src/request_processors/random_routing.rs | 172 -- .../src/request_processors/stats.rs | 38 - .../src/response_processors/mod.rs | 10 - .../response_processors/response_signals.rs | 98 - .../src/response_processors/stats.rs | 249 -- .../switchyard-components/src/stage_router.rs | 15 - .../src/stats/accumulator.rs | 792 ------- .../src/stats/cache_eligibility.rs | 164 -- .../src/stats/context.rs | 72 - .../switchyard-components/src/stats/cost.rs | 269 --- crates/switchyard-components/src/stats/mod.rs | 25 - .../switchyard-components/src/stats/usage.rs | 221 -- crates/switchyard-components/src/telemetry.rs | 97 - .../tests/adversarial_multi_llm_backend.rs | 1268 ---------- .../tests/adversarial_native_backends.rs | 1160 --------- .../tests/adversarial_random_routing.rs | 209 -- .../switchyard-components/tests/contracts.rs | 186 -- .../tests/stats_accumulator.rs | 789 ------- .../tests/stats_processors.rs | 838 ------- .../tests/stats_usage_shapes.rs | 241 -- .../tests/support/config.rs | 67 - .../tests/support/mod.rs | 289 --- crates/switchyard-py/Cargo.toml | 5 - .../switchyard-py/src/component_bindings.rs | 25 - .../src/component_bindings/backends.rs | 350 --- .../src/component_bindings/config.rs | 533 ----- .../component_bindings/dimension_collector.rs | 398 ---- .../component_bindings/request_processors.rs | 62 - .../component_bindings/response_processors.rs | 79 - .../src/component_bindings/stage_router.rs | 154 -- .../src/component_bindings/stats.rs | 241 -- crates/switchyard-py/src/errors.rs | 205 +- crates/switchyard-py/src/interop.rs | 20 - crates/switchyard-py/src/interop/context.rs | 361 --- crates/switchyard-py/src/interop/request.rs | 70 - crates/switchyard-py/src/interop/response.rs | 613 ----- crates/switchyard-py/src/interop/roles.rs | 119 - crates/switchyard-py/src/interop/subagent.rs | 51 - crates/switchyard-py/src/lib.rs | 6 - crates/switchyard-py/src/libsy_bindings.rs | 18 +- crates/switchyard-py/src/py_serde.rs | 11 - crates/switchyard-py/src/translation.rs | 123 - crates/switchyard-server/README.md | 7 +- docs/cli_reference.md | 8 +- docs/getting_started.md | 7 +- docs/internal/metrics_reference.md | 3 +- .../stage_router_routing.md | 29 +- examples/minimal.py | 62 - examples/route.yaml | 12 - examples/utils.py | 97 - pyproject.toml | 60 +- switchyard/__init__.py | 138 +- switchyard/cli/command_utils.py | 11 +- .../cli/launchers/claude_code_launcher.py | 2 +- .../cli/launchers/codex_cli_launcher.py | 2 +- .../{lib => cli/launchers}/cost_estimator.py | 5 +- switchyard/cli/launchers/launcher_runtime.py | 54 +- switchyard/cli/launchers/live_stats_footer.py | 32 +- switchyard/cli/launchers/openclaw_launcher.py | 2 +- switchyard/cli/launchers/session_summary.py | 2 +- .../{server => cli/launchers}/shell_tui.py | 0 switchyard/cli/model_catalog/__init__.py | 5 - .../cli/model_catalog/model_discovery.py | 50 - switchyard/cli/route_bundle.py | 231 -- switchyard/cli/switchyard_cli.py | 85 +- switchyard/lib/__init__.py | 15 - switchyard/lib/backends/__init__.py | 30 - .../backends/anthropic_native_llm_backend.py | 8 - .../lib/backends/backend_format_resolver.py | 425 ---- switchyard/lib/backends/llm_target.py | 106 - switchyard/lib/backends/multi_llm_backend.py | 54 - switchyard/lib/backends/openai_llm_backend.py | 8 - .../lib/backends/openai_native_backend.py | 8 - switchyard/lib/backends/stats_llm_backend.py | 8 - switchyard/lib/chat_request/__init__.py | 17 - switchyard/lib/chat_request/anthropic.py | 12 - switchyard/lib/chat_request/base.py | 14 - switchyard/lib/chat_request/openai_chat.py | 12 - .../lib/chat_request/openai_responses.py | 12 - switchyard/lib/chat_response/__init__.py | 39 - switchyard/lib/chat_response/anthropic.py | 19 - switchyard/lib/chat_response/base.py | 14 - switchyard/lib/chat_response/openai_chat.py | 15 - .../lib/chat_response/openai_responses.py | 19 - .../streaming_response_accumulator.py | 752 ------ switchyard/lib/conversation_turn.py | 65 - switchyard/lib/endpoints/__init__.py | 64 - .../endpoints/anthropic_messages_endpoint.py | 123 - switchyard/lib/endpoints/base.py | 39 - switchyard/lib/endpoints/dispatch.py | 126 - switchyard/lib/endpoints/error_envelope.py | 171 -- switchyard/lib/endpoints/models_endpoint.py | 42 - .../lib/endpoints/openai_chat_endpoint.py | 102 - switchyard/lib/endpoints/outcome_metrics.py | 249 -- .../lib/endpoints/prometheus_emitter.py | 70 - .../lib/endpoints/responses_endpoint.py | 101 - switchyard/lib/endpoints/route_selection.py | 68 - .../endpoints/routing_log_stats_endpoint.py | 45 - switchyard/lib/endpoints/sse_helpers.py | 207 -- switchyard/lib/endpoints/stats_endpoint.py | 100 - switchyard/lib/endpoints/upstream_error.py | 233 -- .../lib/endpoints/upstream_error_log.py | 90 - switchyard/lib/llm_client.py | 190 -- switchyard/lib/model_listing.py | 131 -- switchyard/lib/processors/__init__.py | 20 - switchyard/lib/processors/format_translate.py | 311 --- .../model_rewrite_request_processor.py | 26 - .../rl_logging_request_processor.py | 43 - .../rl_logging_response_processor.py | 187 -- .../routing_log_response_processor.py | 205 -- .../lib/processors/stats_request_processor.py | 8 - .../stats_response_processor_accumulator.py | 8 - switchyard/lib/prometheus_exposition.py | 216 -- switchyard/lib/proxy_context.py | 123 - switchyard/lib/request_metadata.py | 142 -- switchyard/lib/roles.py | 48 - switchyard/lib/route_table.py | 170 -- switchyard/lib/startup_timing.py | 56 - switchyard/lib/stats_accumulator.py | 8 - switchyard/lib/switchyard.py | 10 - switchyard/lib/tracing.py | 72 - switchyard/server/__init__.py | 34 - switchyard/server/server_util.py | 389 ---- switchyard/server/switchyard_app.py | 214 -- switchyard/telemetry.py | 64 - switchyard_rust/__init__.py | 153 +- switchyard_rust/_native.py | 34 + switchyard_rust/components.py | 90 - switchyard_rust/components.pyi | 270 --- switchyard_rust/core.py | 793 ------- switchyard_rust/libsy.py | 8 +- switchyard_rust/server.py | 4 +- switchyard_rust/translation.py | 527 ----- tests/_chain_test_helpers.py | 234 -- tests/conftest.py | 23 - tests/contract/__init__.py | 14 - tests/contract/test_platform_imports.py | 68 - tests/contract/test_proxy_context.py | 67 - tests/contract/test_request_response_types.py | 121 - tests/e2e/_helpers.py | 103 - tests/e2e/conftest.py | 249 -- tests/e2e/test_passthrough_e2e.py | 552 ----- tests/e2e/test_passthrough_responses_e2e.py | 557 ----- tests/e2e_multiturn_responses.py | 369 --- tests/getting_started/test_getting_started.py | 2 +- tests/readme/test_readme.py | 2 +- tests/test_anthropic_native_llm_backend.py | 74 - tests/test_anthropic_openai_translation.py | 101 - tests/test_anthropic_output_config_strip.py | 34 - tests/test_anthropic_probe.py | 156 -- tests/test_backend_format_resolver.py | 399 ---- tests/test_build_and_serve.py | 384 --- tests/test_chat_request.py | 213 -- tests/test_chat_response.py | 770 ------ tests/test_cli_reference_docs.py | 10 +- tests/test_codex_multiturn_traces.py | 510 ---- tests/test_context_error_translation.py | 50 - .../test_context_window_exceeded_endpoint.py | 73 - tests/test_cost_estimator_gemini.py | 2 +- tests/test_endpoint_state_contract.py | 119 - tests/test_error_source_annotation.py | 156 -- tests/test_format_translate_processor.py | 593 ----- tests/test_inference_e2e.py | 599 ----- tests/test_infra.py | 146 -- tests/test_init_all_exports.py | 23 - tests/test_launchers.py | 4 +- tests/test_live_stats_footer.py | 98 +- tests/test_llm_client.py | 205 -- tests/test_metrics_endpoint.py | 155 -- tests/test_no_stale_module_paths.py | 31 +- tests/test_outcome_metrics.py | 304 --- tests/test_prometheus_emitter.py | 56 - tests/test_prometheus_exposition.py | 176 -- tests/test_python_server_passthrough.py | 482 ---- tests/test_request_metadata.py | 129 -- tests/test_request_translation_engine.py | 495 ---- ...st_request_translation_engine_to_any_of.py | 162 -- tests/test_response_translation_engine.py | 540 ----- tests/test_responses_openai_translation.py | 1137 --------- tests/test_rl_logging.py | 336 --- tests/test_rl_logging_e2e.py | 121 - tests/test_route_bundle.py | 176 -- tests/test_route_selection_headers.py | 233 -- tests/test_route_table.py | 282 --- tests/test_routing_log_response_processor.py | 297 --- tests/test_shell_tui.py | 8 +- tests/test_sse_stream_close.py | 159 -- tests/test_stats_accumulator.py | 247 -- tests/test_stream_close_chain.py | 212 -- tests/test_stream_leak_repro.py | 261 --- tests/test_switchyard.py | 517 ----- tests/test_switchyard_app_factory.py | 88 - tests/test_switchyard_app_lifecycle.py | 49 - ...test_switchyard_rust_component_bindings.py | 233 -- tests/test_switchyard_rust_core_bindings.py | 273 --- tests/test_telemetry.py | 123 - tests/test_tool_result_signal_collector.py | 192 -- tests/test_tracing.py | 59 - tests/test_translation_engine_chaos.py | 2064 ----------------- tests/test_upstream_error_log.py | 132 -- tests/test_upstream_error_passthrough.py | 317 --- tests/translation/__init__.py | 2 - .../test_format_fidelity_contract.py | 148 -- uv.lock | 503 +--- 241 files changed, 325 insertions(+), 41894 deletions(-) delete mode 100644 .agents/skills/switchyard-stage-router-scorer/SKILL.md delete mode 100644 benchmark/score_staged_run.py delete mode 100644 crates/switchyard-components/Cargo.toml delete mode 100644 crates/switchyard-components/src/backends/anthropic.rs delete mode 100644 crates/switchyard-components/src/backends/common.rs delete mode 100644 crates/switchyard-components/src/backends/context_overflow.rs delete mode 100644 crates/switchyard-components/src/backends/mod.rs delete mode 100644 crates/switchyard-components/src/backends/multi.rs delete mode 100644 crates/switchyard-components/src/backends/openai.rs delete mode 100644 crates/switchyard-components/src/backends/selection.rs delete mode 100644 crates/switchyard-components/src/backends/stats.rs delete mode 100644 crates/switchyard-components/src/contracts/backend.rs delete mode 100644 crates/switchyard-components/src/contracts/context.rs delete mode 100644 crates/switchyard-components/src/contracts/error.rs delete mode 100644 crates/switchyard-components/src/contracts/ids.rs delete mode 100644 crates/switchyard-components/src/contracts/mod.rs delete mode 100644 crates/switchyard-components/src/contracts/roles.rs delete mode 100644 crates/switchyard-components/src/contracts/types.rs delete mode 100644 crates/switchyard-components/src/dimension_collector/mod.rs delete mode 100644 crates/switchyard-components/src/dimension_collector/response/checks.rs delete mode 100644 crates/switchyard-components/src/dimension_collector/response/mod.rs delete mode 100644 crates/switchyard-components/src/dimension_collector/tool_signals.rs delete mode 100644 crates/switchyard-components/src/lib.rs delete mode 100644 crates/switchyard-components/src/request_processors/dimension_collector.rs delete mode 100644 crates/switchyard-components/src/request_processors/mod.rs delete mode 100644 crates/switchyard-components/src/request_processors/random_routing.rs delete mode 100644 crates/switchyard-components/src/request_processors/stats.rs delete mode 100644 crates/switchyard-components/src/response_processors/mod.rs delete mode 100644 crates/switchyard-components/src/response_processors/response_signals.rs delete mode 100644 crates/switchyard-components/src/response_processors/stats.rs delete mode 100644 crates/switchyard-components/src/stage_router.rs delete mode 100644 crates/switchyard-components/src/stats/accumulator.rs delete mode 100644 crates/switchyard-components/src/stats/cache_eligibility.rs delete mode 100644 crates/switchyard-components/src/stats/context.rs delete mode 100644 crates/switchyard-components/src/stats/cost.rs delete mode 100644 crates/switchyard-components/src/stats/mod.rs delete mode 100644 crates/switchyard-components/src/stats/usage.rs delete mode 100644 crates/switchyard-components/src/telemetry.rs delete mode 100644 crates/switchyard-components/tests/adversarial_multi_llm_backend.rs delete mode 100644 crates/switchyard-components/tests/adversarial_native_backends.rs delete mode 100644 crates/switchyard-components/tests/adversarial_random_routing.rs delete mode 100644 crates/switchyard-components/tests/contracts.rs delete mode 100644 crates/switchyard-components/tests/stats_accumulator.rs delete mode 100644 crates/switchyard-components/tests/stats_processors.rs delete mode 100644 crates/switchyard-components/tests/stats_usage_shapes.rs delete mode 100644 crates/switchyard-components/tests/support/config.rs delete mode 100644 crates/switchyard-components/tests/support/mod.rs delete mode 100644 crates/switchyard-py/src/component_bindings.rs delete mode 100644 crates/switchyard-py/src/component_bindings/backends.rs delete mode 100644 crates/switchyard-py/src/component_bindings/config.rs delete mode 100644 crates/switchyard-py/src/component_bindings/dimension_collector.rs delete mode 100644 crates/switchyard-py/src/component_bindings/request_processors.rs delete mode 100644 crates/switchyard-py/src/component_bindings/response_processors.rs delete mode 100644 crates/switchyard-py/src/component_bindings/stage_router.rs delete mode 100644 crates/switchyard-py/src/component_bindings/stats.rs delete mode 100644 crates/switchyard-py/src/interop.rs delete mode 100644 crates/switchyard-py/src/interop/context.rs delete mode 100644 crates/switchyard-py/src/interop/request.rs delete mode 100644 crates/switchyard-py/src/interop/response.rs delete mode 100644 crates/switchyard-py/src/interop/roles.rs delete mode 100644 crates/switchyard-py/src/interop/subagent.rs delete mode 100644 crates/switchyard-py/src/translation.rs delete mode 100644 examples/minimal.py delete mode 100644 examples/route.yaml delete mode 100644 examples/utils.py rename switchyard/{lib => cli/launchers}/cost_estimator.py (99%) rename switchyard/{server => cli/launchers}/shell_tui.py (100%) delete mode 100644 switchyard/cli/model_catalog/__init__.py delete mode 100644 switchyard/cli/model_catalog/model_discovery.py delete mode 100644 switchyard/cli/route_bundle.py delete mode 100644 switchyard/lib/__init__.py delete mode 100644 switchyard/lib/backends/__init__.py delete mode 100644 switchyard/lib/backends/anthropic_native_llm_backend.py delete mode 100644 switchyard/lib/backends/backend_format_resolver.py delete mode 100644 switchyard/lib/backends/llm_target.py delete mode 100644 switchyard/lib/backends/multi_llm_backend.py delete mode 100644 switchyard/lib/backends/openai_llm_backend.py delete mode 100644 switchyard/lib/backends/openai_native_backend.py delete mode 100644 switchyard/lib/backends/stats_llm_backend.py delete mode 100644 switchyard/lib/chat_request/__init__.py delete mode 100644 switchyard/lib/chat_request/anthropic.py delete mode 100644 switchyard/lib/chat_request/base.py delete mode 100644 switchyard/lib/chat_request/openai_chat.py delete mode 100644 switchyard/lib/chat_request/openai_responses.py delete mode 100644 switchyard/lib/chat_response/__init__.py delete mode 100644 switchyard/lib/chat_response/anthropic.py delete mode 100644 switchyard/lib/chat_response/base.py delete mode 100644 switchyard/lib/chat_response/openai_chat.py delete mode 100644 switchyard/lib/chat_response/openai_responses.py delete mode 100644 switchyard/lib/chat_response/streaming_response_accumulator.py delete mode 100644 switchyard/lib/conversation_turn.py delete mode 100644 switchyard/lib/endpoints/__init__.py delete mode 100644 switchyard/lib/endpoints/anthropic_messages_endpoint.py delete mode 100644 switchyard/lib/endpoints/base.py delete mode 100644 switchyard/lib/endpoints/dispatch.py delete mode 100644 switchyard/lib/endpoints/error_envelope.py delete mode 100644 switchyard/lib/endpoints/models_endpoint.py delete mode 100644 switchyard/lib/endpoints/openai_chat_endpoint.py delete mode 100644 switchyard/lib/endpoints/outcome_metrics.py delete mode 100644 switchyard/lib/endpoints/prometheus_emitter.py delete mode 100644 switchyard/lib/endpoints/responses_endpoint.py delete mode 100644 switchyard/lib/endpoints/route_selection.py delete mode 100644 switchyard/lib/endpoints/routing_log_stats_endpoint.py delete mode 100644 switchyard/lib/endpoints/sse_helpers.py delete mode 100644 switchyard/lib/endpoints/stats_endpoint.py delete mode 100644 switchyard/lib/endpoints/upstream_error.py delete mode 100644 switchyard/lib/endpoints/upstream_error_log.py delete mode 100644 switchyard/lib/llm_client.py delete mode 100644 switchyard/lib/model_listing.py delete mode 100644 switchyard/lib/processors/__init__.py delete mode 100644 switchyard/lib/processors/format_translate.py delete mode 100644 switchyard/lib/processors/model_rewrite_request_processor.py delete mode 100644 switchyard/lib/processors/rl_logging_request_processor.py delete mode 100644 switchyard/lib/processors/rl_logging_response_processor.py delete mode 100644 switchyard/lib/processors/routing_log_response_processor.py delete mode 100644 switchyard/lib/processors/stats_request_processor.py delete mode 100644 switchyard/lib/processors/stats_response_processor_accumulator.py delete mode 100644 switchyard/lib/prometheus_exposition.py delete mode 100644 switchyard/lib/proxy_context.py delete mode 100644 switchyard/lib/request_metadata.py delete mode 100644 switchyard/lib/roles.py delete mode 100644 switchyard/lib/route_table.py delete mode 100644 switchyard/lib/startup_timing.py delete mode 100644 switchyard/lib/stats_accumulator.py delete mode 100644 switchyard/lib/switchyard.py delete mode 100644 switchyard/lib/tracing.py delete mode 100644 switchyard/server/__init__.py delete mode 100644 switchyard/server/server_util.py delete mode 100644 switchyard/server/switchyard_app.py delete mode 100644 switchyard/telemetry.py create mode 100644 switchyard_rust/_native.py delete mode 100644 switchyard_rust/components.py delete mode 100644 switchyard_rust/components.pyi delete mode 100644 switchyard_rust/core.py delete mode 100644 switchyard_rust/translation.py delete mode 100644 tests/_chain_test_helpers.py delete mode 100644 tests/conftest.py delete mode 100644 tests/contract/__init__.py delete mode 100644 tests/contract/test_platform_imports.py delete mode 100644 tests/contract/test_proxy_context.py delete mode 100644 tests/contract/test_request_response_types.py delete mode 100644 tests/e2e/_helpers.py delete mode 100644 tests/e2e/conftest.py delete mode 100644 tests/e2e/test_passthrough_e2e.py delete mode 100644 tests/e2e/test_passthrough_responses_e2e.py delete mode 100644 tests/e2e_multiturn_responses.py delete mode 100644 tests/test_anthropic_native_llm_backend.py delete mode 100644 tests/test_anthropic_openai_translation.py delete mode 100644 tests/test_anthropic_output_config_strip.py delete mode 100644 tests/test_anthropic_probe.py delete mode 100644 tests/test_backend_format_resolver.py delete mode 100644 tests/test_build_and_serve.py delete mode 100644 tests/test_chat_request.py delete mode 100644 tests/test_chat_response.py delete mode 100644 tests/test_codex_multiturn_traces.py delete mode 100644 tests/test_context_error_translation.py delete mode 100644 tests/test_context_window_exceeded_endpoint.py delete mode 100644 tests/test_endpoint_state_contract.py delete mode 100644 tests/test_error_source_annotation.py delete mode 100644 tests/test_format_translate_processor.py delete mode 100644 tests/test_inference_e2e.py delete mode 100644 tests/test_infra.py delete mode 100644 tests/test_init_all_exports.py delete mode 100644 tests/test_llm_client.py delete mode 100644 tests/test_metrics_endpoint.py delete mode 100644 tests/test_outcome_metrics.py delete mode 100644 tests/test_prometheus_emitter.py delete mode 100644 tests/test_prometheus_exposition.py delete mode 100644 tests/test_python_server_passthrough.py delete mode 100644 tests/test_request_metadata.py delete mode 100644 tests/test_request_translation_engine.py delete mode 100644 tests/test_request_translation_engine_to_any_of.py delete mode 100644 tests/test_response_translation_engine.py delete mode 100644 tests/test_responses_openai_translation.py delete mode 100644 tests/test_rl_logging.py delete mode 100644 tests/test_rl_logging_e2e.py delete mode 100644 tests/test_route_bundle.py delete mode 100644 tests/test_route_selection_headers.py delete mode 100644 tests/test_route_table.py delete mode 100644 tests/test_routing_log_response_processor.py delete mode 100644 tests/test_sse_stream_close.py delete mode 100644 tests/test_stats_accumulator.py delete mode 100644 tests/test_stream_close_chain.py delete mode 100644 tests/test_stream_leak_repro.py delete mode 100644 tests/test_switchyard.py delete mode 100644 tests/test_switchyard_app_factory.py delete mode 100644 tests/test_switchyard_app_lifecycle.py delete mode 100644 tests/test_switchyard_rust_component_bindings.py delete mode 100644 tests/test_switchyard_rust_core_bindings.py delete mode 100644 tests/test_telemetry.py delete mode 100644 tests/test_tool_result_signal_collector.py delete mode 100644 tests/test_tracing.py delete mode 100644 tests/test_translation_engine_chaos.py delete mode 100644 tests/test_upstream_error_log.py delete mode 100644 tests/test_upstream_error_passthrough.py delete mode 100644 tests/translation/__init__.py delete mode 100644 tests/translation/test_format_fidelity_contract.py diff --git a/.agents/skills/switchyard-stage-router-scorer/SKILL.md b/.agents/skills/switchyard-stage-router-scorer/SKILL.md deleted file mode 100644 index b11cea0e8..000000000 --- a/.agents/skills/switchyard-stage-router-scorer/SKILL.md +++ /dev/null @@ -1,97 +0,0 @@ ---- -name: switchyard-stage-router-scorer -description: Score benchmark run trajectories through the stage-router Rust scorer and picker, then visualise score distributions. Use when you want to replay trajectories through the picker, analyse routing splits, or compare score distributions across configs. ---- - -# Skill: switchyard-stage-router-scorer - -## Scripts - -| Script | Purpose | Input | Output | -|--------|---------|-------|--------| -| `benchmark/score_staged_run.py` | Score a live run dir via real picker | run dir path | per-turn JSONL + per-task CSV | - -## Quick Reference - -```bash -# Score a run -uv run python benchmark/score_staged_run.py --run benchmark/tb_runs/ -# → /tmp/-scores.jsonl (per turn) -# → /tmp/-per-task.csv (per task) - -# Custom threshold or window -uv run python benchmark/score_staged_run.py \ - --run benchmark/tb_runs/ \ - --threshold 0.15 --window 3 -``` - -## Scoring pipeline (what score_staged_run.py does per turn) - -``` -trajectory step (tool_use + tool_result) - → append to cumulative Anthropic messages list - → ChatRequest.anthropic({"model": ..., "messages": messages}) # Rust binding - → dc.process(ctx, request) # DimensionCollector — one per task, accumulates state - → get_tool_result_signal(ctx) # read signal from ctx - → stage_score_signal(signal) # raw (score, confidence) from the Rust scorer (for analysis) - → stage_pick_tier(signal, "capable_first", threshold) # actual cf decision - → stage_pick_tier(signal, "efficient_first", threshold) # actual ef decision -``` - -**Key:** `dc.process()` is called **once per turn** on a single context. Both picker modes use the -same extracted signal, with no duplicate processing. - -**What the picker does beyond raw score:** -- **escalate** (`should_escalate`): `compacted` OR `severity >= 1.0` → force CAPABLE -- **de-escalate** (`should_deescalate`): `tests_passed AND recent_write+edit >= 1 AND severity <= 0` → force EFFICIENT -- `confidence < threshold` → fall_open to default tier (CAPABLE for cf, EFFICIENT for ef) -- Only when `confidence >= threshold`: route by score direction - -## Per-turn JSONL schema - -```json -{ - "task_name": "terminal-bench/...", "trial_name": "...", "run_id": "...", - "reward": 1.0, "turn_depth": 5, - "score": -0.83, "confidence": 0.95, - "tool_name": "Bash", "is_error": false, - "write_count": 2, "edit_count": 1, "read_count": 3, - "no_error_streak": 4, "pure_bash_streak": 2, "tests_passed": false, - "pick_cf": 1, - "pick_ef": 0 -} -``` - -`pick_cf` / `pick_ef`: `1` = CAPABLE (Opus), `0` = EFFICIENT (Nemotron) - -## Per-task CSV columns - -`run_id, task_name, reward, n_turns, mean_score, mean_confidence, -pct_strong_clear, pct_strong_uncertain, pct_weak_uncertain, pct_weak_clear, -opus_pct_cf, nemotron_pct_cf, opus_pct_ef, nemotron_pct_ef` - -`opus_pct_cf` / `opus_pct_ef` are derived from actual `pick_cf` / `pick_ef` decisions, not score bands. - -## Band definitions (for histogram colouring, threshold T) - -| Band | Score range | cf default | ef default | -|------|-------------|------------|------------| -| strong_clear | ≥ T | Opus | Opus | -| strong_uncertain | (0, T) | Opus (fall_open) | Nemotron (fall_open) | -| weak_uncertain | (-T, 0) | Opus (fall_open) | Nemotron (fall_open) | -| weak_clear | ≤ -T | Nemotron | Nemotron | - -Overrides can change any band. Use `pick_cf`/`pick_ef` for the true decision. - -## Key findings (v0.2.0 baseline tarballs, T=0.20) - -- Distribution is **highly bimodal** — most turns land in strong_clear or weak_clear -- `cf, t=0.20`: ~34% Opus on partial run; ~42% on full baseline -- `ef, t=0.20`: ~20% Opus on partial run; ~41% on full baseline -- Live routing split differs from shadow score — Nemotron trajectories are shorter and reshape the distribution - -## Anti-patterns - -- Don't call `dc.process()` multiple times per turn for different picker modes. Extract one signal per turn and reuse it. -- Don't infer routing split from score bands alone — overrides and fall_open change the actual decision. Always use `pick_cf` / `pick_ef`. -- Don't compare cost directly across configs: Nemotron has ~39% cache hit rate vs ~92% for Opus. diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index bcbb7eed7..26283f7d7 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -16,7 +16,7 @@ Minimal steps to reproduce. Include the command line, the inbound request shape, ```bash # example -switchyard serve --routes examples/route.yaml --port 4000 +switchyard-server --config routes.toml --port 4000 curl -s http://localhost:4000/v1/chat/completions -d '{"model":"...","messages":[...]}' ``` diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67e3ae3b9..9d46b6ab3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -168,7 +168,7 @@ jobs: runs-on: ubuntu-latest env: SWITCHYARD_DEFAULT_PACKAGE: "nemo-switchyard @ file://${{ github.workspace }}" - SWITCHYARD_EXTRAS_PACKAGE: "nemo-switchyard[cli,server] @ file://${{ github.workspace }}" + SWITCHYARD_EXTRAS_PACKAGE: "nemo-switchyard[cli] @ file://${{ github.workspace }}" steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v6 @@ -213,7 +213,7 @@ jobs: sys.exit(f"FAIL: heavy packages pulled into slim install: {extras}") print(f"OK: all {len(forbidden)} heavy packages absent from slim install.") PY - - name: Verify CLI and server imports work with extras + - name: Verify CLI works with its extra working-directory: /tmp run: | uv run --isolated --no-project --python 3.12 \ @@ -225,9 +225,6 @@ jobs: uv run --isolated --no-project --python 3.12 \ --with "${SWITCHYARD_EXTRAS_PACKAGE}" \ switchyard launch claude --help 2>&1 | grep -q -- '--config' || { echo 'FAIL: --config flag missing from help'; exit 1; } - uv run --isolated --no-project --python 3.12 \ - --with "${SWITCHYARD_EXTRAS_PACKAGE}" \ - switchyard serve --help > /dev/null # Single required check for branch protection. Gates on every job above so # the ruleset never needs editing when jobs are added or renamed. diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml index cd797d567..32f1209da 100644 --- a/.github/workflows/perf.yml +++ b/.github/workflows/perf.yml @@ -55,8 +55,8 @@ jobs: run: uv pip install aiperf # ------------------------------------------------------------------ - # Start a local zero-latency mock OpenAI upstream. This replaces the - # The proxy serves a real `type: passthrough` chain that + # Start a local zero-latency mock OpenAI upstream. The proxy serves a + # real `type: passthrough` route that # forwards to this loopback stub, which returns a fixed completion # instantly. The extra loopback hop adds a small constant overhead. # ------------------------------------------------------------------ @@ -127,21 +127,30 @@ jobs: echo "STUB_PID=$!" >> "$GITHUB_ENV" # ------------------------------------------------------------------ - # Start the proxy: a `type: passthrough` route pointed at the local stub. + # Build and start the native proxy against the local stub. # ------------------------------------------------------------------ + - name: Build switchyard server + run: cargo build --locked --release -p switchyard-server + - name: Start switchyard proxy run: | - cat > bench.yaml < bench.toml <> "$GITHUB_ENV" - name: Wait for proxy to be ready diff --git a/AGENTS.md b/AGENTS.md index cc046923e..463ba90c9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -130,97 +130,59 @@ testing, and review do not require loading a skill. | `switchyard-coding-agent-launchers` | Claude Code, Codex, or OpenClaw launcher behavior | | `switchyard-docs` | Published MkDocs pages, strict builds, previews, and docs CI | | `switchyard-rust-review` | Focused review of Rust, PyO3, async, streaming, and crate boundaries | -| `switchyard-stage-router-scorer` | Replaying benchmark trajectories through the stage-router scorer and picker | | `switchyard-testing-ci` | Selecting non-obvious validation or diagnosing CI failures | Skills should contain stable operational constraints, not mutable architecture inventories. Read the current source and CI workflows for implementation details. -## Architecture: staged chain +## Architecture -Everything flows through a fixed-shape chain enforced at construction time: +The supported serving path is native Rust: ``` -request-side component* → LLMBackend → response-side component* → TranslationEngine +HTTP request → switchyard-server → libsy Algorithm → LlmTarget/RoutedLlmClient + → switchyard-translation → upstream model ``` -The chain executor is `Switchyard` (`switchyard/lib/switchyard.py`). `LLMBackend` is the shared -Python role class re-exported from `switchyard/lib/roles.py`; native implementations register with -it, and request-side and response-side processors are plain async components with `process(...)` -methods. -Direct Rust bindings for migrated concrete processors/backends are exposed from -`switchyard_rust.components` and implemented under `crates/switchyard-py/src/component_bindings/`. +`switchyard-server` loads explicit TOML deployments and exposes the OpenAI Chat, +OpenAI Responses, and Anthropic Messages APIs. `switchyard-libsy` owns routing +algorithms, `switchyard-protocol` owns provider-neutral request and response +types, and `switchyard-llm-client` performs translated HTTP calls. -| Stage | Binding | Method | Purpose | -|------|-----|--------|---------| -| Request component | Plain Python/Rust object | `async process(ctx, request) -> ChatRequest` | Pre-process (routing, buffering, auth) | -| `LLMBackend` | `switchyard.lib.roles` | `async call(ctx, request) -> ChatResponse` | Make the LLM call. Exactly one per chain. | -| Response component | Plain Python/Rust object | `async process(ctx, response) -> ChatResponse` | Post-process (logging, stats) | -| `TranslationEngine` | `switchyard_rust.translation` | `async translate(ctx, request, response) -> Any` | Convert to client's wire format | +Python is an integration layer. `switchyard launch` hosts the native server for +coding agents, `switchyard.libsy` exposes selected algorithms, and +`switchyard_rust.server` exposes the native server lifecycle through PyO3. ## Project Structure ``` switchyard/ -├── __init__.py # Public API — all exports live here -├── lib/ # Core library -│ ├── roles.py # Python LLMBackend re-export and translation aliases -│ ├── switchyard.py # Switchyard — chain executor -│ ├── proxy_context.py # ProxyContext — per-request state carrier -│ ├── route_table.py # RouteTable — model-id dispatch to runnable chains -│ ├── llm_client.py # OpenAILLMClient -│ ├── cost_estimator.py # Token-cost bookkeeping -│ ├── stats_accumulator.py # Stats accumulation helpers -│ ├── request_metadata.py # RequestMetadata -│ ├── chat_response/ # Rust-backed response re-exports + stream adapters -│ │ ├── base.py # ChatResponse, ChatResponseType compatibility re-export -│ │ ├── openai_chat.py # ResponseStream -│ │ ├── openai_responses.py # ResponsesApiStream -│ │ └── anthropic.py # AnthropicResponseStream -│ ├── backends/ # LLMBackend implementations -│ │ ├── openai_llm_backend.py # OpenAiPassthroughBackend -│ │ ├── openai_native_backend.py # OpenAiNativeBackend -│ │ ├── anthropic_native_llm_backend.py # AnthropicNativeBackend -│ │ ├── llm_target.py # LlmTarget, BackendFormat -│ │ ├── multi_llm_backend.py # MultiLlmBackend helpers -│ │ ├── stats_llm_backend.py # StatsLlmBackend -│ │ └── backend_format_resolver.py # BackendFormatResolver -│ ├── processors/ # Request-side / response-side component implementations -│ │ ├── format_translate.py -│ │ ├── stats_request_processor.py -│ │ └── stats_response_processor_accumulator.py -│ ├── endpoints/ # FastAPI endpoint wrappers (require `nemo-switchyard[server]`) -│ │ ├── openai_chat_endpoint.py # OpenAIChatEndpoint -│ │ ├── anthropic_messages_endpoint.py # AnthropicMessagesEndpoint -│ │ ├── responses_endpoint.py # ResponsesEndpoint -│ │ ├── stats_endpoint.py # StatsEndpoint -│ │ ├── sse_helpers.py -│ │ └── base.py +├── __init__.py # Package version ├── cli/ # CLI (requires `nemo-switchyard[cli]`) │ ├── switchyard_cli.py # `switchyard` entry point │ ├── launch_command.py # `switchyard launch` -│ ├── command_utils.py │ ├── defaults/ # packaged OpenRouter TOML deployment -│ ├── launchers/ # Claude, Codex, and OpenClaw launchers -│ ├── model_catalog/ # model_discovery -│ └── route_bundle.py # YAML route parsing for `serve` -└── server/ # FastAPI app factory + server utilities - ├── switchyard_app.py # build_switchyard_app() - ├── server_util.py # Shared CLI / server plumbing - └── shell_tui.py # Shell TUI session +│ └── launchers/ # Claude, Codex, and OpenClaw launchers +└── libsy/ # typed Python wrappers for libsy algorithms + +switchyard_rust/ # Python facades over the PyO3 extension +crates/libsy/ # routing algorithms and driver +crates/libsy-llm-client/ # translated HTTP LLM client +crates/protocol/ # provider-neutral protocol types +crates/switchyard-server/ # native HTTP server and TOML config +crates/switchyard-translation/ # wire-format codecs +crates/switchyard-py/ # libsy and server PyO3 bindings tests/ # Unit tests (pytest) ``` ## Tech Stack -- **Python 3.12+**, async-first (asyncio) -- **FastAPI + Uvicorn** for HTTP (`nemo-switchyard[server]`), **sse-starlette** for SSE streaming -- **OpenAI SDK** (`openai>=2.30`) — primary client; the translation engine converts all inbound formats to Chat Completions -- **Anthropic SDK** (`anthropic>=0.94`) -- **httpx** for direct HTTP (health polling, Anthropic Messages) +- **Rust 1.96.1**, edition 2024, Tokio, Axum, and PyO3 +- **Python 3.12+** for launchers and native bindings +- **prompt-toolkit** for interactive launcher sessions - **uv** as the package manager (preferred over pip) -- **pytest + pytest-asyncio** for testing, **respx** for HTTP mocking +- **Cargo test + pytest** for testing - **ruff** for linting, **mypy** (strict) for type checking ## Setup @@ -237,20 +199,20 @@ and their transitives never appear in downstream vulnerability scans. ## Commands -### Running the server +### Running Switchyard ```bash export OPENROUTER_API_KEY="sk-or-..." -# Serve the minimal Python YAML bundle (noop and passthrough only). -switchyard serve --routes examples/route.yaml --port 4000 - # Launch against the packaged OpenRouter deployment. switchyard launch claude --model switchyard switchyard launch codex --model switchyard # Or select a route from a custom native TOML deployment. switchyard launch claude --model my-route --config routes.toml + +# Run a standalone native server. +switchyard-server --config routes.toml --port 4000 ``` ### Testing @@ -260,8 +222,7 @@ switchyard launch claude --model my-route --config routes.toml uv run pytest tests/ -v # Single test file / function -uv run pytest tests/test_switchyard.py -v -uv run pytest tests/test_route_bundle.py::test_noop_route_returns_ok_without_an_upstream -v +uv run pytest tests/test_launchers.py -v # Live end-to-end tests are not part of the public test suite; if you write # one, set the provider key explicitly and run it directly, e.g.: @@ -270,49 +231,7 @@ uv run pytest tests/test_route_bundle.py::test_noop_route_returns_ok_without_an_ # Lint / type check (run before every commit) uv run ruff check . uv run mypy switchyard -``` - -### Adding a new component - -1. Pick the right stage: request component (pre-call), response component (post-call), `LLMBackend` (rare), or Rust translation codec work. -2. Create a file with the explicit name (`snake_case` of the class name), one class per file. -3. Implement the async method for that stage (`process` for components, `call` for backends). -4. Wire it into the owning explicit chain or route-bundle builder. -5. Add tests under `tests/`. -6. Export from the relevant `__init__.py` and from `switchyard/__init__.py`'s `__all__`. - -### Example: custom request component - -```python -from switchyard.lib.proxy_context import ProxyContext -from switchyard import ChatRequest - - -class MyRequestComponent: - async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: - ctx.metadata["my_key"] = "my_value" - return request -``` - -### App factory - -```python -from switchyard import BackendFormat, LlmTarget, Switchyard, TranslationEngine -from switchyard.lib.backends import OpenAiNativeBackend -from switchyard import build_switchyard_app -import uvicorn - -target = LlmTarget( - model="gpt-4o", - format=BackendFormat.OPENAI, - api_key="sk-...", - base_url="https://api.openai.com/v1", -) -switchyard = Switchyard( - backend=OpenAiNativeBackend(target), - translator=TranslationEngine(), -) -uvicorn.run(build_switchyard_app(switchyard), port=4000) +cargo test --workspace ``` ## Environment Variables @@ -349,16 +268,15 @@ uvicorn.run(build_switchyard_app(switchyard), port=4000) ### Always do - File name = snake_case of the primary class exported. Rename on touch. - Run `uv run ruff check .` (zero errors) and `uv run pytest tests/` before pushing. -- Export new public classes from `switchyard/__init__.py` with `__all__`. -- Write unit tests for new roles and bug fixes. -- Use `ProxyContext.metadata` for cross-component state within a request. -- In a new `LLMBackend`, map upstream context-window 4xx to `SwitchyardError::ContextWindowExceeded` (Rust) — the chain executor uses it for evict-and-retry. See [Context-Window Handling](docs/operations/context_window.md). +- Run `cargo test --workspace` for Rust behavior changes. +- Write focused unit tests for new behavior and bug fixes. +- Keep provider-neutral request and response types in `switchyard-protocol`. +- Map upstream context-window errors to `SwitchyardError::ContextWindowExceeded`. ### Ask first - Modifying `pyproject.toml` dependencies. -- Changes to the chain shape or public role classes in `switchyard/lib/roles.py`. - Adding new HTTP endpoints. -- Removing or renaming any public API currently in `switchyard/__init__.__all__`. +- Removing or renaming public Rust, PyO3, or Python APIs. ### Never do - Commit API keys or secrets (`secrets/` is gitignored). diff --git a/CHANGELOG.md b/CHANGELOG.md index 444c44956..de81bbc1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ All notable changes to Switchyard are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Removed + +- **Deprecated Python server stack** — `switchyard serve`, YAML route bundles, + the FastAPI endpoints and legacy chain, the `switchyard-components` crate, + and their compatibility PyO3 bindings are removed. Use `switchyard-server` + with native TOML deployments, or `switchyard launch` for coding agents. + ## [0.2.0] Switchyard 0.2.0 introduces the native Rust server and libsy library path, diff --git a/Cargo.lock b/Cargo.lock index e5e151e0a..39871f0e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2263,27 +2263,6 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" -[[package]] -name = "switchyard-components" -version = "0.2.0" -dependencies = [ - "async-stream", - "async-trait", - "futures-core", - "futures-util", - "parking_lot", - "rand 0.10.2", - "reqwest", - "serde", - "serde_json", - "switchyard-libsy", - "switchyard-protocol", - "switchyard-translation", - "thiserror 2.0.18", - "tokio", - "tracing", -] - [[package]] name = "switchyard-libsy" version = "0.2.0" @@ -2349,22 +2328,17 @@ name = "switchyard-py" version = "0.2.0" dependencies = [ "async-trait", - "futures-util", "http", - "parking_lot", "pyo3", "pyo3-async-runtimes", "pythonize", "serde", "serde_json", - "switchyard-components", "switchyard-libsy", "switchyard-llm-client", "switchyard-protocol", "switchyard-server", - "switchyard-translation", "tokio", - "tracing", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 0c05c4767..07d133bb3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,6 @@ resolver = "3" members = [ "crates/libsy", "crates/libsy-llm-client", - "crates/switchyard-components", "crates/switchyard-py", "crates/protocol", "crates/switchyard-server", @@ -36,7 +35,6 @@ rand = "0.10" reqwest = { version = "0.13.4", default-features = false, features = ["json", "rustls", "stream"] } serde = { version = "1", features = ["derive"] } serde_json = "1" -switchyard-components = { path = "crates/switchyard-components", version = "0.2.0" } switchyard-libsy = { path = "crates/libsy", version = "0.2.0" } switchyard-llm-client = { path = "crates/libsy-llm-client", version = "0.2.0" } switchyard-protocol = { path = "crates/protocol", version = "0.2.0" } diff --git a/INSTALLATION.md b/INSTALLATION.md index b63ae8d58..29c74ae63 100644 --- a/INSTALLATION.md +++ b/INSTALLATION.md @@ -1,186 +1,81 @@ # Installation Guide -Switchyard supports modular installation based on your use case. Install only the dependencies you need. +Switchyard has separate packages for Python integrations and standalone Rust +serving. -## System Requirements +## Requirements -- Python 3.12 or newer. If the active interpreter is older, run - `uv pip install --python 3.12 nemo-switchyard`. -- Linux x86_64 wheels require an x86-64-v3 / AVX2-class CPU (post 2013). -- Linux aarch64 wheels require a Neoverse N1-class CPU (post 2020). +- Python 3.12 or newer for `nemo-switchyard` +- Rust 1.96.1 or newer for `switchyard-server` and the Rust libraries +- Linux x86_64 wheels require an x86-64-v3 / AVX2-class CPU +- Linux aarch64 wheels require a Neoverse N1-class CPU -## Core Installation (Library Only) +## Python Bindings -For applications that use Switchyard as a Python library for routing and recipe composition: +Install the Python package to embed libsy algorithms or host the native server +through PyO3: ```bash pip install nemo-switchyard ``` -**Includes:** -- Core routing logic -- All recipe factories (Passthrough, RandomRouting, etc.) -- Format translation engine (Anthropic ↔ OpenAI) -- Request/response processors +The base package has no Python runtime dependencies. Its native extension owns +the libsy and server implementations. -**Does NOT include:** -- FastAPI / Uvicorn (server) -- prompt-toolkit (interactive command-line UI) +## Coding-Agent Launchers -**Use case:** Library users, middleware plugins, embedded integrations. - -## Optional Extras - -### `[server]` — Run as a Proxy Server - -Add FastAPI and Uvicorn to run Switchyard as a standalone HTTP proxy: +Install the CLI extra to launch Claude Code, Codex CLI, or OpenClaw through the +packaged native server: ```bash -pip install nemo-switchyard[server] +uv tool install --python 3.12 "nemo-switchyard[cli]" +export OPENROUTER_API_KEY="your-openrouter-key" # pragma: allowlist secret +switchyard launch claude --model switchyard ``` -**Adds:** -- FastAPI -- Uvicorn with standard extras -- sse-starlette (for SSE streaming) +The selected coding agent must already be installed and available on `PATH`. +Use `--config routes.toml` to select a custom native TOML deployment. -**Use case:** Deploying Switchyard as a service, e2e proxy operations. +## Standalone Server -### `[cli]` — Command-Line Tools - -Add prompt-toolkit support for interactive command-line workflows: +Install the native Rust proxy from crates.io: ```bash -pip install nemo-switchyard[cli] +cargo install --locked switchyard-server +switchyard-server --config routes.toml --dry-run +switchyard-server --config routes.toml --port 4000 ``` -## Combined Extras - -### Full Installation - -Install all optional dependencies: +See [Getting Started](docs/getting_started.md#server-path) for a complete TOML +deployment and [`switchyard-server`](crates/switchyard-server/README.md) for the +configuration reference. -```bash -pip install nemo-switchyard[all] -``` - -Equivalent to: `nemo-switchyard[server,cli,tracing,affinity-redis]` +## Rust Libraries -### Common Combinations +Add the crates needed by an embedded application: -**Middleware plugin (no server/CLI):** -```bash -pip install nemo-switchyard # Core only +```toml +[dependencies] +switchyard-libsy = "0.2.0" +switchyard-protocol = "0.2.0" +switchyard-llm-client = "0.2.0" +switchyard-translation = "0.2.0" ``` -**Proxy server:** -```bash -pip install nemo-switchyard[server] -``` - -**Command-line tools:** -```bash -pip install nemo-switchyard[cli] # Includes core -``` - -**Production deployment with all features:** -```bash -pip install nemo-switchyard[all] -``` - -## Dependency Structure - -### Core (Always Installed) -``` -- openai>=2.34.0,<3.0 -- anthropic>=0.99.0,<1.0 -- httpx>=0.28.1,<1.0 -- pydantic>=2.13.3,<3.0 -``` - -### Optional Dependencies -| Extra | Size | Purpose | -|-------|------|---------| -| `[server]` | ~50 MB | HTTP proxy (FastAPI + Uvicorn) | -| `[cli]` | ~5 MB | Interactive terminal UI (prompt-toolkit) | - -## Embedding in Your Own Application - -### For Custom Applications - -Embed Switchyard with minimal overhead: - -```python -from switchyard import LlmTarget, OpenAiNativeBackend, Switchyard, TranslationEngine - -# Core library only — no server/CLI dependencies -target = LlmTarget( - id="direct", - model="gpt-4o-mini", - format="openai", - api_key="sk-...", - base_url="https://api.openai.com/v1", -) -switchyard = Switchyard( - backend=OpenAiNativeBackend(target), - translator=TranslationEngine(), -) -``` - -## Troubleshooting - -### Import Error: "No module named 'fastapi'" - -You're trying to run the HTTP server without the `[server]` extra: - -```bash -# Install with server support -pip install nemo-switchyard[server] -``` - -```python -from switchyard import Switchyard # OK (core) -from switchyard.server.switchyard_app import build_switchyard_app # needs [server] -``` - -### Import Error: "No module named 'prompt_toolkit'" - -You're using an interactive command-line workflow without the `[cli]` extra: - -```bash -# Install CLI support -pip install nemo-switchyard[cli] -``` +`switchyard-libsy` owns algorithms, `switchyard-protocol` owns provider-neutral +request and response types, `switchyard-translation` owns wire conversion, and +`switchyard-llm-client` performs translated HTTP calls. ## Development -For development with all testing tools, use `uv` (recommended): +From a checkout: ```bash -uv sync # core + dev tooling (dev is uv's default group) -uv sync --all-extras # add every user-facing extra as well +uv sync +uv run maturin develop +cargo test --workspace +uv run pytest tests/ -v ``` -Or with pip ≥ 25.1 from a checkout: - -```bash -pip install -e ".[all]" -pip install --group dev . -``` - -This includes: -- Core + all optional extras -- pytest, ruff, mypy, respx for testing - -> **Note:** dev tooling lives in a PEP 735 dependency group, not an extra, -> so `pip install nemo-switchyard[dev]` is **not supported** and dev tooling -> never appears in the published wheel's METADATA (it's invisible to -> downstream vulnerability scans). - -## Version Compatibility - -Switchyard requires Python 3.12 or later. - -Supported versions: -- Python 3.12 -- Python 3.13 +The `dev` dependency group contains testing and linting tools and is not exposed +in the published wheel metadata. diff --git a/README.md b/README.md index 7ead4efd3..cdc2b51de 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ not already available, then install the published Switchyard tool: ```bash curl -LsSf https://astral.sh/uv/install.sh | sh source "$HOME/.local/bin/env" -uv tool install --python 3.12 "nemo-switchyard[cli,server]" +uv tool install --python 3.12 "nemo-switchyard[cli]" ``` The coding agent you launch must also be installed and on your `PATH`. This does diff --git a/benchmark/README.md b/benchmark/README.md index 3a78eeaf0..86feef084 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -288,23 +288,6 @@ Neither artifact provides task or trial attribution. The runner writes them only and while the Rust server is still reachable; otherwise the manifest records them as missing. `routing_requests.jsonl` and `routing_stats_by_task.json` are not produced by the Rust server. -### Replay Stage-Router Scores - -Replay completed trajectories through the stage-router scorer and both picker policies: - -```bash -uv run python benchmark/score_staged_run.py \ - --run benchmark/tb_runs/ -``` - -Use `--threshold` and `--window` to override the scorer defaults. The command writes per-turn JSONL -to `/tmp/-scores.jsonl` and a per-task summary to -`/tmp/-per-task.csv` unless `--output` or `--csv` is supplied. - -The script processes each turn once, then applies both pickers to the same signal. Use `pick_cf` and -`pick_ef` for actual routing decisions; score bands alone do not include picker overrides or -fall-open behavior. - ## Docker Image Notes Baseline runs build `switchyard-baseline:local` from diff --git a/benchmark/score_staged_run.py b/benchmark/score_staged_run.py deleted file mode 100644 index 6e4c9a2f6..000000000 --- a/benchmark/score_staged_run.py +++ /dev/null @@ -1,283 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Score a benchmark run directory via the stage-router Rust scorer. - -Reads trajectory.json from each completed task, feeds each tool-use turn through: - DimensionCollector.process(ctx, request) → stage_pick_tier - -The picker functions replicate live routing exactly: overrides, scorer, and fall_open. - -Usage: - uv run python benchmark/score_staged_run.py --run benchmark/tb_runs/ - uv run python benchmark/score_staged_run.py --run benchmark/tb_runs/ \\ - --output /tmp/scores.jsonl --threshold 0.20 --window 3 -""" -import argparse -import asyncio -import csv -import json -import sys -from collections import defaultdict -from pathlib import Path -from statistics import mean - -from switchyard_rust.components import ( - DimensionCollector, - get_tool_result_signal, - stage_pick_tier, - stage_score_signal, -) -from switchyard_rust.core import ChatRequest, ProxyContext - -RECENT_WINDOW = 3 -CAPABLE = 1 - - -def _tool_use_id(message: str) -> str: - parts = message.split() - return parts[-1] if len(parts) >= 2 else f"tu_{id(message)}" - - -async def score_trajectory( - traj: dict, - reward: float | None, - task_name: str, - trial_name: str, - run_id: str, - recent_window: int, - confidence_threshold: float, -) -> list[dict]: - steps = traj.get("steps", []) - - # One DimensionCollector per task — accumulates state across turns - dc = DimensionCollector(recent_window=recent_window) - await dc.startup() - - rows: list[dict] = [] - messages: list[dict] = [] - - for s in steps: - if s["source"] == "user" and not (s.get("extra") or {}).get("is_sidechain"): - messages.append({"role": "user", "content": [{"type": "text", "text": s["message"]}]}) - break - - if not messages: - return rows - - for s in steps: - if s["source"] != "agent": - continue - extra = s.get("extra") or {} - if extra.get("is_sidechain"): - continue - - tool_name = extra.get("tool_use_name") - if not tool_name: - text = s.get("message", "") - if text: - messages.append({"role": "assistant", "content": [{"type": "text", "text": text}]}) - continue - - tool_use_id = _tool_use_id(s.get("message", "")) - raw_args = extra.get("raw_arguments") or {} - is_error = bool(extra.get("tool_result_is_error", False)) - - metadata = extra.get("metadata") or {} - raw_result = metadata.get("raw_tool_result") or {} - content = raw_result.get("content", "") - if not isinstance(content, str): - content = json.dumps(content) - - messages.append({ - "role": "assistant", - "content": [{"type": "tool_use", "id": tool_use_id, "name": tool_name, "input": raw_args}], - }) - messages.append({ - "role": "user", - "content": [{"type": "tool_result", "tool_use_id": tool_use_id, "content": content, "is_error": is_error}], - }) - - # Build Anthropic ChatRequest and process once through DimensionCollector - ctx = ProxyContext() - request = ChatRequest.anthropic({ - "model": "claude-opus-4-8", - "max_tokens": 8096, - "messages": messages, - }) - await dc.process(ctx, request) - - signal = get_tool_result_signal(ctx) - if signal is None: - continue - - # Raw score for analysis (Rust scorer): wrong signals score positive - # (→CAPABLE), progress negative (→EFFICIENT). Picker-independent, so the - # histogram shows the capable/efficient separation directly. - score, confidence = stage_score_signal(signal) - - # Apply both picker modes to the same signal without reprocessing the turn. - outcome_cf = stage_pick_tier(signal, "capable_first", confidence_threshold) - outcome_ef = stage_pick_tier(signal, "efficient_first", confidence_threshold) - tier_cf_name = outcome_cf.tier if outcome_cf.resolved else outcome_cf.default_tier - tier_ef_name = outcome_ef.tier if outcome_ef.resolved else outcome_ef.default_tier - tier_cf = int(tier_cf_name == "capable") - tier_ef = int(tier_ef_name == "capable") - - rows.append({ - "task_name": task_name, - "trial_name": trial_name, - "run_id": run_id, - "reward": reward, - "turn_depth": signal.turn_depth, - "score": score, - "confidence": confidence, - "tool_name": tool_name, - "is_error": is_error, - "write_count": signal.write_count, - "edit_count": signal.edit_count, - "read_count": signal.read_count, - "no_error_streak": signal.no_error_streak, - "pure_bash_streak": signal.pure_bash_streak, - "tests_passed": signal.tests_passed, - "pick_cf": tier_cf, # CAPABLE=1, EFFICIENT=0 - "pick_ef": tier_ef, - }) - - return rows - - -def band(score: float, threshold: float) -> str: - if score >= threshold: - return "strong_clear" - if score > 0: - return "strong_uncertain" - if score <= -threshold: - return "weak_clear" - if score < 0: - return "weak_uncertain" - return "zero" - - -def write_per_task_csv(all_rows: list[dict], csv_path: Path, threshold: float) -> None: - groups: dict[tuple, list] = defaultdict(list) - for r in all_rows: - groups[(r["run_id"], r["task_name"])].append(r) - - fieldnames = [ - "run_id", "task_name", "reward", "n_turns", - "mean_score", "mean_confidence", - "pct_strong_clear", "pct_strong_uncertain", "pct_weak_uncertain", "pct_weak_clear", - "opus_pct_cf", "nemotron_pct_cf", "opus_pct_ef", "nemotron_pct_ef", - ] - rows_out = [] - for (run_id, task_name), turns in sorted(groups.items()): - n = len(turns) - scores = [t["score"] for t in turns] - confs = [t["confidence"] for t in turns] - reward = turns[0]["reward"] - bands = [band(s, threshold) for s in scores] - n_sc = bands.count("strong_clear") - n_su = bands.count("strong_uncertain") - n_wu = bands.count("weak_uncertain") - n_wc = bands.count("weak_clear") - n_cf_opus = sum(1 for t in turns if t["pick_cf"] == CAPABLE) - n_ef_opus = sum(1 for t in turns if t["pick_ef"] == CAPABLE) - rows_out.append({ - "run_id": run_id, - "task_name": task_name, - "reward": reward, - "n_turns": n, - "mean_score": round(mean(scores), 4), - "mean_confidence": round(mean(confs), 4), - "pct_strong_clear": round(n_sc / n, 4), - "pct_strong_uncertain": round(n_su / n, 4), - "pct_weak_uncertain": round(n_wu / n, 4), - "pct_weak_clear": round(n_wc / n, 4), - "opus_pct_cf": round(n_cf_opus / n, 4), - "nemotron_pct_cf": round(1 - n_cf_opus / n, 4), - "opus_pct_ef": round(n_ef_opus / n, 4), - "nemotron_pct_ef": round(1 - n_ef_opus / n, 4), - }) - - csv_path.parent.mkdir(parents=True, exist_ok=True) - with open(csv_path, "w", newline="") as f: - w = csv.DictWriter(f, fieldnames=fieldnames) - w.writeheader() - w.writerows(rows_out) - print(f"Per-task CSV → {csv_path} ({len(rows_out)} rows)") - - -async def score_run(run_dir: Path, recent_window: int, threshold: float) -> list[dict]: - jobs_dirs = list(run_dir.glob("jobs/*/")) or [run_dir] - run_id = run_dir.name - - all_rows: list[dict] = [] - for jobs_dir in jobs_dirs: - task_dirs = [d for d in jobs_dir.iterdir() if d.is_dir() and d.name != "verifier"] - print(f"{jobs_dir.name}: {len(task_dirs)} task dirs") - - for task_dir in sorted(task_dirs): - traj_path = task_dir / "agent" / "trajectory.json" - result_path = task_dir / "result.json" - if not traj_path.exists(): - continue - - traj = json.loads(traj_path.read_text()) - reward: float | None = None - task_name = task_dir.name - if result_path.exists(): - res = json.loads(result_path.read_text()) - task_name = res.get("task_name") or task_name - r = (res.get("verifier_result") or {}).get("rewards", {}).get("reward") - reward = float(r) if r is not None else None - - rows = await score_trajectory( - traj, reward, task_name, task_dir.name, run_id, recent_window, threshold - ) - all_rows.extend(rows) - print(f" {task_dir.name}: {len(rows)} turns, reward={reward}") - - return all_rows - - -async def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--run", required=True, help="Path to run directory under benchmark/tb_runs/") - parser.add_argument("--output", help="Per-turn JSONL (default: /tmp/-scores.jsonl)") - parser.add_argument("--csv", help="Per-task CSV (default: /tmp/-per-task.csv)") - parser.add_argument("--threshold", type=float, default=0.20) - parser.add_argument("--window", type=int, default=RECENT_WINDOW) - args = parser.parse_args() - - run_dir = Path(args.run) - if not run_dir.exists(): - sys.exit(f"Run directory not found: {run_dir}") - - jsonl_out = Path(args.output or f"/tmp/{run_dir.name}-scores.jsonl") - csv_out = Path(args.csv or f"/tmp/{run_dir.name}-per-task.csv") - - all_rows = await score_run(run_dir, args.window, args.threshold) - - jsonl_out.parent.mkdir(parents=True, exist_ok=True) - with open(jsonl_out, "w") as fh: - for row in all_rows: - fh.write(json.dumps(row) + "\n") - print(f"Per-turn JSONL → {jsonl_out} ({len(all_rows)} rows)") - - if all_rows: - write_per_task_csv(all_rows, csv_out, args.threshold) - total = len(all_rows) - scores = [r["score"] for r in all_rows] - cf_opus = sum(1 for r in all_rows if r["pick_cf"] == CAPABLE) - ef_opus = sum(1 for r in all_rows if r["pick_ef"] == CAPABLE) - print(f"\nGlobal summary ({total} turns, threshold={args.threshold}):") - for bn in ["strong_clear", "strong_uncertain", "weak_uncertain", "weak_clear"]: - n = sum(1 for s in scores if band(s, args.threshold) == bn) - print(f" {bn:<22} {n:>5} ({100*n/total:.1f}%)") - print(f" cf → Opus: {cf_opus}/{total} ({100*cf_opus/total:.1f}%)") - print(f" ef → Opus: {ef_opus}/{total} ({100*ef_opus/total:.1f}%)") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index 0bede8d71..1db5943cf 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -671,8 +671,7 @@ fn set_json_model(body: &mut Value, model: &str) { // later turns from an Anthropic one. Clients such as Claude Code send // `context_management` on every turn, so the Anthropic leg must strip it or the // upstream rejects the request (for example `clear_thinking_20251015` strategy -// requires `thinking` to be enabled or adaptive). Mirrors -// `switchyard-components`' `strip_anthropic_incompatible_fields`. +// requires `thinking` to be enabled or adaptive). fn strip_anthropic_incompatible_fields(body: &mut Value) { if let Value::Object(object) = body { object.remove("reasoning_effort"); @@ -686,8 +685,7 @@ fn strip_anthropic_incompatible_fields(body: &mut Value) { // turns of a session from an OpenAI-format target whose thinking blocks are // unsigned, so the Anthropic leg must drop them or the upstream rejects the // request. Bedrock enforces this (surfacing as a SigV4 signature mismatch) where -// Azure-hosted Anthropic currently does not. Mirrors `switchyard-components`' -// `strip_unsigned_thinking_blocks`. +// Azure-hosted Anthropic currently does not. fn strip_unsigned_thinking_blocks(body: &mut Value) { let Value::Object(object) = body else { return; diff --git a/crates/libsy-llm-client/src/error.rs b/crates/libsy-llm-client/src/error.rs index b99850a45..0a1219da5 100644 --- a/crates/libsy-llm-client/src/error.rs +++ b/crates/libsy-llm-client/src/error.rs @@ -3,10 +3,8 @@ //! Canonical client error re-export and shared context-window-overflow detection. //! -//! The overflow detection is ported from -//! `switchyard-components/src/backends/context_overflow.rs`; that helper is -//! crate-private there and this crate cannot depend on `switchyard-components`, -//! so the small, self-contained logic is vendored here. +//! The client owns overflow detection so callers receive one stable error +//! classification across providers. use serde_json::Value; diff --git a/crates/switchyard-components/Cargo.toml b/crates/switchyard-components/Cargo.toml deleted file mode 100644 index 8775026ef..000000000 --- a/crates/switchyard-components/Cargo.toml +++ /dev/null @@ -1,32 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -[package] -name = "switchyard-components" -version.workspace = true -description = "Concrete Switchyard backends and processors" -authors.workspace = true -edition.workspace = true -license.workspace = true -repository.workspace = true -rust-version.workspace = true - -[dependencies] -async-stream.workspace = true -async-trait.workspace = true -futures-core = "0.3" -futures-util.workspace = true -parking_lot.workspace = true -rand.workspace = true -reqwest.workspace = true -serde.workspace = true -serde_json.workspace = true -switchyard-libsy.workspace = true -switchyard-protocol.workspace = true -switchyard-translation.workspace = true -tokio.workspace = true -thiserror.workspace = true -tracing.workspace = true - -[dev-dependencies] -tokio.workspace = true diff --git a/crates/switchyard-components/src/backends/anthropic.rs b/crates/switchyard-components/src/backends/anthropic.rs deleted file mode 100644 index 8fb611373..000000000 --- a/crates/switchyard-components/src/backends/anthropic.rs +++ /dev/null @@ -1,684 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Anthropic-compatible Messages backend. - -use std::collections::BTreeMap; -use std::env; -use std::fmt; -use std::sync::Arc; - -use crate::{ - BackendFormat, BoxResponseStream, ChatRequest, ChatRequestType, ChatResponse, LlmBackend, - LlmTarget, LlmTargetId, ProxyContext, Result, StreamEvent, SwitchyardError, - merge_target_extra_body, -}; -use async_stream::try_stream; -use async_trait::async_trait; -use futures_util::StreamExt; -use serde_json::{Map, Value, json}; -use switchyard_translation::{ - TranslationEngine, TranslationPolicy, WireFormat, normalize_anthropic_tool_use_ids, -}; - -use super::BackendSelection; -use super::common::{ - ParsedSseFrame, build_reqwest_client, decode_sse_frame, drain_next_sse_frame, - has_non_whitespace_bytes, parse_json_sse_frame, request_wire_format, set_json_model, - shared_translation_engine, -}; -use crate::telemetry::{SWITCHYARD_VERSION_HEADER, telemetry_header_value}; - -const DEFAULT_ANTHROPIC_BASE_URL: &str = "https://api.anthropic.com"; -const ANTHROPIC_API_KEY_ENV: &str = "ANTHROPIC_API_KEY"; -const ANTHROPIC_VERSION: &str = "2023-06-01"; -static ANTHROPIC_ONLY: [ChatRequestType; 1] = [ChatRequestType::Anthropic]; - -/// Backend that calls an Anthropic-compatible Messages API. -pub struct AnthropicNativeBackend { - /// Resolved target used for endpoint credentials and model rewriting. - target: LlmTarget, - /// HTTP transport, injectable for deterministic tests. - transport: Arc, - /// Shared request translator for non-Anthropic inbound payloads. - translation: Arc, - /// Translation policy kept explicit so backend behavior is inspectable. - translation_policy: TranslationPolicy, -} - -impl fmt::Debug for AnthropicNativeBackend { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("AnthropicNativeBackend") - .field("target", &self.target) - .finish_non_exhaustive() - } -} - -impl AnthropicNativeBackend { - /// Creates an Anthropic-compatible backend for one target. - pub fn new(target: LlmTarget) -> Result { - let transport = Arc::new(ReqwestAnthropicTransport::new( - target.endpoint.timeout_secs, - )?); - Self::with_transport(target, transport) - } - - /// Returns the configured upstream target. - pub fn target(&self) -> &LlmTarget { - &self.target - } - - fn with_transport(target: LlmTarget, transport: Arc) -> Result { - validate_target_format(&target)?; - Ok(Self { - target, - transport, - translation: shared_translation_engine(), - translation_policy: TranslationPolicy::default(), - }) - } - - fn outbound_body(&self, request: &ChatRequest) -> Result { - let mut body = match request.request_type() { - ChatRequestType::Anthropic => request.body().clone(), - source => { - self.translation - .translate_request( - request_wire_format(source), - WireFormat::AnthropicMessages, - request.body(), - &self.translation_policy, - ) - .map_err(|error| { - SwitchyardError::Backend(format!( - "failed to translate {source:?} request to Anthropic Messages: {error}" - )) - })? - .body - } - }; - set_json_model(&mut body, self.target.model.as_str()); - strip_anthropic_incompatible_fields(&mut body); - normalize_anthropic_body(&mut body); - // Per-target ``extra_body`` merged last; caller wins on key - // conflicts (see :func:`merge_target_extra_body`). - merge_target_extra_body(&mut body, self.target.extra_body.as_ref()); - Ok(body) - } - - /// Calls this target without requiring chain-local `ProxyContext` state. - pub async fn call_without_context(&self, request: &ChatRequest) -> Result { - let http_request = self.http_request(request)?; - self.send_http_request(http_request).await - } - - // Builds the upstream HTTP request before any context observations are recorded. - fn http_request(&self, request: &ChatRequest) -> Result { - let body = self.outbound_body(request)?; - let stream = body.get("stream").and_then(Value::as_bool).unwrap_or(false); - Ok(AnthropicHttpRequest { - target_id: self.target.id.clone(), - url: messages_url(self.target.endpoint.base_url.as_deref()), - api_key: anthropic_api_key(self.target.endpoint.api_key.as_deref()), - body, - stream, - extra_headers: self.target.extra_headers.clone(), - }) - } - - // Sends an already-normalized upstream request. - async fn send_http_request(&self, request: AnthropicHttpRequest) -> Result { - match self.transport.send(request).await? { - AnthropicHttpResponse::Buffered(body) => Ok(ChatResponse::anthropic_completion(body)), - AnthropicHttpResponse::Stream(stream) => Ok(ChatResponse::AnthropicStream(stream)), - } - } -} - -#[async_trait] -impl LlmBackend for AnthropicNativeBackend { - fn supported_request_types(&self) -> &[ChatRequestType] { - &ANTHROPIC_ONLY - } - - async fn call(&self, ctx: &mut ProxyContext, request: &ChatRequest) -> Result { - let http_request = self.http_request(request)?; - - ctx.inbound_format = ctx.inbound_format.or(Some(request.request_type())); - let previous_selection = ctx.get::().cloned(); - ctx.insert(BackendSelection::native_target_observation( - previous_selection.as_ref(), - self.target.id.clone(), - self.target.model.clone(), - request.model().map(str::to_string), - )); - - self.send_http_request(http_request).await - } -} - -#[derive(Clone, Debug, PartialEq)] -struct AnthropicHttpRequest { - /// Target ID used only for logging and diagnostics. - target_id: LlmTargetId, - /// Fully resolved Messages API URL. - url: String, - /// Per-target API key or process environment fallback. - api_key: Option, - /// Already-normalized Anthropic Messages request body. - body: Value, - /// Whether the upstream call should be treated as SSE. - stream: bool, - /// Per-target headers merged onto the outbound request. - extra_headers: BTreeMap, -} - -enum AnthropicHttpResponse { - /// Complete JSON response from a non-streaming upstream call. - Buffered(Value), - /// Streamed SSE response converted into Switchyard stream events. - Stream(BoxResponseStream), -} - -#[async_trait] -trait AnthropicTransport: Send + Sync { - /// Sends one already-normalized Anthropic Messages request. - async fn send(&self, request: AnthropicHttpRequest) -> Result; -} - -struct ReqwestAnthropicTransport { - /// Reused async HTTP client with configured timeout behavior. - client: reqwest::Client, -} - -impl ReqwestAnthropicTransport { - fn new(timeout_secs: Option) -> Result { - let client = build_reqwest_client("Anthropic", timeout_secs)?; - Ok(Self { client }) - } -} - -#[async_trait] -impl AnthropicTransport for ReqwestAnthropicTransport { - async fn send(&self, request: AnthropicHttpRequest) -> Result { - let target_id = request.target_id.clone(); - let mut builder = self - .client - .post(&request.url) - .header("anthropic-version", ANTHROPIC_VERSION) - .json(&request.body); - if let Some(api_key) = request.api_key { - builder = builder.header("x-api-key", api_key); - } - if let Some(version) = telemetry_header_value() { - builder = builder.header(SWITCHYARD_VERSION_HEADER, version); - } - for (name, value) in &request.extra_headers { - builder = builder.header(name, value); - } - - let response = builder.send().await.map_err(|error| { - tracing::warn!( - target_id = %target_id, - error = %error, - "Anthropic messages request failed" - ); - SwitchyardError::Upstream(format!("Anthropic messages request failed: {error}")) - })?; - let status = response.status(); - if !status.is_success() { - let body = response - .text() - .await - .unwrap_or_else(|error| format!("")); - tracing::warn!( - target_id = %target_id, - status = %status, - "Anthropic messages returned error status" - ); - if status == reqwest::StatusCode::BAD_REQUEST && is_context_overflow(&body) { - let model = request - .body - .get("model") - .and_then(Value::as_str) - .unwrap_or("") - .to_string(); - return Err(SwitchyardError::ContextWindowExceeded { - target_id: target_id.to_string(), - model, - message: body, - }); - } - return Err(SwitchyardError::UpstreamHttp { - provider: "Anthropic messages".to_string(), - status_code: status.as_u16(), - body, - }); - } - - if request.stream { - return Ok(AnthropicHttpResponse::Stream(anthropic_sse_stream( - response, - ))); - } - - let body = response.json::().await.map_err(|error| { - SwitchyardError::Upstream(format!("Anthropic messages returned invalid JSON: {error}")) - })?; - Ok(AnthropicHttpResponse::Buffered(body)) - } -} - -fn validate_target_format(target: &LlmTarget) -> Result<()> { - match target.format { - BackendFormat::Anthropic => Ok(()), - BackendFormat::Auto | BackendFormat::OpenAi | BackendFormat::Responses => { - Err(SwitchyardError::InvalidConfig(format!( - "AnthropicNativeBackend requires a target with resolved Anthropic format, got {:?} for {}", - target.format, target.id - ))) - } - } -} - -// Drop fields accepted by OpenAI-like APIs but rejected by Anthropic Messages. -fn strip_anthropic_incompatible_fields(body: &mut Value) { - if let Value::Object(object) = body { - object.remove("reasoning_effort"); - object.remove("context_management"); - } -} - -// Normalize translated Anthropic payloads before applying target overrides. -fn normalize_anthropic_body(body: &mut Value) { - let Value::Object(object) = body else { - return; - }; - if let Some(messages) = object.remove("messages") { - // AWS documents message-level `role: "system"` as an Opus 4.8-only - // Anthropic dialect: - // https://docs.aws.amazon.com/bedrock/latest/userguide/claude-messages-mid-conversation-system.html - // - // Keep the default conservative for older Bedrock/LiteLLM targets by - // lifting those turns into top-level `system`. - // TODO: add target-level Anthropic dialect support so - // Opus 4.8 can opt into mid-conversation system messages explicitly. - let (messages, system_text) = lift_message_level_system(messages); - append_lifted_system_text(object, system_text); - let messages = normalize_anthropic_tool_use_ids(messages); - object.insert( - "messages".to_string(), - strip_unsigned_thinking_blocks(messages), - ); - } -} - -// Moves Anthropic Opus-4.8-style message-level system turns out of the -// conversation so legacy Anthropic-compatible backends do not reject them. -fn lift_message_level_system(messages: Value) -> (Value, Vec) { - let Value::Array(messages) = messages else { - return (messages, Vec::new()); - }; - - let mut kept_messages = Vec::with_capacity(messages.len()); - let mut system_text = Vec::new(); - for message in messages { - if is_message_level_system(&message) { - if let Some(text) = system_text_from_message(&message) { - system_text.push(text); - } - } else { - kept_messages.push(message); - } - } - - (Value::Array(kept_messages), system_text) -} - -// Treats `system` and OpenAI/Codex `developer` roles as instruction-like -// turns. `developer` is not an Anthropic Opus 4.8 role; lifting it matches the -// existing OpenAI-to-Anthropic translator behavior and prevents malformed -// Anthropic-bound traffic from leaking an invalid role upstream. -fn is_message_level_system(message: &Value) -> bool { - matches!( - message.get("role").and_then(Value::as_str), - Some("system") | Some("developer") - ) -} - -// Extracts text from an invalid message-level system/developer turn. -fn system_text_from_message(message: &Value) -> Option { - message.get("content").and_then(system_text_from_content) -} - -// Converts Anthropic text-ish content into top-level system text. -fn system_text_from_content(content: &Value) -> Option { - match content { - Value::String(text) if !text.is_empty() => Some(text.clone()), - Value::String(_) | Value::Null => None, - Value::Array(blocks) => { - let parts = blocks - .iter() - .filter_map(system_text_from_content_block) - .collect::>(); - (!parts.is_empty()).then(|| parts.join("\n\n")) - } - other => Some(other.to_string()), - } -} - -// Extracts the supported text shape from one structured content block. -fn system_text_from_content_block(block: &Value) -> Option { - match block { - Value::String(text) if !text.is_empty() => Some(text.clone()), - Value::Object(object) => match object.get("type").and_then(Value::as_str) { - Some("text") | Some("input_text") => object - .get("text") - .and_then(Value::as_str) - .filter(|text| !text.is_empty()) - .map(ToOwned::to_owned), - // Message-level system/developer turns are downgraded for legacy Anthropic - // compatibility. Only text-like instruction content is replayed; images and - // other non-text blocks are intentionally not promoted into top-level system. - _ => None, - }, - _ => None, - } -} - -// Appends lifted message-level system text onto any existing Anthropic system field. -fn append_lifted_system_text(object: &mut Map, system_text: Vec) { - if system_text.is_empty() { - return; - } - - let joined = system_text.join("\n\n"); - match object.remove("system") { - None | Some(Value::Null) => { - object.insert("system".to_string(), Value::String(joined)); - } - Some(Value::String(existing)) if existing.is_empty() => { - object.insert("system".to_string(), Value::String(joined)); - } - Some(Value::String(existing)) => { - object.insert( - "system".to_string(), - Value::String(format!("{existing}\n\n{joined}")), - ); - } - Some(Value::Array(mut blocks)) => { - blocks.extend( - system_text - .into_iter() - .map(|text| json!({"type": "text", "text": text})), - ); - object.insert("system".to_string(), Value::Array(blocks)); - } - Some(other) => { - object.insert( - "system".to_string(), - Value::String(format!("{other}\n\n{joined}")), - ); - } - } -} - -// Anthropic requires signed thinking blocks on replay; remove unsigned blocks -// so passthrough and translated requests remain accepted by the API. -fn strip_unsigned_thinking_blocks(messages: Value) -> Value { - match messages { - Value::Array(messages) => Value::Array( - messages - .into_iter() - .map(strip_unsigned_thinking_from_message) - .collect(), - ), - other => other, - } -} - -fn strip_unsigned_thinking_from_message(message: Value) -> Value { - match message { - Value::Object(mut message) => { - let Some(content) = message.remove("content") else { - return Value::Object(message); - }; - let Value::Array(blocks) = content else { - message.insert("content".to_string(), content); - return Value::Object(message); - }; - - let kept = blocks - .into_iter() - .filter(|block| !is_unsigned_thinking_block(block)) - .collect::>(); - let content = if kept.is_empty() { - Value::String(String::new()) - } else { - Value::Array(kept) - }; - message.insert("content".to_string(), content); - Value::Object(message) - } - other => other, - } -} - -fn is_unsigned_thinking_block(block: &Value) -> bool { - if block.get("type").and_then(Value::as_str) != Some("thinking") { - return false; - } - !matches!( - block.get("signature").and_then(Value::as_str), - Some(signature) if !signature.is_empty() - ) -} - -fn messages_url(base_url: Option<&str>) -> String { - let base_url = base_url - .unwrap_or(DEFAULT_ANTHROPIC_BASE_URL) - .trim_end_matches('/'); - if base_url.ends_with("/v1/messages") { - base_url.to_string() - } else if base_url.ends_with("/v1") { - format!("{base_url}/messages") - } else { - format!("{base_url}/v1/messages") - } -} - -fn anthropic_api_key(configured: Option<&str>) -> Option { - // Resolve per call so long-lived backends can pick up rotated environment credentials. - configured - .map(str::to_string) - .or_else(|| env::var(ANTHROPIC_API_KEY_ENV).ok()) - .filter(|value| !value.trim().is_empty()) -} - -// Canonical Anthropic 4xx looks like -// `{"type":"error","error":{"type":"invalid_request_error","message":"prompt is too long: ..."}}`. -// Anthropic has no structured `error.code` field, so detection is phrase-based only. -const ANTHROPIC_OVERFLOW_PHRASES: &[&str] = &[ - "prompt is too long", - "maximum number of tokens", - "context window", - "context length", -]; - -fn is_context_overflow(body: &str) -> bool { - super::context_overflow::is_overflow_body(body, |_| false, ANTHROPIC_OVERFLOW_PHRASES) -} - -fn anthropic_sse_stream(response: reqwest::Response) -> BoxResponseStream { - Box::pin(try_stream! { - let mut chunks = response.bytes_stream(); - let mut buffer = Vec::new(); - - while let Some(chunk) = chunks.next().await { - let chunk = chunk.map_err(|error| { - SwitchyardError::Upstream(format!("Anthropic stream read failed: {error}")) - })?; - buffer.extend_from_slice(&chunk); - - // Anthropic SSE has named events, but the payload we care about is - // still the JSON `data:` line. - while let Some(frame) = drain_next_sse_frame(&mut buffer, "Anthropic")? { - match parse_json_sse_frame(&frame, "Anthropic", Some("[DONE]"))? { - ParsedSseFrame::Json(value) => yield StreamEvent::Json(value), - ParsedSseFrame::Done => return, - ParsedSseFrame::Empty => {} - } - } - } - - // Preserve the last frame when an upstream closes without a final SSE - // separator. - if has_non_whitespace_bytes(&buffer) { - let frame = decode_sse_frame(&buffer, "Anthropic")?; - match parse_json_sse_frame(&frame, "Anthropic", Some("[DONE]"))? { - ParsedSseFrame::Json(value) => yield StreamEvent::Json(value), - ParsedSseFrame::Done | ParsedSseFrame::Empty => {} - } - } - }) -} - -#[cfg(test)] -mod tests { - use crate::{EndpointConfig, LlmTargetId, ModelId}; - use parking_lot::Mutex; - use serde_json::json; - - use super::*; - - struct FakeAnthropicTransport { - requests: Mutex>, - response: Mutex>>, - } - - impl FakeAnthropicTransport { - fn with_error(message: &str) -> Self { - Self { - requests: Mutex::new(Vec::new()), - response: Mutex::new(Some(Err(SwitchyardError::Upstream(message.to_string())))), - } - } - } - - #[async_trait] - impl AnthropicTransport for FakeAnthropicTransport { - async fn send(&self, request: AnthropicHttpRequest) -> Result { - self.requests.lock().push(request); - self.response.lock().take().ok_or_else(|| { - SwitchyardError::Other("fake transport response already consumed".to_string()) - })? - } - } - - fn anthropic_target() -> LlmTarget { - LlmTarget { - id: LlmTargetId::from_static("primary"), - model: ModelId::from_static("target-claude"), - format: BackendFormat::Anthropic, - endpoint: EndpointConfig { - base_url: Some("https://example.test/v1".to_string()), - api_key: Some("secret".to_string()), - timeout_secs: None, - }, - extra_body: None, - extra_headers: BTreeMap::new(), - } - } - - #[tokio::test] - async fn transport_errors_are_backend_errors() -> Result<()> { - let transport = Arc::new(FakeAnthropicTransport::with_error("upstream exploded")); - let backend = AnthropicNativeBackend::with_transport(anthropic_target(), transport)?; - let request = ChatRequest::anthropic(json!({ - "model": "client-model", - "max_tokens": 128, - "messages": [], - })); - let mut ctx = ProxyContext::new(); - - let Err(error) = backend.call(&mut ctx, &request).await else { - return Err(SwitchyardError::Other( - "backend call should fail".to_string(), - )); - }; - - assert!(matches!(error, SwitchyardError::Upstream(_))); - assert!(error.to_string().contains("upstream exploded")); - Ok(()) - } - - #[test] - fn rejects_openai_targets() -> Result<()> { - let mut target = anthropic_target(); - target.format = BackendFormat::OpenAi; - - let Err(error) = AnthropicNativeBackend::new(target) else { - return Err(SwitchyardError::Other( - "OpenAI target should be rejected".to_string(), - )); - }; - - assert!(matches!(error, SwitchyardError::InvalidConfig(_))); - Ok(()) - } - - #[test] - fn formats_messages_urls_for_root_v1_and_explicit_paths() { - assert_eq!( - messages_url(Some("https://example.test")), - "https://example.test/v1/messages" - ); - assert_eq!( - messages_url(Some("https://example.test/v1")), - "https://example.test/v1/messages" - ); - assert_eq!( - messages_url(Some("https://example.test/v1/messages")), - "https://example.test/v1/messages" - ); - } - - #[test] - fn parses_anthropic_sse_json_frames() -> Result<()> { - let ParsedSseFrame::Json(value) = parse_json_sse_frame( - "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"m\"}}\n", - "Anthropic", - None, - )? - else { - return Err(SwitchyardError::Other( - "JSON frame should produce a JSON value".to_string(), - )); - }; - assert_eq!(value["type"], "message_start"); - assert!(matches!( - parse_json_sse_frame("event: ping\n\n", "Anthropic", None)?, - ParsedSseFrame::Empty - )); - Ok(()) - } - - #[test] - fn anthropic_context_overflow_canonical_shape_matches() { - let body = r#"{"type":"error","error":{"type":"invalid_request_error","message":"prompt is too long: 200001 tokens > 200000 maximum"}}"#; - assert!(is_context_overflow(body)); - } - - #[test] - fn anthropic_context_overflow_max_tokens_phrase_matches() { - let body = r#"{"type":"error","error":{"message":"exceeds the maximum number of tokens for this model"}}"#; - assert!(is_context_overflow(body)); - } - - #[test] - fn anthropic_context_overflow_unrelated_400_does_not_match() { - let body = r#"{"type":"error","error":{"type":"invalid_request_error","message":"missing system prompt"}}"#; - assert!(!is_context_overflow(body)); - } -} diff --git a/crates/switchyard-components/src/backends/common.rs b/crates/switchyard-components/src/backends/common.rs deleted file mode 100644 index fd4d1c4a5..000000000 --- a/crates/switchyard-components/src/backends/common.rs +++ /dev/null @@ -1,192 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Shared helpers for native backend implementations. - -use std::sync::{Arc, OnceLock}; -use std::time::Duration; - -use crate::{ChatRequestType, Result, SwitchyardError}; -use serde_json::{Map, Value}; -use switchyard_translation::{TranslationEngine, WireFormat}; - -pub(crate) enum ParsedSseFrame { - /// Frame contained a JSON payload. - Json(Value), - /// Frame contained the provider's terminal marker. - Done, - /// Frame had no data payload. - Empty, -} - -/// Returns the shared translation engine used by native backends. -pub(crate) fn shared_translation_engine() -> Arc { - static ENGINE: OnceLock> = OnceLock::new(); - Arc::clone(ENGINE.get_or_init(|| Arc::new(TranslationEngine::default()))) -} - -/// Builds a reqwest client with validated optional timeout. -pub(crate) fn build_reqwest_client( - backend_name: &str, - timeout_secs: Option, -) -> Result { - validate_timeout_secs(backend_name, timeout_secs)?; - let mut builder = reqwest::Client::builder(); - if let Some(timeout_secs) = timeout_secs { - builder = builder.timeout(Duration::from_secs_f64(timeout_secs)); - } - builder.build().map_err(|error| { - SwitchyardError::InvalidConfig(format!( - "failed to build {backend_name} HTTP client: {error}" - )) - }) -} - -/// Validates timeout values before they reach reqwest. -pub(crate) fn validate_timeout_secs(backend_name: &str, timeout_secs: Option) -> Result<()> { - if let Some(timeout_secs) = timeout_secs - && (!timeout_secs.is_finite() || timeout_secs <= 0.0) - { - return Err(SwitchyardError::InvalidConfig(format!( - "{backend_name} target timeout_secs must be finite and positive, got {timeout_secs:?}" - ))); - } - Ok(()) -} - -/// Maps a Switchyard request type to its wire format. -pub(crate) fn request_wire_format(request_type: ChatRequestType) -> WireFormat { - match request_type { - ChatRequestType::OpenAiChat => WireFormat::OpenAiChat, - ChatRequestType::OpenAiResponses => WireFormat::OpenAiResponses, - ChatRequestType::Anthropic => WireFormat::AnthropicMessages, - } -} - -/// Sets or creates the JSON `model` field for an outbound provider request. -pub(crate) fn set_json_model(body: &mut Value, model: &str) { - match body { - Value::Object(object) => { - object.insert("model".to_string(), Value::String(model.to_string())); - } - other => { - let mut object = Map::new(); - object.insert("model".to_string(), Value::String(model.to_string())); - *other = Value::Object(object); - } - } -} - -/// Drains one complete SSE frame from the buffer when a boundary is present. -pub(crate) fn drain_next_sse_frame( - buffer: &mut Vec, - backend_name: &str, -) -> Result> { - let Some((index, separator_len)) = next_sse_boundary(buffer) else { - return Ok(None); - }; - let frame = decode_sse_frame(&buffer[..index], backend_name)?; - buffer.drain(..index + separator_len); - Ok(Some(frame)) -} - -/// Decodes one raw SSE frame as UTF-8. -pub(crate) fn decode_sse_frame(frame: &[u8], backend_name: &str) -> Result { - std::str::from_utf8(frame) - .map(str::to_string) - .map_err(|error| { - SwitchyardError::Upstream(format!( - "{backend_name} stream emitted invalid UTF-8 frame: {error}" - )) - }) -} - -/// Returns whether the buffer has any non-whitespace bytes. -pub(crate) fn has_non_whitespace_bytes(buffer: &[u8]) -> bool { - buffer.iter().any(|byte| !byte.is_ascii_whitespace()) -} - -/// Parses data lines from one SSE frame into JSON, terminal, or empty states. -pub(crate) fn parse_json_sse_frame( - frame: &str, - backend_name: &str, - done_marker: Option<&str>, -) -> Result { - let mut data_lines = Vec::new(); - for line in frame.lines() { - // SSE comments and blank lines do not contribute data. - if line.is_empty() || line.starts_with(':') { - continue; - } - if let Some(data) = line.strip_prefix("data:") { - data_lines.push(data.trim_start().to_string()); - } - } - - if data_lines.is_empty() { - return Ok(ParsedSseFrame::Empty); - } - - let data = data_lines.join("\n"); - if done_marker.is_some_and(|marker| data.trim() == marker) { - return Ok(ParsedSseFrame::Done); - } - - let value = serde_json::from_str::(&data).map_err(|error| { - SwitchyardError::Upstream(format!( - "{backend_name} stream emitted invalid JSON frame: {error}" - )) - })?; - Ok(ParsedSseFrame::Json(value)) -} - -/// Finds the next CRLF or LF SSE frame boundary. -fn next_sse_boundary(buffer: &[u8]) -> Option<(usize, usize)> { - match (find_bytes(buffer, b"\r\n\r\n"), find_bytes(buffer, b"\n\n")) { - (Some(crlf), Some(lf)) if crlf < lf => Some((crlf, 4)), - (Some(_), Some(lf)) => Some((lf, 2)), - (Some(crlf), None) => Some((crlf, 4)), - (None, Some(lf)) => Some((lf, 2)), - (None, None) => None, - } -} - -/// Finds a byte needle inside a byte haystack. -fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option { - haystack - .windows(needle.len()) - .position(|window| window == needle) -} - -#[cfg(test)] -mod tests { - use serde_json::json; - - use super::*; - - // Multi-byte UTF-8 split across network chunks should wait for a full frame. - #[test] - fn buffers_incomplete_utf8_until_a_complete_sse_frame_arrives() -> Result<()> { - let mut buffer = b"data: {\"text\":\"".to_vec(); - let multibyte = "é".as_bytes(); - buffer.extend_from_slice(&multibyte[..1]); - assert!(drain_next_sse_frame(&mut buffer, "test")?.is_none()); - - buffer.extend_from_slice(&multibyte[1..]); - buffer.extend_from_slice(b"\"}\n\n"); - - let Some(frame) = drain_next_sse_frame(&mut buffer, "test")? else { - return Err(SwitchyardError::Other( - "complete SSE frame should be drained".to_string(), - )); - }; - let ParsedSseFrame::Json(value) = parse_json_sse_frame(&frame, "test", None)? else { - return Err(SwitchyardError::Other( - "SSE frame should parse as JSON".to_string(), - )); - }; - assert_eq!(value, json!({"text": "é"})); - assert!(buffer.is_empty()); - Ok(()) - } -} diff --git a/crates/switchyard-components/src/backends/context_overflow.rs b/crates/switchyard-components/src/backends/context_overflow.rs deleted file mode 100644 index 110a307a4..000000000 --- a/crates/switchyard-components/src/backends/context_overflow.rs +++ /dev/null @@ -1,92 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Shared detection for upstream 4xx context-window-overflow bodies. -//! -//! Provider-specific detectors (`openai`, `anthropic`) supply a structured -//! check (against the parsed error envelope) and a list of substring phrases. -//! The common shell parses the body once, runs the structured check, then -//! falls back to matching phrases against `error.message` or — if the body -//! isn't JSON — the raw body. Centralising the shape here means each new -//! provider-wrap (e.g. NVIDIA/LiteLLM's wrapping of the canonical OpenAI -//! error) is one-line phrase entry per provider, not a duplicated rewrite. - -use serde_json::Value; - -/// Detect a context-overflow body using a provider-supplied structured check -/// and substring phrase list. See module docs for the matching strategy. -pub(super) fn is_overflow_body(body: &str, structured_check: F, phrases: &[&str]) -> bool -where - F: Fn(&Value) -> bool, -{ - if let Ok(value) = serde_json::from_str::(body) { - if structured_check(&value) { - return true; - } - if let Some(message) = value - .get("error") - .and_then(|err| err.get("message")) - .and_then(Value::as_str) - && contains_any(message, phrases) - { - return true; - } - } - // Some upstream proxies return plain-text bodies; fall through to a - // string match on the raw body. - contains_any(body, phrases) -} - -fn contains_any(message: &str, phrases: &[&str]) -> bool { - let lower = message.to_ascii_lowercase(); - phrases.iter().any(|phrase| lower.contains(phrase)) -} - -#[cfg(test)] -mod tests { - use super::*; - - const PHRASES: &[&str] = &["context window", "too long"]; - - fn never(_value: &Value) -> bool { - false - } - - #[test] - fn structured_check_short_circuits() { - let body = r#"{"error":{"code":"context_length_exceeded","message":"unrelated"}}"#; - let matched = is_overflow_body( - body, - |value| { - value - .get("error") - .and_then(|err| err.get("code")) - .and_then(Value::as_str) - == Some("context_length_exceeded") - }, - &[], // empty phrases — structured check is the only path - ); - assert!(matched); - } - - #[test] - fn falls_back_to_message_phrase_match() { - let body = r#"{"error":{"message":"prompt too long"}}"#; - assert!(is_overflow_body(body, never, PHRASES)); - } - - #[test] - fn matches_plain_text_body() { - assert!(is_overflow_body( - "plain text mentioning context window", - never, - PHRASES - )); - } - - #[test] - fn non_match_returns_false() { - let body = r#"{"error":{"message":"rate limit exceeded"}}"#; - assert!(!is_overflow_body(body, never, PHRASES)); - } -} diff --git a/crates/switchyard-components/src/backends/mod.rs b/crates/switchyard-components/src/backends/mod.rs deleted file mode 100644 index 7d20b1011..000000000 --- a/crates/switchyard-components/src/backends/mod.rs +++ /dev/null @@ -1,18 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Built-in backend implementations. - -pub mod anthropic; -mod common; -mod context_overflow; -pub mod multi; -pub mod openai; -mod selection; -pub mod stats; - -pub use anthropic::AnthropicNativeBackend; -pub use multi::{LlmTargetBackend, MultiLlmBackend}; -pub use openai::{OpenAiNativeBackend, OpenAiPassthroughBackend}; -pub use selection::{BackendSelection, BackendSelectionReason}; -pub use stats::StatsLlmBackend; diff --git a/crates/switchyard-components/src/backends/multi.rs b/crates/switchyard-components/src/backends/multi.rs deleted file mode 100644 index 0180adea4..000000000 --- a/crates/switchyard-components/src/backends/multi.rs +++ /dev/null @@ -1,330 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Multi-target LLM backend dispatch. -//! -//! This backend owns the mechanical part of routing once a processor or caller -//! has selected a target: rewrite the request model, stamp typed context, and -//! delegate to the configured backend for that target. Selection policy stays -//! outside this type; processors and algorithms decide which target should run. - -use std::collections::HashSet; -use std::fmt; -use std::sync::Arc; - -use crate::{ - ChatRequest, ChatRequestType, ChatResponse, LlmBackend, LlmTarget, LlmTargetId, ProxyContext, - Result, SwitchyardError, -}; -use async_trait::async_trait; - -use super::{BackendSelection, BackendSelectionReason}; - -const DEFAULT_SUPPORTED_REQUEST_TYPES: [ChatRequestType; 3] = [ - ChatRequestType::OpenAiChat, - ChatRequestType::OpenAiResponses, - ChatRequestType::Anthropic, -]; - -/// One configured upstream target and the backend that can call it. -#[derive(Clone)] -pub struct LlmTargetBackend { - // Target metadata used for model rewriting and public stats. - target: LlmTarget, - // Backend implementation that executes calls for the target. - backend: Arc, -} - -impl LlmTargetBackend { - /// Creates a target/backend pair. - pub fn new(target: LlmTarget, backend: Arc) -> Self { - Self { target, backend } - } - - /// Returns the target metadata. - pub fn target(&self) -> &LlmTarget { - &self.target - } - - /// Returns the backend configured for the target. - pub fn backend(&self) -> &dyn LlmBackend { - self.backend.as_ref() - } -} - -impl fmt::Debug for LlmTargetBackend { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("LlmTargetBackend") - .field("target", &self.target) - .finish_non_exhaustive() - } -} - -/// Backend that delegates each request to one of several configured targets. -#[derive(Clone)] -pub struct MultiLlmBackend { - // Target/backends are stored in lifecycle order. - targets: Vec, - // Request formats advertised by this backend. - supported_request_types: Vec, - // Deterministic fallback when no router selected a target. - default_target_id: Option, -} - -impl MultiLlmBackend { - /// Creates a multi-target backend with support for all Switchyard request formats. - pub fn new(targets: impl IntoIterator) -> Result { - let targets = targets.into_iter().collect::>(); - validate_targets(&targets)?; - Ok(Self { - targets, - supported_request_types: DEFAULT_SUPPORTED_REQUEST_TYPES.to_vec(), - default_target_id: None, - }) - } - - /// Replaces the advertised request formats. - pub fn with_supported_request_types( - mut self, - request_types: impl IntoIterator, - ) -> Result { - self.supported_request_types = normalize_request_types(request_types)?; - Ok(self) - } - - /// Sets the target used when no router selected a target. - pub fn with_default_target(mut self, target_id: LlmTargetId) -> Result { - if self.target(&target_id).is_none() { - return Err(SwitchyardError::InvalidConfig(format!( - "default target {target_id} is not configured; known targets: {}", - self.known_target_ids() - ))); - } - self.default_target_id = Some(target_id); - Ok(self) - } - - /// Returns the deterministic default target, when configured. - pub fn default_target_id(&self) -> Option<&LlmTargetId> { - self.default_target_id.as_ref() - } - - /// Returns configured target/backend pairs in lifecycle order. - pub fn targets(&self) -> &[LlmTargetBackend] { - &self.targets - } - - /// Looks up a configured target by ID. - pub fn target(&self, target_id: &LlmTargetId) -> Option<&LlmTargetBackend> { - self.targets - .iter() - .find(|entry| &entry.target.id == target_id) - } - - fn selected_target<'a>( - &'a self, - ctx: &ProxyContext, - request: &ChatRequest, - ) -> Result<(&'a LlmTargetBackend, BackendSelectionReason)> { - // Explicit context selection wins because request processors are the - // routing policy layer. - if let Some(target_id) = ctx.selected_target() { - let Some(target) = self.target(target_id) else { - return Err(SwitchyardError::InvalidConfig(format!( - "selected target {target_id} is not configured; known targets: {}", - self.known_target_ids() - ))); - }; - return Ok((target, BackendSelectionReason::ContextTarget)); - } - - // A configured default only applies when no processor selected a target. - if let Some(target_id) = &self.default_target_id { - let Some(target) = self.target(target_id) else { - return Err(SwitchyardError::InvalidConfig(format!( - "default target {target_id} is not configured; known targets: {}", - self.known_target_ids() - ))); - }; - return Ok((target, BackendSelectionReason::DefaultTarget)); - } - - // A single-target backend is unambiguous even without a router. - if self.targets.len() == 1 { - let Some(target) = self.targets.first() else { - return Err(SwitchyardError::InvalidConfig( - "MultiLlmBackend requires at least one target".to_string(), - )); - }; - return Ok((target, BackendSelectionReason::SingleTarget)); - } - - // As a final convenience, match the request model to a unique target - // model. Duplicate matches remain ambiguous. - self.target_for_request_model(request) - } - - /// Selects a target by matching the request model to configured target models. - fn target_for_request_model<'a>( - &'a self, - request: &ChatRequest, - ) -> Result<(&'a LlmTargetBackend, BackendSelectionReason)> { - let Some(model) = request.model() else { - return Err(self.missing_selection_error(None)); - }; - - let matches = self - .targets - .iter() - .filter(|entry| entry.target.model.as_str() == model) - .collect::>(); - - match matches.as_slice() { - [target] => Ok((*target, BackendSelectionReason::RequestModel)), - [] => Err(self.missing_selection_error(Some(model))), - _ => Err(SwitchyardError::InvalidConfig(format!( - "request model {model:?} matches multiple targets; set selected_target explicitly" - ))), - } - } - - /// Builds the error used when no routing signal identifies one target. - fn missing_selection_error(&self, request_model: Option<&str>) -> SwitchyardError { - let request_model = request_model - .map(|model| format!(" and request model {model:?} did not match a configured target")) - .unwrap_or_default(); - SwitchyardError::InvalidConfig(format!( - "MultiLlmBackend has multiple targets but no selected target{request_model}; known targets: {}", - self.known_target_ids() - )) - } - - /// Formats configured target IDs for diagnostics. - fn known_target_ids(&self) -> String { - self.targets - .iter() - .map(|entry| entry.target.id.to_string()) - .collect::>() - .join(", ") - } -} - -impl fmt::Debug for MultiLlmBackend { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("MultiLlmBackend") - .field( - "targets", - &self - .targets - .iter() - .map(|entry| &entry.target) - .collect::>(), - ) - .field("supported_request_types", &self.supported_request_types) - .field("default_target_id", &self.default_target_id) - .finish() - } -} - -#[async_trait] -impl LlmBackend for MultiLlmBackend { - // Returns request formats this backend accepts before delegation. - fn supported_request_types(&self) -> &[ChatRequestType] { - &self.supported_request_types - } - - // Rewrites the request model to the selected target and delegates once. - async fn call(&self, ctx: &mut ProxyContext, request: &ChatRequest) -> Result { - let request_type = request.request_type(); - if !self.supported_request_types.contains(&request_type) { - return Err(SwitchyardError::UnsupportedRequestType { - component: "MultiLlmBackend".to_string(), - request_type, - }); - } - - let (target, reason) = self.selected_target(ctx, request)?; - let mut routed_request = request.clone(); - routed_request.set_model(target.target.model.as_str()); - - // Stamping BackendSelection lets stats processors attribute the final - // provider call without coupling to this backend's internals. - let _ = ctx.insert(BackendSelection::for_target( - target.target.id.clone(), - target.target.model.clone(), - request.model().map(str::to_string), - reason, - )); - - target.backend.call(ctx, &routed_request).await - } - - // Starts child backends in configured order and rolls back on failure. - async fn startup(&self) -> Result<()> { - let mut started: Vec<&LlmTargetBackend> = Vec::new(); - for target in &self.targets { - if let Err(error) = target.backend.startup().await { - for started_target in started.into_iter().rev() { - let _ = started_target.backend.shutdown().await; - } - return Err(error); - } - started.push(target); - } - Ok(()) - } - - // Shuts child backends down in reverse order while preserving first error. - async fn shutdown(&self) -> Result<()> { - let mut first_error = None; - for target in self.targets.iter().rev() { - if let Err(error) = target.backend.shutdown().await { - first_error.get_or_insert(error); - } - } - match first_error { - Some(error) => Err(error), - None => Ok(()), - } - } -} - -/// Validates constructor invariants for target lists. -fn validate_targets(targets: &[LlmTargetBackend]) -> Result<()> { - if targets.is_empty() { - return Err(SwitchyardError::InvalidConfig( - "MultiLlmBackend requires at least one target".to_string(), - )); - } - - let mut seen = HashSet::new(); - for target in targets { - if !seen.insert(target.target.id.clone()) { - return Err(SwitchyardError::InvalidConfig(format!( - "duplicate LLM target id: {}", - target.target.id - ))); - } - } - Ok(()) -} - -/// Deduplicates request types while preserving caller order. -fn normalize_request_types( - request_types: impl IntoIterator, -) -> Result> { - let mut normalized = Vec::new(); - for request_type in request_types { - if !normalized.contains(&request_type) { - normalized.push(request_type); - } - } - if normalized.is_empty() { - return Err(SwitchyardError::InvalidConfig( - "MultiLlmBackend must support at least one request type".to_string(), - )); - } - Ok(normalized) -} diff --git a/crates/switchyard-components/src/backends/openai.rs b/crates/switchyard-components/src/backends/openai.rs deleted file mode 100644 index 7cdaf5550..000000000 --- a/crates/switchyard-components/src/backends/openai.rs +++ /dev/null @@ -1,778 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! OpenAI-compatible backend for Chat Completions or Responses targets. - -use std::collections::BTreeMap; -use std::env; -use std::fmt; -use std::sync::Arc; - -use crate::{ - BackendFormat, BoxResponseStream, ChatRequest, ChatRequestType, ChatResponse, EndpointConfig, - LlmBackend, LlmTarget, LlmTargetId, ModelId, ProxyContext, Result, StreamEvent, - SwitchyardError, merge_target_extra_body, -}; -use async_stream::try_stream; -use async_trait::async_trait; -use futures_util::StreamExt; -use serde_json::{Map, Value}; -use switchyard_translation::{TranslationEngine, TranslationPolicy, WireFormat}; - -use super::common::{ - ParsedSseFrame, build_reqwest_client, decode_sse_frame, drain_next_sse_frame, - has_non_whitespace_bytes, parse_json_sse_frame, request_wire_format, set_json_model, - shared_translation_engine, -}; -use super::{BackendSelection, BackendSelectionReason}; -use crate::telemetry::{SWITCHYARD_VERSION_HEADER, telemetry_header_value}; - -const DEFAULT_OPENAI_BASE_URL: &str = "https://api.openai.com/v1"; -const OPENAI_API_KEY_ENV: &str = "OPENAI_API_KEY"; -static OPENAI_CHAT_ONLY: [ChatRequestType; 1] = [ChatRequestType::OpenAiChat]; -static OPENAI_RESPONSES_ONLY: [ChatRequestType; 1] = [ChatRequestType::OpenAiResponses]; -static OPENAI_PASSTHROUGH_TARGET_ID: &str = "passthrough"; - -/// Backend that calls an OpenAI-compatible Chat Completions or Responses API. -pub struct OpenAiNativeBackend { - /// Resolved target used for endpoint credentials and model rewriting. - target: LlmTarget, - /// HTTP transport, injectable for deterministic tests. - transport: Arc, - /// Shared request translator for non-OpenAI inbound payloads. - translation: Arc, - /// Translation policy kept explicit so future server policy remains visible. - translation_policy: TranslationPolicy, -} - -impl fmt::Debug for OpenAiNativeBackend { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("OpenAiNativeBackend") - .field("target", &self.target) - .finish_non_exhaustive() - } -} - -impl OpenAiNativeBackend { - /// Creates an OpenAI-compatible backend for one target. - pub fn new(target: LlmTarget) -> Result { - let transport = Arc::new(ReqwestOpenAiTransport::new(target.endpoint.timeout_secs)?); - Self::with_transport(target, transport) - } - - /// Returns the configured upstream target. - pub fn target(&self) -> &LlmTarget { - &self.target - } - - fn with_transport(target: LlmTarget, transport: Arc) -> Result { - validate_target_format(&target)?; - Ok(Self { - target, - transport, - translation: shared_translation_engine(), - translation_policy: TranslationPolicy::default(), - }) - } - - fn target_request_type(&self) -> ChatRequestType { - match self.target.format { - BackendFormat::OpenAi => ChatRequestType::OpenAiChat, - BackendFormat::Responses => ChatRequestType::OpenAiResponses, - BackendFormat::Auto | BackendFormat::Anthropic => { - unreachable!("OpenAiNativeBackend target format is validated at construction") - } - } - } - - fn target_wire_format(&self) -> WireFormat { - match self.target_request_type() { - ChatRequestType::OpenAiChat => WireFormat::OpenAiChat, - ChatRequestType::OpenAiResponses => WireFormat::OpenAiResponses, - ChatRequestType::Anthropic => { - unreachable!("OpenAiNativeBackend only targets OpenAI wire formats") - } - } - } - - fn outbound_body(&self, request: &ChatRequest) -> Result { - let target_request_type = self.target_request_type(); - let mut body = match request.request_type() { - source if source == target_request_type => request.body().clone(), - source => { - self.translation - .translate_request( - request_wire_format(source), - self.target_wire_format(), - request.body(), - &self.translation_policy, - ) - .map_err(|error| { - SwitchyardError::Backend(format!( - "failed to translate {source:?} request to {:?}: {error}", - self.target.format - )) - })? - .body - } - }; - set_json_model(&mut body, self.target.model.as_str()); - if self.target.format == BackendFormat::OpenAi { - ensure_stream_usage(&mut body); - } - // Merge per-target ``extra_body`` last so e.g. DeepSeek V4's - // ``chat_template_kwargs.enable_thinking=False`` reaches the - // upstream. Caller wins on key conflicts (see - // :func:`merge_target_extra_body`). - merge_target_extra_body(&mut body, self.target.extra_body.as_ref()); - Ok(body) - } - - /// Calls this target without requiring chain-local `ProxyContext` state. - pub async fn call_without_context(&self, request: &ChatRequest) -> Result { - let http_request = self.http_request(request)?; - self.send_http_request(http_request).await - } - - // Builds the upstream HTTP request before any context observations are recorded. - fn http_request(&self, request: &ChatRequest) -> Result { - let body = self.outbound_body(request)?; - let stream = body.get("stream").and_then(Value::as_bool).unwrap_or(false); - let endpoint = endpoint_for_backend_format(self.target.format)?; - Ok(OpenAiHttpRequest { - target_id: self.target.id.clone(), - url: openai_url(self.target.endpoint.base_url.as_deref(), endpoint), - api_key: openai_api_key(self.target.endpoint.api_key.as_deref()), - body, - stream, - extra_headers: self.target.extra_headers.clone(), - endpoint, - }) - } - - // Sends an already-normalized upstream request. - async fn send_http_request(&self, request: OpenAiHttpRequest) -> Result { - let endpoint = request.endpoint; - match self.transport.send(request).await? { - OpenAiHttpResponse::Buffered(body) => match endpoint { - OpenAiEndpoint::ChatCompletions => Ok(ChatResponse::openai_completion(body)), - OpenAiEndpoint::Responses => Ok(ChatResponse::openai_responses_completion(body)), - }, - OpenAiHttpResponse::Stream(stream) => match endpoint { - OpenAiEndpoint::ChatCompletions => Ok(ChatResponse::OpenAiStream(stream)), - OpenAiEndpoint::Responses => Ok(ChatResponse::OpenAiResponsesStream(stream)), - }, - } - } -} - -/// Backend that calls an OpenAI-compatible Chat Completions API without rewriting `model`. -pub struct OpenAiPassthroughBackend { - /// Endpoint used without an owning LLM target or model rewrite. - endpoint: EndpointConfig, - /// HTTP transport, injectable for tests. - transport: Arc, - /// Shared request translator for supported non-OpenAI inbound payloads. - translation: Arc, - /// Translation policy kept local to the backend. - translation_policy: TranslationPolicy, -} - -impl fmt::Debug for OpenAiPassthroughBackend { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("OpenAiPassthroughBackend") - .field("base_url", &self.endpoint.base_url) - .field("timeout_secs", &self.endpoint.timeout_secs) - .finish_non_exhaustive() - } -} - -impl OpenAiPassthroughBackend { - /// Creates a passthrough OpenAI-compatible backend. - pub fn new(endpoint: EndpointConfig) -> Result { - let transport = Arc::new(ReqwestOpenAiTransport::new(endpoint.timeout_secs)?); - Self::with_transport(endpoint, transport) - } - - /// Returns the configured upstream endpoint. - pub fn endpoint(&self) -> &EndpointConfig { - &self.endpoint - } - - fn with_transport( - endpoint: EndpointConfig, - transport: Arc, - ) -> Result { - Ok(Self { - endpoint, - transport, - translation: shared_translation_engine(), - translation_policy: TranslationPolicy::default(), - }) - } - - fn outbound_body(&self, request: &ChatRequest) -> Result { - let mut body = match request.request_type() { - ChatRequestType::OpenAiChat => request.body().clone(), - source => { - self.translation - .translate_request( - request_wire_format(source), - WireFormat::OpenAiChat, - request.body(), - &self.translation_policy, - ) - .map_err(|error| { - SwitchyardError::Backend(format!( - "failed to translate {source:?} request to OpenAI Chat: {error}" - )) - })? - .body - } - }; - ensure_stream_usage(&mut body); - Ok(body) - } -} - -#[async_trait] -impl LlmBackend for OpenAiNativeBackend { - fn supported_request_types(&self) -> &[ChatRequestType] { - match self.target.format { - BackendFormat::OpenAi => &OPENAI_CHAT_ONLY, - BackendFormat::Responses => &OPENAI_RESPONSES_ONLY, - BackendFormat::Auto | BackendFormat::Anthropic => { - unreachable!("OpenAiNativeBackend target format is validated at construction") - } - } - } - - async fn call(&self, ctx: &mut ProxyContext, request: &ChatRequest) -> Result { - let http_request = self.http_request(request)?; - - ctx.inbound_format = ctx.inbound_format.or(Some(request.request_type())); - let previous_selection = ctx.get::().cloned(); - ctx.insert(BackendSelection::native_target_observation( - previous_selection.as_ref(), - self.target.id.clone(), - self.target.model.clone(), - request.model().map(str::to_string), - )); - - self.send_http_request(http_request).await - } -} - -#[async_trait] -impl LlmBackend for OpenAiPassthroughBackend { - fn supported_request_types(&self) -> &[ChatRequestType] { - &OPENAI_CHAT_ONLY - } - - async fn call(&self, ctx: &mut ProxyContext, request: &ChatRequest) -> Result { - let body = self.outbound_body(request)?; - let stream = body.get("stream").and_then(Value::as_bool).unwrap_or(false); - let http_request = OpenAiHttpRequest { - target_id: LlmTargetId::from_static(OPENAI_PASSTHROUGH_TARGET_ID), - url: openai_url( - self.endpoint.base_url.as_deref(), - OpenAiEndpoint::ChatCompletions, - ), - api_key: openai_api_key(self.endpoint.api_key.as_deref()), - body, - stream, - extra_headers: BTreeMap::new(), - endpoint: OpenAiEndpoint::ChatCompletions, - }; - - ctx.inbound_format = ctx.inbound_format.or(Some(request.request_type())); - if let Some(model) = http_request - .body - .get("model") - .and_then(Value::as_str) - .and_then(|model| ModelId::new(model.to_string()).ok()) - { - ctx.insert(BackendSelection::for_model( - model, - request.model().map(str::to_string), - BackendSelectionReason::PassthroughModel, - )); - } - - match self.transport.send(http_request).await? { - OpenAiHttpResponse::Buffered(body) => Ok(ChatResponse::openai_completion(body)), - OpenAiHttpResponse::Stream(stream) => Ok(ChatResponse::OpenAiStream(stream)), - } - } -} - -#[derive(Clone, Debug, PartialEq)] -struct OpenAiHttpRequest { - /// Target ID used only for logging and diagnostics. - target_id: LlmTargetId, - /// Fully resolved OpenAI-compatible endpoint URL. - url: String, - /// Per-target API key or process environment fallback. - api_key: Option, - /// Already-normalized OpenAI-compatible request body. - body: Value, - /// Whether the upstream call should be treated as SSE. - stream: bool, - /// Per-target extra headers (e.g. ``X-Inference-Priority: batch`` - /// for NIH evals gateway routing on DeepSeek V4). Empty for the - /// passthrough backend, which has no LlmTarget. - extra_headers: BTreeMap, - /// Upstream endpoint family used for response wrapping and diagnostics. - endpoint: OpenAiEndpoint, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum OpenAiEndpoint { - /// OpenAI Chat Completions API. - ChatCompletions, - /// OpenAI Responses API. - Responses, -} - -impl OpenAiEndpoint { - fn label(self) -> &'static str { - match self { - Self::ChatCompletions => "OpenAI chat completions", - Self::Responses => "OpenAI responses", - } - } -} - -enum OpenAiHttpResponse { - /// Complete JSON response from a non-streaming upstream call. - Buffered(Value), - /// Streamed SSE response converted into Switchyard stream events. - Stream(BoxResponseStream), -} - -#[async_trait] -trait OpenAiTransport: Send + Sync { - /// Sends one already-normalized OpenAI-compatible request. - async fn send(&self, request: OpenAiHttpRequest) -> Result; -} - -struct ReqwestOpenAiTransport { - /// Reused async HTTP client with configured timeout behavior. - client: reqwest::Client, -} - -impl ReqwestOpenAiTransport { - fn new(timeout_secs: Option) -> Result { - let client = build_reqwest_client("OpenAI", timeout_secs)?; - Ok(Self { client }) - } -} - -#[async_trait] -impl OpenAiTransport for ReqwestOpenAiTransport { - async fn send(&self, request: OpenAiHttpRequest) -> Result { - let target_id = request.target_id.clone(); - let endpoint = request.endpoint; - let mut builder = self.client.post(&request.url).json(&request.body); - if let Some(api_key) = request.api_key { - builder = builder.bearer_auth(api_key); - } - if let Some(version) = telemetry_header_value() { - builder = builder.header(SWITCHYARD_VERSION_HEADER, version); - } - for (name, value) in &request.extra_headers { - builder = builder.header(name, value); - } - - let response = builder.send().await.map_err(|error| { - tracing::warn!( - target_id = %target_id, - error = %error, - endpoint = endpoint.label(), - "OpenAI request failed" - ); - SwitchyardError::Upstream(format!("{} request failed: {error}", endpoint.label())) - })?; - let status = response.status(); - if !status.is_success() { - let body = response - .text() - .await - .unwrap_or_else(|error| format!("")); - tracing::warn!( - target_id = %target_id, - status = %status, - endpoint = endpoint.label(), - "OpenAI request returned error status" - ); - if status == reqwest::StatusCode::BAD_REQUEST && is_context_overflow(&body) { - let model = request - .body - .get("model") - .and_then(Value::as_str) - .unwrap_or("") - .to_string(); - return Err(SwitchyardError::ContextWindowExceeded { - target_id: target_id.to_string(), - model, - message: body, - }); - } - return Err(SwitchyardError::UpstreamHttp { - provider: endpoint.label().to_string(), - status_code: status.as_u16(), - body, - }); - } - - if request.stream { - return Ok(OpenAiHttpResponse::Stream(openai_sse_stream(response))); - } - - let body = response.json::().await.map_err(|error| { - SwitchyardError::Upstream(format!( - "{} returned invalid JSON: {error}", - endpoint.label() - )) - })?; - Ok(OpenAiHttpResponse::Buffered(body)) - } -} - -fn validate_target_format(target: &LlmTarget) -> Result<()> { - match target.format { - BackendFormat::OpenAi | BackendFormat::Responses => Ok(()), - BackendFormat::Auto | BackendFormat::Anthropic => { - Err(SwitchyardError::InvalidConfig(format!( - "OpenAiNativeBackend requires a target with resolved OpenAI format, got {:?} for {}", - target.format, target.id - ))) - } - } -} - -fn endpoint_for_backend_format(format: BackendFormat) -> Result { - match format { - BackendFormat::OpenAi => Ok(OpenAiEndpoint::ChatCompletions), - BackendFormat::Responses => Ok(OpenAiEndpoint::Responses), - BackendFormat::Auto | BackendFormat::Anthropic => Err(SwitchyardError::InvalidConfig( - format!("OpenAiNativeBackend cannot dispatch target format {format:?}"), - )), - } -} - -// OpenAI-compatible streaming users need usage events for stats accounting. -fn ensure_stream_usage(body: &mut Value) { - let Value::Object(object) = body else { - return; - }; - if !object - .get("stream") - .and_then(Value::as_bool) - .unwrap_or(false) - { - return; - } - - match object.get_mut("stream_options") { - Some(Value::Object(options)) => { - options - .entry("include_usage".to_string()) - .or_insert(Value::Bool(true)); - } - _ => { - let mut options = Map::new(); - options.insert("include_usage".to_string(), Value::Bool(true)); - object.insert("stream_options".to_string(), Value::Object(options)); - } - } -} - -// Accept either a root `/v1` URL or an already-specific OpenAI endpoint URL. -fn openai_url(base_url: Option<&str>, endpoint: OpenAiEndpoint) -> String { - let base_url = base_url - .unwrap_or(DEFAULT_OPENAI_BASE_URL) - .trim_end_matches('/'); - let base_root = base_url - .strip_suffix("/chat/completions") - .or_else(|| base_url.strip_suffix("/responses")) - .unwrap_or(base_url); - let suffix = match endpoint { - OpenAiEndpoint::ChatCompletions => "/chat/completions", - OpenAiEndpoint::Responses => "/responses", - }; - format!("{base_root}{suffix}") -} - -fn openai_api_key(configured: Option<&str>) -> Option { - // Resolve per call so long-lived backends can pick up rotated environment credentials. - configured - .map(str::to_string) - .or_else(|| env::var(OPENAI_API_KEY_ENV).ok()) - .filter(|value| !value.trim().is_empty()) -} - -// Best-effort match for OpenAI-shape context-window-overflow error bodies. -// Canonical signal is `error.code == "context_length_exceeded"`; NVIDIA and -// other proxies sometimes only set the human-readable message, so we fall back -// to substring matching. Only reached for upstream 400s, and a false positive -// triggers a single bounded evict-and-retry (never an infinite loop), so erring -// toward matching the message is safe. -// OpenAI canonical phrase + NVIDIA/LiteLLM wrap variants. Adding a new -// provider-wrap is a one-line entry here, not a fork of the parsing logic. -const OPENAI_OVERFLOW_PHRASES: &[&str] = &[ - "maximum context length", - "context length exceeded", - "context window", - "context length is only", - "please reduce the length of the input", - "exceeds the maximum allowed input length", -]; - -fn is_context_overflow(body: &str) -> bool { - super::context_overflow::is_overflow_body( - body, - |value| { - value - .get("error") - .and_then(|err| err.get("code")) - .and_then(Value::as_str) - == Some("context_length_exceeded") - }, - OPENAI_OVERFLOW_PHRASES, - ) -} - -fn openai_sse_stream(response: reqwest::Response) -> BoxResponseStream { - Box::pin(try_stream! { - let mut chunks = response.bytes_stream(); - let mut buffer = Vec::new(); - - while let Some(chunk) = chunks.next().await { - let chunk = chunk.map_err(|error| { - SwitchyardError::Upstream(format!("OpenAI stream read failed: {error}")) - })?; - buffer.extend_from_slice(&chunk); - - // Drain complete frames immediately while preserving partial frames - // across TCP chunks. - while let Some(frame) = drain_next_sse_frame(&mut buffer, "OpenAI")? { - match parse_json_sse_frame(&frame, "OpenAI", Some("[DONE]"))? { - ParsedSseFrame::Json(value) => yield StreamEvent::Json(value), - ParsedSseFrame::Done => return, - ParsedSseFrame::Empty => {} - } - } - } - - // A non-standard upstream might omit the final double newline; parse a - // trailing complete frame instead of losing its usage chunk. - if has_non_whitespace_bytes(&buffer) { - let frame = decode_sse_frame(&buffer, "OpenAI")?; - match parse_json_sse_frame(&frame, "OpenAI", Some("[DONE]"))? { - ParsedSseFrame::Json(value) => yield StreamEvent::Json(value), - ParsedSseFrame::Done | ParsedSseFrame::Empty => {} - } - } - }) -} - -#[cfg(test)] -mod tests { - use crate::{EndpointConfig, LlmTargetId, ModelId}; - use parking_lot::Mutex; - use serde_json::json; - - use super::*; - - struct FakeOpenAiTransport { - requests: Mutex>, - response: Mutex>>, - } - - impl FakeOpenAiTransport { - fn with_error(message: &str) -> Self { - Self { - requests: Mutex::new(Vec::new()), - response: Mutex::new(Some(Err(SwitchyardError::Upstream(message.to_string())))), - } - } - } - - #[async_trait] - impl OpenAiTransport for FakeOpenAiTransport { - async fn send(&self, request: OpenAiHttpRequest) -> Result { - self.requests.lock().push(request); - self.response.lock().take().ok_or_else(|| { - SwitchyardError::Other("fake transport response already consumed".to_string()) - })? - } - } - - fn openai_target() -> LlmTarget { - LlmTarget { - id: LlmTargetId::from_static("primary"), - model: ModelId::from_static("target-model"), - format: BackendFormat::OpenAi, - endpoint: EndpointConfig { - base_url: Some("https://example.test/v1".to_string()), - api_key: Some("secret".to_string()), - timeout_secs: None, - }, - extra_body: None, - extra_headers: BTreeMap::new(), - } - } - - #[test] - fn outbound_body_merges_target_extra_body() -> Result<()> { - // Use-case: DeepSeek V4 on NVIDIA Inference Hub. The target sets - // ``chat_template_kwargs.enable_thinking=False`` so V4 skips its - // chain-of-thought pass. Without this the model 504s at -n 8 - // concurrency from the Hub gateway timeout. - let mut target = openai_target(); - target.extra_body = Some(json!({ - "chat_template_kwargs": {"enable_thinking": false} - })); - let transport = Arc::new(FakeOpenAiTransport::with_error("ignored")); - let backend = OpenAiNativeBackend::with_transport(target, transport)?; - let request = ChatRequest::openai_chat(json!({ - "model": "client-model", - "messages": [{"role": "user", "content": "hi"}], - })); - let body = backend.outbound_body(&request)?; - assert_eq!( - body.get("chat_template_kwargs"), - Some(&json!({"enable_thinking": false})), - "target.extra_body should land at the top level of the outbound body", - ); - // Caller fields preserved. - assert_eq!( - body.get("model").and_then(|v| v.as_str()), - Some("target-model"), - "outbound_body should rewrite model to the target's", - ); - Ok(()) - } - - #[test] - fn outbound_body_caller_wins_on_extra_body_key_conflict() -> Result<()> { - let mut target = openai_target(); - target.extra_body = Some(json!({ - "chat_template_kwargs": {"enable_thinking": false}, - "logit_bias": {"50256": -100}, - })); - let transport = Arc::new(FakeOpenAiTransport::with_error("ignored")); - let backend = OpenAiNativeBackend::with_transport(target, transport)?; - // Caller sets chat_template_kwargs explicitly with thinking=true. - let request = ChatRequest::openai_chat(json!({ - "model": "client-model", - "messages": [], - "chat_template_kwargs": {"enable_thinking": true}, - })); - let body = backend.outbound_body(&request)?; - // Caller-supplied chat_template_kwargs wins on the top-level key - // (the merge is shallow / caller-wins). Target's logit_bias - // still lands because caller didn't set it. - assert_eq!( - body.get("chat_template_kwargs"), - Some(&json!({"enable_thinking": true})), - ); - assert_eq!(body.get("logit_bias"), Some(&json!({"50256": -100})),); - Ok(()) - } - - #[tokio::test] - async fn transport_errors_are_backend_errors() -> Result<()> { - let transport = Arc::new(FakeOpenAiTransport::with_error("upstream exploded")); - let backend = OpenAiNativeBackend::with_transport(openai_target(), transport)?; - let request = ChatRequest::openai_chat(json!({ - "model": "client-model", - "messages": [], - })); - let mut ctx = ProxyContext::new(); - - let Err(error) = backend.call(&mut ctx, &request).await else { - return Err(SwitchyardError::Other( - "backend call should fail".to_string(), - )); - }; - - assert!(matches!(error, SwitchyardError::Upstream(_))); - assert!(error.to_string().contains("upstream exploded")); - Ok(()) - } - - #[test] - fn rejects_anthropic_targets() -> Result<()> { - let mut target = openai_target(); - target.format = BackendFormat::Anthropic; - - let Err(error) = OpenAiNativeBackend::new(target) else { - return Err(SwitchyardError::Other( - "Anthropic target should be rejected".to_string(), - )); - }; - - assert!(matches!(error, SwitchyardError::InvalidConfig(_))); - Ok(()) - } - - #[test] - fn parses_openai_sse_json_frames_and_done() -> Result<()> { - let ParsedSseFrame::Json(value) = parse_json_sse_frame( - "event: message\ndata: {\"choices\":[]}\n", - "OpenAI", - Some("[DONE]"), - )? - else { - return Err(SwitchyardError::Other( - "JSON frame should produce a JSON value".to_string(), - )); - }; - assert_eq!(value, json!({"choices": []})); - - let ParsedSseFrame::Done = - parse_json_sse_frame("data: [DONE]\n", "OpenAI", Some("[DONE]"))? - else { - return Err(SwitchyardError::Other("DONE frame should stop".to_string())); - }; - Ok(()) - } - - #[test] - fn context_overflow_canonical_code_matches() { - let body = r#"{"error":{"code":"context_length_exceeded","message":"x","type":"invalid_request_error"}}"#; - assert!(is_context_overflow(body)); - } - - #[test] - fn context_overflow_nvidia_message_matches() { - let body = r#"{"error":{"message":"This model's maximum context length is 131072 tokens, however you requested ..."}}"#; - assert!(is_context_overflow(body)); - } - - #[test] - fn context_overflow_unrelated_400_does_not_match() { - let body = r#"{"error":{"code":"invalid_api_key","message":"bad key"}}"#; - assert!(!is_context_overflow(body)); - } - - #[test] - fn context_overflow_hub_glm_matches() { - // Hub GLM error via LiteLLM: code is "400" (not context_length_exceeded), - // so detection relies on phrase matching. - let body = r#"{"error":{"message":"Input length 877338 exceeds the maximum allowed input length of 639968 tokens","code":"400"}}"#; - assert!(is_context_overflow(body)); - } - - #[test] - fn context_overflow_nvidia_litellm_wrap_matches() { - // Body shape observed from inference-api.nvidia.com's LiteLLM proxy - // wrapping a Nemotron context-window overflow. - let body = r#"{"error":{"message":"litellm.BadRequestError: OpenAIException - {\"error\":{\"message\":\"You passed 131041 input tokens and requested 32 output tokens. However, the model's context length is only 131072 tokens, resulting in a maximum input length of 131040 tokens. Please reduce the length of the input prompt. (parameter=input_tokens, value=131041)\",\"type\":\"BadRequestError\",\"param\":\"input_tokens\",\"code\":400}}","type":null,"param":null,"code":"400"}}"#; - assert!(is_context_overflow(body)); - } -} diff --git a/crates/switchyard-components/src/backends/selection.rs b/crates/switchyard-components/src/backends/selection.rs deleted file mode 100644 index 1416b5871..000000000 --- a/crates/switchyard-components/src/backends/selection.rs +++ /dev/null @@ -1,92 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Backend execution metadata for observability components. - -use crate::{LlmTargetId, ModelId}; -use serde::{Deserialize, Serialize}; - -/// How a backend resolved the final upstream target/model for a request. -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum BackendSelectionReason { - /// A router or caller set a selected target on `ProxyContext`. - ContextTarget, - /// The backend was configured with a deterministic default target. - DefaultTarget, - /// Only one target is configured, so there is no routing ambiguity. - SingleTarget, - /// The inbound request model uniquely matched a configured target model. - RequestModel, - /// A native backend has exactly one configured target. - NativeTarget, - /// A passthrough backend used the caller-provided model. - PassthroughModel, -} - -/// Final upstream backend selection for a request. -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] -pub struct BackendSelection { - /// Selected target ID when the backend resolved a concrete configured target. - pub target_id: Option, - /// Final upstream model name used for the provider call. - pub model: ModelId, - /// Client-provided model name before backend routing or rewriting. - pub original_model: Option, - /// Reason the backend selected this target/model. - pub reason: BackendSelectionReason, -} - -impl BackendSelection { - /// Creates a selection for a concrete target-backed backend call. - pub fn for_target( - target_id: LlmTargetId, - model: ModelId, - original_model: Option, - reason: BackendSelectionReason, - ) -> Self { - Self { - target_id: Some(target_id), - model, - original_model, - reason, - } - } - - /// Creates a selection for a backend call that only resolved a model. - pub fn for_model( - model: ModelId, - original_model: Option, - reason: BackendSelectionReason, - ) -> Self { - Self { - target_id: None, - model, - original_model, - reason, - } - } - - /// Records a native backend call while preserving an upstream routing reason - /// when a parent backend already selected the same target/model. - pub fn native_target_observation( - previous: Option<&Self>, - target_id: LlmTargetId, - model: ModelId, - original_model: Option, - ) -> Self { - let matching_previous = previous.filter(|selection| { - selection.target_id.as_ref() == Some(&target_id) && selection.model == model - }); - Self { - target_id: Some(target_id), - model, - original_model: matching_previous - .and_then(|selection| selection.original_model.clone()) - .or(original_model), - reason: matching_previous - .map(|selection| selection.reason) - .unwrap_or(BackendSelectionReason::NativeTarget), - } - } -} diff --git a/crates/switchyard-components/src/backends/stats.rs b/crates/switchyard-components/src/backends/stats.rs deleted file mode 100644 index 777f7026d..000000000 --- a/crates/switchyard-components/src/backends/stats.rs +++ /dev/null @@ -1,88 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Backend wrapper that records backend-call stats. - -use std::fmt; -use std::sync::Arc; -use std::time::Instant; - -use crate::{ChatRequest, ChatRequestType, ChatResponse, LlmBackend, ProxyContext, Result}; -use async_trait::async_trait; - -use crate::stats::{ - StatsAccumulator, StatsBackendLatency, selected_stats_model, selected_stats_tier, -}; - -/// Transparent backend wrapper that records call success/error and backend latency. -#[derive(Clone)] -pub struct StatsLlmBackend { - inner: Arc, - accumulator: StatsAccumulator, -} - -impl StatsLlmBackend { - /// Creates a stats wrapper around an existing backend. - pub fn new(inner: Arc, accumulator: StatsAccumulator) -> Self { - Self { inner, accumulator } - } - - /// Returns the wrapped backend. - pub fn inner(&self) -> &dyn LlmBackend { - self.inner.as_ref() - } - - /// Returns the shared accumulator. - pub fn accumulator(&self) -> &StatsAccumulator { - &self.accumulator - } -} - -impl fmt::Debug for StatsLlmBackend { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("StatsLlmBackend") - .field("accumulator", &self.accumulator) - .finish_non_exhaustive() - } -} - -#[async_trait] -impl LlmBackend for StatsLlmBackend { - fn supported_request_types(&self) -> &[ChatRequestType] { - self.inner.supported_request_types() - } - - async fn call(&self, ctx: &mut ProxyContext, request: &ChatRequest) -> Result { - let request_model = request.model().map(str::to_string); - let started_at = Instant::now(); - match self.inner.call(ctx, request).await { - Ok(response) => { - let latency = started_at.elapsed(); - ctx.insert(StatsBackendLatency(latency)); - let model = selected_stats_model(ctx, request_model.as_deref()); - let tier = selected_stats_tier(ctx); - self.accumulator.record_success( - model, - Some(latency.as_secs_f64() * 1000.0), - tier.as_deref(), - )?; - Ok(response) - } - Err(error) => { - let model = selected_stats_model(ctx, request_model.as_deref()); - let tier = selected_stats_tier(ctx); - self.accumulator.record_error(model, tier.as_deref())?; - Err(error) - } - } - } - - async fn startup(&self) -> Result<()> { - self.inner.startup().await - } - - async fn shutdown(&self) -> Result<()> { - self.inner.shutdown().await - } -} diff --git a/crates/switchyard-components/src/contracts/backend.rs b/crates/switchyard-components/src/contracts/backend.rs deleted file mode 100644 index 9beef8a1a..000000000 --- a/crates/switchyard-components/src/contracts/backend.rs +++ /dev/null @@ -1,145 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! LLM target configuration shared by compatibility routing and factory code. - -use std::collections::BTreeMap; - -use serde::{Deserialize, Serialize}; -use serde_json::Value; - -use super::ids::{LlmTargetId, ModelId}; - -/// Wire format expected by an LLM target. -#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum BackendFormat { - /// Format has not been resolved yet and must not reach native backend build. - #[default] - Auto, - /// OpenAI-compatible Chat Completions fallback wire format. - #[serde(rename = "openai")] - OpenAi, - /// OpenAI Responses API wire format for native `/v1/responses` targets. - Responses, - /// Anthropic Messages wire format. - Anthropic, -} - -/// Optional endpoint overrides for an LLM target. -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct EndpointConfig { - /// Optional upstream base URL, usually ending in `/v1`. - pub base_url: Option, - /// Optional upstream API key; environment fallback remains backend-specific. - pub api_key: Option, - /// Optional upstream request timeout in seconds. - pub timeout_secs: Option, -} - -impl EndpointConfig { - /// Merges a shared endpoint with target-local overrides. - /// - /// Fields set directly on the target win over fields inherited from the - /// shared endpoint definition. - pub fn with_overrides(&self, overrides: &Self) -> Self { - Self { - base_url: overrides.base_url.clone().or_else(|| self.base_url.clone()), - api_key: overrides.api_key.clone().or_else(|| self.api_key.clone()), - timeout_secs: overrides.timeout_secs.or(self.timeout_secs), - } - } -} - -/// A concrete upstream model target that routing processors can select. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct LlmTarget { - /// Stable target ID used by routers and config references. - pub id: LlmTargetId, - /// Upstream model name sent to the provider. - pub model: ModelId, - /// Native wire format expected by the upstream target. - pub format: BackendFormat, - /// Connection settings for the upstream target. - #[serde(default)] - pub endpoint: EndpointConfig, - /// Per-target outbound request extensions, merged into the request - /// body by the wire-specific backend before the upstream call. - /// - /// Use cases: - /// - /// * **`chat_template_kwargs.enable_thinking=False`** for DeepSeek - /// V4 (Flash / Pro) on NVIDIA Inference Hub. V4 is a chain-of- - /// thought model whose default reasoning blows past Hub's proxy - /// gateway timeout at ``-n 8`` concurrency (504s on ~5% of - /// requests); the flag disables thinking and pegs response - /// times at ~5 s flat. Cannot be set client-side because Hub - /// ignores the request-level ``reasoning_effort`` field for - /// these models. - /// * Provider-specific options (vLLM ``guided_json``, - /// ``logit_bias``, ``response_format`` shimming) that should - /// apply to *every* request to this target. - /// - /// Merge semantics are shallow and **caller-wins**: keys already - /// present in the inbound request body are not overridden. Set on - /// the target only what the caller is not expected to set itself. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub extra_body: Option, - /// Per-target outbound HTTP headers attached to every upstream - /// request issued for this target. - /// - /// Distinct from :attr:`extra_body` because some gateway-level - /// routing keys live in headers, not the request body. Example: - /// NVIDIA Inference Hub exposes an evals/benchmarking gateway - /// behind ``X-Inference-Priority: batch`` — required for DeepSeek - /// V4 calls during long benchmark sweeps because the regular - /// gateway enforces a ~6-min timeout that under ``-n 8`` - /// concurrency manifests as cascading 504s. - /// - /// Headers added here are appended to whatever the backend would - /// already send (``Authorization``, ``anthropic-version``, - /// telemetry). Reserved header names supplied by the backend - /// (``Authorization`` / ``x-api-key`` / ``anthropic-version``) - /// are still authoritative; ``extra_headers`` cannot override - /// them — the underlying ``reqwest`` builder appends each entry - /// rather than replacing existing ones, so a duplicate name - /// would create a multi-valued header rather than a - /// silent-override security hazard. - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub extra_headers: BTreeMap, -} - -impl LlmTarget { - /// Creates a target with automatic format detection and no endpoint overrides. - pub fn new(id: LlmTargetId, model: ModelId) -> Self { - Self { - id, - model, - format: BackendFormat::Auto, - endpoint: EndpointConfig::default(), - extra_body: None, - extra_headers: BTreeMap::new(), - } - } -} - -/// Shallow-merges ``target_extra`` into ``body``: keys already present -/// in ``body`` are preserved (caller wins); new keys from -/// ``target_extra`` are added. No-op if either side is not an object. -/// -/// Used by wire-specific backends to inject per-target -/// :attr:`LlmTarget.extra_body` into outbound request bodies without -/// stomping caller-supplied fields. -pub fn merge_target_extra_body(body: &mut Value, target_extra: Option<&Value>) { - let Some(Value::Object(extra)) = target_extra else { - return; - }; - let Value::Object(body_map) = body else { - return; - }; - for (key, value) in extra { - body_map.entry(key.clone()).or_insert_with(|| value.clone()); - } -} diff --git a/crates/switchyard-components/src/contracts/context.rs b/crates/switchyard-components/src/contracts/context.rs deleted file mode 100644 index f864508b6..000000000 --- a/crates/switchyard-components/src/contracts/context.rs +++ /dev/null @@ -1,216 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Per-request context and typed extension storage for compatibility components. - -use std::any::{Any, TypeId}; -use std::collections::{HashMap, HashSet}; -use std::fmt; - -use super::ids::{LlmTargetId, RequestId}; -use super::types::ChatRequestType; - -/// Mutable state shared across processors and the backend for one request. -#[derive(Default)] -pub struct ProxyContext { - /// Optional request ID propagated across processors and backends. - pub request_id: Option, - /// Optional inbound wire format recorded by endpoint or translation code. - pub inbound_format: Option, - /// Optional target selected by a request processor for backend dispatch. - pub selected_target: Option, - extensions: Extensions, -} - -impl ProxyContext { - /// Creates an empty request context. - pub fn new() -> Self { - Self::default() - } - - /// Creates a context with a known request identifier. - pub fn with_request_id(request_id: RequestId) -> Self { - Self { - request_id: Some(request_id), - ..Self::default() - } - } - - /// Returns read-only access to typed extension values. - pub fn extensions(&self) -> &Extensions { - &self.extensions - } - - /// Returns mutable access to typed extension values. - pub fn extensions_mut(&mut self) -> &mut Extensions { - &mut self.extensions - } - - /// Returns the selected target for backend dispatch. - pub fn selected_target(&self) -> Option<&LlmTargetId> { - self.selected_target.as_ref() - } - - /// Replaces the selected target, returning the previous target. - pub fn set_selected_target(&mut self, target_id: LlmTargetId) -> Option { - self.selected_target.replace(target_id) - } - - /// Clears the selected target. - pub fn clear_selected_target(&mut self) -> Option { - self.selected_target.take() - } - - /// Inserts a typed extension, returning the previous value of the same type. - pub fn insert(&mut self, value: T) -> Option - where - T: Send + Sync + 'static, - { - self.extensions.insert(value) - } - - /// Gets an immutable typed extension by Rust type. - pub fn get(&self) -> Option<&T> - where - T: Send + Sync + 'static, - { - self.extensions.get() - } - - /// Gets a mutable typed extension by Rust type. - pub fn get_mut(&mut self) -> Option<&mut T> - where - T: Send + Sync + 'static, - { - self.extensions.get_mut() - } - - /// Removes a typed extension and returns it when present. - pub fn remove(&mut self) -> Option - where - T: Send + Sync + 'static, - { - self.extensions.remove() - } -} - -impl fmt::Debug for ProxyContext { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("ProxyContext") - .field("request_id", &self.request_id) - .field("inbound_format", &self.inbound_format) - .field("selected_target", &self.selected_target) - .field("extensions_len", &self.extensions.len()) - .finish() - } -} - -/// Type-indexed storage used for cross-component request metadata. -#[derive(Default)] -pub struct Extensions { - values: HashMap>, -} - -impl Extensions { - /// Creates an empty extension map. - pub fn new() -> Self { - Self::default() - } - - /// Stores a value under its concrete Rust type. - pub fn insert(&mut self, value: T) -> Option - where - T: Send + Sync + 'static, - { - self.values - .insert(TypeId::of::(), Box::new(value)) - .and_then(|previous| previous.downcast::().ok()) - .map(|boxed| *boxed) - } - - /// Gets a value by its concrete Rust type. - pub fn get(&self) -> Option<&T> - where - T: Send + Sync + 'static, - { - self.values - .get(&TypeId::of::()) - .and_then(|value| value.downcast_ref()) - } - - /// Gets a mutable value by its concrete Rust type. - pub fn get_mut(&mut self) -> Option<&mut T> - where - T: Send + Sync + 'static, - { - self.values - .get_mut(&TypeId::of::()) - .and_then(|value| value.downcast_mut()) - } - - /// Removes a value by its concrete Rust type. - pub fn remove(&mut self) -> Option - where - T: Send + Sync + 'static, - { - self.values - .remove(&TypeId::of::()) - .and_then(|value| value.downcast::().ok()) - .map(|boxed| *boxed) - } - - /// Returns whether a value of type `T` is present. - pub fn contains(&self) -> bool - where - T: Send + Sync + 'static, - { - self.values.contains_key(&TypeId::of::()) - } - - /// Returns the number of stored extension values. - pub fn len(&self) -> usize { - self.values.len() - } - - /// Returns whether the extension map is empty. - pub fn is_empty(&self) -> bool { - self.values.is_empty() - } -} - -impl fmt::Debug for Extensions { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Extensions") - .field("len", &self.values.len()) - .finish() - } -} - -/// Targets evicted from a routing pool after a `ContextWindowExceeded` failure. -/// -/// Stored in `ProxyContext` for compatibility routers that retry the same -/// logical request on a fallback target after an upstream context-window error. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct EvictedTargets(HashSet); - -impl EvictedTargets { - /// Returns whether a target has already overflowed on this request. - pub fn contains(&self, target_id: &LlmTargetId) -> bool { - self.0.contains(target_id) - } - - /// Records a target as evicted, returning whether it was newly inserted. - pub fn insert(&mut self, target_id: LlmTargetId) -> bool { - self.0.insert(target_id) - } - - /// Returns whether no targets have been evicted yet. - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - - /// Iterate the evicted target ids in arbitrary order. - pub fn iter(&self) -> impl Iterator { - self.0.iter() - } -} diff --git a/crates/switchyard-components/src/contracts/error.rs b/crates/switchyard-components/src/contracts/error.rs deleted file mode 100644 index 115e3b92e..000000000 --- a/crates/switchyard-components/src/contracts/error.rs +++ /dev/null @@ -1,103 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Errors shared by compatibility backends and Python bindings. - -use thiserror::Error; - -use super::ids::{InvalidId, ModelId}; -use super::types::ChatRequestType; - -/// Result alias for core Switchyard operations. -pub type Result = std::result::Result; - -/// Shared error enum for configuration, processor, and backend failures. -#[derive(Debug, Error)] -pub enum SwitchyardError { - #[error("invalid configuration: {0}")] - InvalidConfig(String), - - #[error(transparent)] - InvalidId(#[from] InvalidId), - - #[error("{kind} {id:?} is already registered")] - DuplicateRegistration { kind: &'static str, id: String }, - - #[error("no model registered for {model}")] - ModelNotFound { model: ModelId }, - - #[error("{component} does not support request type {request_type:?}")] - UnsupportedRequestType { - component: String, - request_type: ChatRequestType, - }, - - // Client sent a structurally valid but semantically invalid request - // (e.g. an empty `messages` array). Surfaced as a 4xx, never a 5xx, - // so agents can distinguish a client bug from a transient server failure. - #[error("{0}")] - InvalidRequest(String), - - #[error("processor failed: {0}")] - Processor(String), - - #[error("backend failed: {0}")] - Backend(String), - - #[error("upstream failed: {0}")] - Upstream(String), - - #[error("upstream failed: {provider} returned HTTP {status_code}: {body}")] - UpstreamHttp { - provider: String, - status_code: u16, - body: String, - }, - - // Target hit its context window; routing runtime may evict + retry once. - #[error("context window exceeded on target {target_id} ({model}): {message}")] - ContextWindowExceeded { - target_id: String, - model: String, - message: String, - }, - - // Every target was evicted; no fallback left to satisfy the request. - #[error("context pool exhausted (last target {last_target_id}): {reason}")] - ContextPoolExhausted { - last_target_id: String, - reason: String, - }, - - #[error("{0}")] - Other(String), -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn context_window_exceeded_renders_target_and_model() { - let err = SwitchyardError::ContextWindowExceeded { - target_id: "weak".into(), - model: "nvidia/deepseek-ai/evals-deepseek-v4-pro".into(), - message: "prompt is too long".into(), - }; - let rendered = err.to_string(); - assert!(rendered.contains("weak")); - assert!(rendered.contains("evals-deepseek-v4-pro")); - assert!(rendered.contains("prompt is too long")); - } - - #[test] - fn context_pool_exhausted_renders_last_target() { - let err = SwitchyardError::ContextPoolExhausted { - last_target_id: "strong".into(), - reason: "all targets evicted".into(), - }; - let rendered = err.to_string(); - assert!(rendered.contains("strong")); - assert!(rendered.contains("all targets evicted")); - } -} diff --git a/crates/switchyard-components/src/contracts/ids.rs b/crates/switchyard-components/src/contracts/ids.rs deleted file mode 100644 index 90104c60f..000000000 --- a/crates/switchyard-components/src/contracts/ids.rs +++ /dev/null @@ -1,111 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Non-empty string identifiers used by compatibility component wiring. - -use std::fmt; -use std::str::FromStr; - -use serde::{Deserialize, Serialize}; -use thiserror::Error; - -/// Error returned when an identifier is empty or whitespace. -#[derive(Clone, Debug, Eq, Error, PartialEq)] -#[error("{kind} must not be empty")] -pub struct InvalidId { - kind: &'static str, -} - -impl InvalidId { - // Keeps the constructor private so only validated ID types can create this error. - fn empty(kind: &'static str) -> Self { - Self { kind } - } -} - -// Defines the repeated non-empty string ID behavior without runtime inheritance. -macro_rules! string_id { - ($name:ident) => { - #[doc = concat!("Validated non-empty identifier for `", stringify!($name), "`.")] - #[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] - #[serde(try_from = "String", into = "String")] - pub struct $name(String); - - impl $name { - const KIND: &'static str = stringify!($name); - - /// Creates a new identifier after rejecting empty or whitespace input. - pub fn new(value: impl Into) -> Result { - let value = value.into(); - if value.trim().is_empty() { - return Err(InvalidId::empty(Self::KIND)); - } - Ok(Self(value)) - } - - /// Creates an identifier from a compile-time string. - pub fn from_static(value: &'static str) -> Self { - match Self::new(value) { - Ok(id) => id, - Err(_) => panic!("static Switchyard IDs must not be empty"), - } - } - - /// Returns the identifier as a borrowed string. - pub fn as_str(&self) -> &str { - &self.0 - } - - /// Consumes the identifier and returns its owned string. - pub fn into_inner(self) -> String { - self.0 - } - } - - impl AsRef for $name { - fn as_ref(&self) -> &str { - self.as_str() - } - } - - impl TryFrom for $name { - type Error = InvalidId; - - fn try_from(value: String) -> Result { - Self::new(value) - } - } - - impl TryFrom<&str> for $name { - type Error = InvalidId; - - fn try_from(value: &str) -> Result { - Self::new(value) - } - } - - impl From<$name> for String { - fn from(value: $name) -> Self { - value.into_inner() - } - } - - impl FromStr for $name { - type Err = InvalidId; - - fn from_str(value: &str) -> Result { - Self::new(value) - } - } - - impl fmt::Display for $name { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(self.as_str()) - } - } - }; -} - -string_id!(LlmTargetId); -string_id!(ModelId); -string_id!(RequestId); diff --git a/crates/switchyard-components/src/contracts/mod.rs b/crates/switchyard-components/src/contracts/mod.rs deleted file mode 100644 index 4ef3dde00..000000000 --- a/crates/switchyard-components/src/contracts/mod.rs +++ /dev/null @@ -1,18 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Contracts retained by the compatibility component and Python layers. - -mod backend; -mod context; -mod error; -mod ids; -mod roles; -mod types; - -pub use backend::*; -pub use context::*; -pub use error::*; -pub use ids::*; -pub use roles::*; -pub use types::*; diff --git a/crates/switchyard-components/src/contracts/roles.rs b/crates/switchyard-components/src/contracts/roles.rs deleted file mode 100644 index 8ad074eca..000000000 --- a/crates/switchyard-components/src/contracts/roles.rs +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Backend trait shared by compatibility LLM callers. - -use async_trait::async_trait; - -use super::context::ProxyContext; -use super::error::Result; -use super::types::{ChatRequest, ChatRequestType, ChatResponse}; - -/// Backend abstraction responsible for making the LLM call. -#[async_trait] -pub trait LlmBackend: Send + Sync { - /// Returns the request formats this backend can accept directly. - fn supported_request_types(&self) -> &[ChatRequestType]; - - /// Calls the backend with the processed request. - async fn call(&self, ctx: &mut ProxyContext, request: &ChatRequest) -> Result; - - /// Starts any resources owned by the backend. - async fn startup(&self) -> Result<()> { - Ok(()) - } - - /// Stops any resources owned by the backend. - async fn shutdown(&self) -> Result<()> { - Ok(()) - } -} diff --git a/crates/switchyard-components/src/contracts/types.rs b/crates/switchyard-components/src/contracts/types.rs deleted file mode 100644 index c0c7e0e52..000000000 --- a/crates/switchyard-components/src/contracts/types.rs +++ /dev/null @@ -1,326 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Provider-agnostic request and response wrappers used by compatibility chains. - -use std::fmt; -use std::pin::Pin; - -use futures_core::Stream; -use serde::{Deserialize, Serialize}; -use serde_json::{Map, Value}; - -use super::error::{Result, SwitchyardError}; - -/// Supported inbound request wire formats. -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)] -pub enum ChatRequestType { - #[serde(rename = "openai_chat")] - OpenAiChat, - #[serde(rename = "openai_responses")] - OpenAiResponses, - #[serde(rename = "anthropic")] - Anthropic, -} - -/// JSON request body wrapper shared by all request variants. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct WireRequest { - body: Value, -} - -impl WireRequest { - /// Wraps an arbitrary JSON request body. - pub fn new(body: Value) -> Self { - Self { body } - } - - /// Returns the wrapped JSON request body. - pub fn body(&self) -> &Value { - &self.body - } - - /// Returns a mutable reference to the wrapped JSON request body. - pub fn body_mut(&mut self) -> &mut Value { - &mut self.body - } - - /// Consumes the wrapper and returns the JSON request body. - pub fn into_body(self) -> Value { - self.body - } -} - -/// Request body tagged by source API format. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(tag = "request_type", content = "request", rename_all = "snake_case")] -pub enum ChatRequest { - #[serde(rename = "openai_chat")] - OpenAiChat(WireRequest), - #[serde(rename = "openai_responses")] - OpenAiResponses(WireRequest), - #[serde(rename = "anthropic")] - Anthropic(WireRequest), -} - -impl ChatRequest { - /// Creates an OpenAI Chat Completions request. - pub fn openai_chat(body: Value) -> Self { - Self::OpenAiChat(WireRequest::new(body)) - } - - /// Creates an OpenAI Responses API request. - pub fn openai_responses(body: Value) -> Self { - Self::OpenAiResponses(WireRequest::new(body)) - } - - /// Creates an Anthropic Messages request. - pub fn anthropic(body: Value) -> Self { - Self::Anthropic(WireRequest::new(body)) - } - - /// Validates the request body before it enters the chain. - /// - /// Catches structurally valid but semantically invalid input so it - /// fails fast with a 4xx instead of reaching the backend and surfacing - /// as an opaque upstream 5xx. Currently rejects a present-but-empty - /// `messages` array on the message-based formats (OpenAI Chat, - /// Anthropic). Absent or non-array `messages` are left for the backend - /// to interpret; the Responses format carries `input`, not `messages`, - /// and is exempt. - pub fn validate(&self) -> Result<()> { - let checks_messages = matches!(self, Self::OpenAiChat(_) | Self::Anthropic(_)); - if checks_messages - && let Some(messages) = self.body().get("messages").and_then(Value::as_array) - && messages.is_empty() - { - return Err(SwitchyardError::InvalidRequest( - "messages must be a non-empty array".to_string(), - )); - } - Ok(()) - } - - /// Returns the request's tagged wire format. - pub fn request_type(&self) -> ChatRequestType { - match self { - Self::OpenAiChat(_) => ChatRequestType::OpenAiChat, - Self::OpenAiResponses(_) => ChatRequestType::OpenAiResponses, - Self::Anthropic(_) => ChatRequestType::Anthropic, - } - } - - /// Returns the request body regardless of source format. - pub fn body(&self) -> &Value { - match self { - Self::OpenAiChat(request) - | Self::OpenAiResponses(request) - | Self::Anthropic(request) => request.body(), - } - } - - /// Returns the mutable request body regardless of source format. - pub fn body_mut(&mut self) -> &mut Value { - match self { - Self::OpenAiChat(request) - | Self::OpenAiResponses(request) - | Self::Anthropic(request) => request.body_mut(), - } - } - - /// Reads the request's `model` field when present and string-valued. - pub fn model(&self) -> Option<&str> { - self.body().get("model").and_then(Value::as_str) - } - - /// Writes the request's `model` field, creating an object body if necessary. - pub fn set_model(&mut self, model: impl Into) { - match self.body_mut() { - Value::Object(body) => { - body.insert("model".to_string(), Value::String(model.into())); - } - body => { - let mut object = Map::new(); - object.insert("model".to_string(), Value::String(model.into())); - *body = Value::Object(object); - } - } - } -} - -/// Supported backend response wire shapes. -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)] -pub enum ChatResponseType { - #[serde(rename = "openai_completion")] - OpenAiCompletion, - #[serde(rename = "openai_stream")] - OpenAiStream, - #[serde(rename = "openai_responses_completion")] - OpenAiResponsesCompletion, - #[serde(rename = "openai_responses_stream")] - OpenAiResponsesStream, - #[serde(rename = "anthropic_completion")] - AnthropicCompletion, - #[serde(rename = "anthropic_stream")] - AnthropicStream, -} - -/// JSON response body wrapper shared by buffered response variants. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct WireResponse { - body: Value, -} - -impl WireResponse { - /// Wraps an arbitrary JSON response body. - pub fn new(body: Value) -> Self { - Self { body } - } - - /// Returns the wrapped JSON response body. - pub fn body(&self) -> &Value { - &self.body - } - - /// Consumes the wrapper and returns the JSON response body. - pub fn into_body(self) -> Value { - self.body - } -} - -/// Streaming event payload carried by streaming response variants. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(tag = "kind", content = "data", rename_all = "snake_case")] -pub enum StreamEvent { - Json(Value), - Text(String), -} - -/// Boxed async stream for backend response events. -pub type BoxResponseStream = Pin> + Send>>; - -/// Backend response tagged by provider format and buffered/streaming shape. -pub enum ChatResponse { - OpenAiCompletion(WireResponse), - OpenAiStream(BoxResponseStream), - OpenAiResponsesCompletion(WireResponse), - OpenAiResponsesStream(BoxResponseStream), - AnthropicCompletion(WireResponse), - AnthropicStream(BoxResponseStream), -} - -impl ChatResponse { - /// Creates a buffered OpenAI Chat Completions response. - pub fn openai_completion(body: Value) -> Self { - Self::OpenAiCompletion(WireResponse::new(body)) - } - - /// Creates a buffered OpenAI Responses API response. - pub fn openai_responses_completion(body: Value) -> Self { - Self::OpenAiResponsesCompletion(WireResponse::new(body)) - } - - /// Creates a buffered Anthropic Messages response. - pub fn anthropic_completion(body: Value) -> Self { - Self::AnthropicCompletion(WireResponse::new(body)) - } - - /// Returns the response's tagged wire shape. - pub fn response_type(&self) -> ChatResponseType { - match self { - Self::OpenAiCompletion(_) => ChatResponseType::OpenAiCompletion, - Self::OpenAiStream(_) => ChatResponseType::OpenAiStream, - Self::OpenAiResponsesCompletion(_) => ChatResponseType::OpenAiResponsesCompletion, - Self::OpenAiResponsesStream(_) => ChatResponseType::OpenAiResponsesStream, - Self::AnthropicCompletion(_) => ChatResponseType::AnthropicCompletion, - Self::AnthropicStream(_) => ChatResponseType::AnthropicStream, - } - } - - /// Returns the JSON body for buffered responses and `None` for streams. - pub fn body(&self) -> Option<&Value> { - match self { - Self::OpenAiCompletion(response) - | Self::OpenAiResponsesCompletion(response) - | Self::AnthropicCompletion(response) => Some(response.body()), - Self::OpenAiStream(_) | Self::OpenAiResponsesStream(_) | Self::AnthropicStream(_) => { - None - } - } - } -} - -impl fmt::Debug for ChatResponse { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::OpenAiCompletion(response) => f - .debug_tuple("OpenAiCompletion") - .field(response.body()) - .finish(), - Self::OpenAiStream(_) => f.debug_tuple("OpenAiStream").field(&"").finish(), - Self::OpenAiResponsesCompletion(response) => f - .debug_tuple("OpenAiResponsesCompletion") - .field(response.body()) - .finish(), - Self::OpenAiResponsesStream(_) => f - .debug_tuple("OpenAiResponsesStream") - .field(&"") - .finish(), - Self::AnthropicCompletion(response) => f - .debug_tuple("AnthropicCompletion") - .field(response.body()) - .finish(), - Self::AnthropicStream(_) => { - f.debug_tuple("AnthropicStream").field(&"").finish() - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn validate_rejects_empty_messages_for_openai_chat() { - let request = ChatRequest::openai_chat(json!({"model": "m", "messages": []})); - match request.validate() { - Err(SwitchyardError::InvalidRequest(message)) => { - assert!(message.contains("messages must be a non-empty array")); - } - other => panic!("expected InvalidRequest, got {other:?}"), - } - } - - #[test] - fn validate_rejects_empty_messages_for_anthropic() { - let request = ChatRequest::anthropic(json!({"model": "m", "messages": []})); - match request.validate() { - Err(SwitchyardError::InvalidRequest(_)) => {} - other => panic!("expected InvalidRequest, got {other:?}"), - } - } - - #[test] - fn validate_accepts_non_empty_messages() { - let request = ChatRequest::openai_chat( - json!({"model": "m", "messages": [{"role": "user", "content": "hi"}]}), - ); - assert!(request.validate().is_ok()); - } - - #[test] - fn validate_is_lenient_when_messages_absent() { - // Absent or non-array `messages` is left for the backend to interpret. - let request = ChatRequest::openai_chat(json!({"model": "m"})); - assert!(request.validate().is_ok()); - } - - #[test] - fn validate_exempts_responses_format() { - // The Responses format carries `input`, not `messages`. - let request = ChatRequest::openai_responses(json!({"model": "m", "input": "hi"})); - assert!(request.validate().is_ok()); - } -} diff --git a/crates/switchyard-components/src/dimension_collector/mod.rs b/crates/switchyard-components/src/dimension_collector/mod.rs deleted file mode 100644 index 95bdecaf0..000000000 --- a/crates/switchyard-components/src/dimension_collector/mod.rs +++ /dev/null @@ -1,18 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Signal extraction for the stage_router. -//! -//! Owns the pure logic for reading a coding-agent request's tool-call history -//! into a [`ToolResultSignal`] (write/edit/read counts, error severity, streaks), -//! plus response-side signals. The -//! [`crate::request_processors::dimension_collector`] module wraps it as a -//! request-side Switchyard component. - -pub mod response; -pub mod tool_signals; - -pub use response::{ResponseFlag, ResponseSignals, extract_response_signals}; -pub use tool_signals::{ - DEFAULT_RECENT_WINDOW, ToolResultSignal, extract_tool_signals, extract_tool_signals_with_window, -}; diff --git a/crates/switchyard-components/src/dimension_collector/response/checks.rs b/crates/switchyard-components/src/dimension_collector/response/checks.rs deleted file mode 100644 index fac3b0c1d..000000000 --- a/crates/switchyard-components/src/dimension_collector/response/checks.rs +++ /dev/null @@ -1,361 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Pure response-side checks emitted by [`super::extract_response_signals`]. -//! -//! Each check is a `fn(&ChatResponse) -> bool` operating on buffered JSON -//! bodies only — streams are treated as "no signal" upstream because they -//! can't be introspected without consuming the stream. Adding a streaming -//! variant is a separate effort (collect-as-you-go, emit at end-of-stream) -//! tracked outside the first-cut scope. -//! -//! Each fn is one OpenAI / Anthropic shape match. Keep them shape-aware -//! rather than running a uniform JSONPath over every wire — the wire -//! formats are different enough that uniform JSONPath would either be -//! lossy or 4x more code than direct shape access. - -use crate::ChatResponse; -use serde_json::Value; - -/// Returns `true` if any `tool_calls[].function.arguments` string is not -/// valid JSON. Operates on OpenAI-Chat and Anthropic responses. -/// -/// Mirrors the failure mode where a weak model emits a tool call shape -/// that looks well-formed at the top level but has malformed JSON in the -/// arguments field. Common with smaller models under tight token budgets. -pub fn is_malformed_tool_call(response: &ChatResponse) -> bool { - let Some(body) = response.body() else { - return false; - }; - if let Some(tool_calls) = openai_chat_tool_calls(body) { - return tool_calls_have_invalid_args(tool_calls); - } - if let Some(content_blocks) = anthropic_content_blocks(body) { - return anthropic_tool_use_blocks_have_invalid_args(content_blocks); - } - false -} - -/// Returns `true` if the response carries no content **and** no tool calls -/// **and** `finish_reason` is not the legitimate `"tool_calls"` / -/// `"tool_use"` sentinel. Distinguishes "model said nothing" from "model -/// finished with a tool call" — the second is normal, the first is a -/// quality failure. -pub fn is_empty_response(response: &ChatResponse) -> bool { - let Some(body) = response.body() else { - return false; - }; - if let Some(choice) = openai_first_choice(body) { - let message = choice.get("message").and_then(Value::as_object); - let content_empty = message - .and_then(|m| m.get("content")) - .map(content_is_empty) - .unwrap_or(true); - let tool_calls_empty = message - .and_then(|m| m.get("tool_calls")) - .and_then(Value::as_array) - .is_none_or(|calls| calls.is_empty()); - let finish_reason = choice - .get("finish_reason") - .and_then(Value::as_str) - .unwrap_or(""); - return content_empty - && tool_calls_empty - && finish_reason != "tool_calls" - && finish_reason != "tool_use"; - } - if let Some(blocks) = anthropic_content_blocks(body) { - let any_content = blocks.iter().any(|block| { - let kind = block.get("type").and_then(Value::as_str).unwrap_or(""); - kind == "text" || kind == "tool_use" - }); - let stop_reason = body - .get("stop_reason") - .and_then(Value::as_str) - .unwrap_or(""); - return !any_content && stop_reason != "tool_use"; - } - false -} - -/// Returns `true` if the response was truncated by the model-side -/// `max_tokens` budget — `finish_reason == "length"` on OpenAI Chat, -/// `stop_reason == "max_tokens"` on Anthropic. A common weak-model -/// failure mode under cost-pressured `max_tokens` caps. -pub fn is_truncated_completion(response: &ChatResponse) -> bool { - let Some(body) = response.body() else { - return false; - }; - if let Some(choice) = openai_first_choice(body) { - return choice - .get("finish_reason") - .and_then(Value::as_str) - .map(|reason| reason == "length") - .unwrap_or(false); - } - body.get("stop_reason") - .and_then(Value::as_str) - .map(|reason| reason == "max_tokens") - .unwrap_or(false) -} - -/// Returns `true` if any emitted tool call is missing a top-level required -/// field per the OpenAI Chat / Anthropic tool-call shape itself. -/// -/// Note: this checks *shape* requirements (must have `name`, must have -/// `arguments`/`input`), **not** the per-tool argument schemas. The -/// per-tool schema check requires the request's tool declarations and is -/// future work — the shape check catches the most common malformed -/// tool-call cases. -pub fn is_missing_required_args(response: &ChatResponse) -> bool { - let Some(body) = response.body() else { - return false; - }; - if let Some(tool_calls) = openai_chat_tool_calls(body) { - return tool_calls.iter().any(|call| { - let function = call.get("function").and_then(Value::as_object); - let has_name = function - .and_then(|f| f.get("name")) - .and_then(Value::as_str) - .is_some_and(|name| !name.is_empty()); - let has_args = function - .and_then(|f| f.get("arguments")) - .and_then(Value::as_str) - .is_some(); - !(has_name && has_args) - }); - } - if let Some(blocks) = anthropic_content_blocks(body) { - return blocks.iter().any(|block| { - let kind = block.get("type").and_then(Value::as_str).unwrap_or(""); - if kind != "tool_use" { - return false; - } - let has_name = block - .get("name") - .and_then(Value::as_str) - .is_some_and(|name| !name.is_empty()); - let has_input = block.get("input").is_some(); - !(has_name && has_input) - }); - } - false -} - -// ─── Shape-access helpers ──────────────────────────────────────────────── - -fn openai_first_choice(body: &Value) -> Option<&serde_json::Map> { - body.get("choices") - .and_then(Value::as_array) - .and_then(|choices| choices.first()) - .and_then(Value::as_object) -} - -fn openai_chat_tool_calls(body: &Value) -> Option<&Vec> { - openai_first_choice(body) - .and_then(|choice| choice.get("message")) - .and_then(Value::as_object) - .and_then(|message| message.get("tool_calls")) - .and_then(Value::as_array) -} - -fn anthropic_content_blocks(body: &Value) -> Option<&Vec> { - body.get("content").and_then(Value::as_array) -} - -fn tool_calls_have_invalid_args(tool_calls: &[Value]) -> bool { - tool_calls.iter().any(|call| { - let Some(args) = call - .get("function") - .and_then(|function| function.get("arguments")) - .and_then(Value::as_str) - else { - // Missing arguments is `is_missing_required_args`'s concern; - // here we only flag *malformed JSON* in the provided string. - return false; - }; - // Empty-string arguments are convention for zero-arg tool calls; - // treat as well-formed (`{}` equivalent). - if args.is_empty() { - return false; - } - serde_json::from_str::(args).is_err() - }) -} - -fn anthropic_tool_use_blocks_have_invalid_args(blocks: &[Value]) -> bool { - // Anthropic emits `input` as already-parsed JSON, not a string, so - // there's no malformed-string failure mode equivalent to OpenAI's. - // Flagged only if `input` is a string that fails to parse as JSON, - // which is non-standard but defensive. - blocks.iter().any(|block| { - let kind = block.get("type").and_then(Value::as_str).unwrap_or(""); - if kind != "tool_use" { - return false; - } - let Some(input) = block.get("input") else { - return false; - }; - match input { - Value::String(s) if !s.is_empty() => serde_json::from_str::(s).is_err(), - _ => false, - } - }) -} - -fn content_is_empty(content: &Value) -> bool { - match content { - Value::Null => true, - Value::String(s) => s.trim().is_empty(), - Value::Array(parts) => parts.iter().all(|part| { - part.as_object() - .and_then(|object| object.get("text")) - .and_then(Value::as_str) - .is_none_or(|text| text.trim().is_empty()) - }), - _ => false, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - fn openai_chat(body: Value) -> ChatResponse { - ChatResponse::openai_completion(body) - } - - fn anthropic(body: Value) -> ChatResponse { - ChatResponse::anthropic_completion(body) - } - - #[test] - fn malformed_tool_call_openai_chat() { - let bad = openai_chat(json!({ - "choices": [{ - "message": { - "tool_calls": [{ - "function": { - "name": "get_weather", - "arguments": "{\"city\": \"new york\"" // missing closing brace - } - }] - } - }] - })); - assert!(is_malformed_tool_call(&bad)); - - let good = openai_chat(json!({ - "choices": [{ - "message": { - "tool_calls": [{ - "function": { - "name": "get_weather", - "arguments": "{\"city\": \"new york\"}" - } - }] - } - }] - })); - assert!(!is_malformed_tool_call(&good)); - } - - #[test] - fn empty_arguments_string_is_well_formed() { - let resp = openai_chat(json!({ - "choices": [{ - "message": { - "tool_calls": [{ - "function": { "name": "ping", "arguments": "" } - }] - } - }] - })); - assert!(!is_malformed_tool_call(&resp)); - } - - #[test] - fn empty_response_distinguishes_from_tool_call_finish() { - let empty = openai_chat(json!({ - "choices": [{ - "message": { "content": null }, - "finish_reason": "stop" - }] - })); - assert!(is_empty_response(&empty)); - - let tool_call_finish = openai_chat(json!({ - "choices": [{ - "message": { - "content": null, - "tool_calls": [{ - "function": { "name": "x", "arguments": "{}" } - }] - }, - "finish_reason": "tool_calls" - }] - })); - assert!(!is_empty_response(&tool_call_finish)); - } - - #[test] - fn truncated_completion_finish_reason_length() { - let truncated = openai_chat(json!({ - "choices": [{ - "message": { "content": "this got cut off..." }, - "finish_reason": "length" - }] - })); - assert!(is_truncated_completion(&truncated)); - - let ok = openai_chat(json!({ - "choices": [{ - "message": { "content": "all done" }, - "finish_reason": "stop" - }] - })); - assert!(!is_truncated_completion(&ok)); - } - - #[test] - fn truncated_anthropic_stop_reason_max_tokens() { - let truncated = anthropic(json!({ - "content": [{ "type": "text", "text": "got cut off" }], - "stop_reason": "max_tokens" - })); - assert!(is_truncated_completion(&truncated)); - } - - #[test] - fn missing_required_args_flags_nameless_or_argless_tool_calls() { - let no_name = openai_chat(json!({ - "choices": [{ - "message": { - "tool_calls": [{ "function": { "arguments": "{}" } }] - } - }] - })); - assert!(is_missing_required_args(&no_name)); - - let no_args = openai_chat(json!({ - "choices": [{ - "message": { - "tool_calls": [{ "function": { "name": "ping" } }] - } - }] - })); - assert!(is_missing_required_args(&no_args)); - - let well_formed = openai_chat(json!({ - "choices": [{ - "message": { - "tool_calls": [{ - "function": { "name": "ping", "arguments": "{}" } - }] - } - }] - })); - assert!(!is_missing_required_args(&well_formed)); - } -} diff --git a/crates/switchyard-components/src/dimension_collector/response/mod.rs b/crates/switchyard-components/src/dimension_collector/response/mod.rs deleted file mode 100644 index 685b185ef..000000000 --- a/crates/switchyard-components/src/dimension_collector/response/mod.rs +++ /dev/null @@ -1,128 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Response-side context signals. -//! -//! Sibling module to the request-side dimension collector. Same character: -//! deterministic, no estimator needed, calibration-friendly. Each -//! [`ResponseFlag`] is the binary outcome of one pure check against a -//! `ChatResponse` body. -//! -//! [`ResponseSignalCollector`] (in `request_processors/`) wraps this as a -//! response-side adapter; the pure logic lives here so it's -//! independently unit-testable and reusable by future routers that want -//! to inspect response quality without going through the full chain. - -pub mod checks; - -use crate::ChatResponse; -use serde::{Deserialize, Serialize}; - -/// Aggregate of response-side quality flags emitted by [`extract_response_signals`]. -/// -/// Stamped into `ProxyContext.extensions` by the -/// `ResponseSignalCollector` adapter. Empty `flags` means all checks -/// passed — the response is considered acceptable from the stage-router -/// router's reactive-escalation viewpoint. -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct ResponseSignals { - pub flags: Vec, -} - -impl ResponseSignals { - /// Returns `true` when at least one check flagged the response. - /// - /// StageRouter routers consume this as the per-attempt acceptability - /// gate; a `true` here triggers escalation to the next tier. - pub fn has_failures(&self) -> bool { - !self.flags.is_empty() - } - - /// Returns `true` if the given flag is present. - pub fn contains(&self, flag: ResponseFlag) -> bool { - self.flags.contains(&flag) - } -} - -/// Closed set of response-side quality failures emitted by the -/// dimension-collector response layer. -/// -/// Adding a new flag is a deliberate API change: downstream estimators -/// match on this enum, and growth here is observable in their behavior. -/// Keep it small. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ResponseFlag { - /// `tool_calls[].function.arguments` doesn't parse as JSON. - MalformedToolCallJson, - /// No `content` and no `tool_calls`; not a legitimate - /// `tool_calls`-finish. - EmptyResponse, - /// `finish_reason == "length"` (OpenAI) or - /// `stop_reason == "max_tokens"` (Anthropic). - TruncatedCompletion, - /// A tool call is missing `name` or `arguments` at the shape level - /// (not per-tool-schema; that's future work). - MissingRequiredArgs, -} - -/// Runs all four checks against a buffered `ChatResponse` and returns -/// the set of failing flags. -/// -/// Streaming responses (where [`ChatResponse::body`] returns `None`) -/// short-circuit to an empty `ResponseSignals` — streams can't be -/// introspected without consuming them. A streaming-aware checker is -/// future work outside the first cut. -pub fn extract_response_signals(response: &ChatResponse) -> ResponseSignals { - if response.body().is_none() { - return ResponseSignals::default(); - } - let mut flags = Vec::new(); - if checks::is_malformed_tool_call(response) { - flags.push(ResponseFlag::MalformedToolCallJson); - } - if checks::is_empty_response(response) { - flags.push(ResponseFlag::EmptyResponse); - } - if checks::is_truncated_completion(response) { - flags.push(ResponseFlag::TruncatedCompletion); - } - if checks::is_missing_required_args(response) { - flags.push(ResponseFlag::MissingRequiredArgs); - } - ResponseSignals { flags } -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn extract_returns_empty_for_well_formed_response() { - let resp = ChatResponse::openai_completion(json!({ - "choices": [{ - "message": { "content": "ok" }, - "finish_reason": "stop" - }] - })); - let signals = extract_response_signals(&resp); - assert!(signals.flags.is_empty()); - assert!(!signals.has_failures()); - } - - #[test] - fn extract_aggregates_multiple_failures() { - // Empty content + truncated finish at the same time. - let resp = ChatResponse::openai_completion(json!({ - "choices": [{ - "message": { "content": "" }, - "finish_reason": "length" - }] - })); - let signals = extract_response_signals(&resp); - assert!(signals.has_failures()); - assert!(signals.contains(ResponseFlag::EmptyResponse)); - assert!(signals.contains(ResponseFlag::TruncatedCompletion)); - } -} diff --git a/crates/switchyard-components/src/dimension_collector/tool_signals.rs b/crates/switchyard-components/src/dimension_collector/tool_signals.rs deleted file mode 100644 index 895af3f7b..000000000 --- a/crates/switchyard-components/src/dimension_collector/tool_signals.rs +++ /dev/null @@ -1,52 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Tool-result context signals — thin adapter over libsy's extractor. -//! -//! The extraction logic lives in [`switchyard_libsy::ToolSignals`]. This module -//! bridges the crate's [`ChatRequest`] (a format-tagged JSON body) to libsy's -//! [`switchyard_protocol::Request`] (raw body + wire-format metadata) so the two -//! request models share one implementation. - -use crate::{ChatRequest, ChatRequestType}; -use switchyard_protocol::{Metadata, Request, WireFormat}; - -/// The tool-signal output type. Re-exported from libsy so downstream consumers -/// (the `ToolResultSignal` stamped on `ProxyContext`) see a single type. -pub use switchyard_libsy::{DEFAULT_RECENT_WINDOW, ToolSignals as ToolResultSignal}; - -/// Adapt a format-tagged [`ChatRequest`] to a [`switchyard_protocol::Request`] -/// carrying the raw body and its wire format, which is all libsy's extractor reads. -fn to_protocol_request(request: &ChatRequest) -> Request { - let wire_format = match request.request_type() { - ChatRequestType::OpenAiChat => WireFormat::OpenAiChat, - ChatRequestType::Anthropic => WireFormat::AnthropicMessages, - ChatRequestType::OpenAiResponses => WireFormat::OpenAiResponses, - }; - // The signals are read off the decoded conversation, so decode here rather - // than handing over a raw body the extractor cannot interpret. A body that - // fails to decode yields no signals, which is what an absent body did before. - let llm_request = - switchyard_translation::decode_request(wire_format, request.body()).unwrap_or_default(); - Request { - llm_request, - raw_request: Some(request.body().clone()), - metadata: Some(Metadata { - wire_format: Some(wire_format), - ..Default::default() - }), - } -} - -/// Extract tool-execution signals using the default `recent_*` window. -pub fn extract_tool_signals(request: &ChatRequest) -> ToolResultSignal { - ToolResultSignal::from_request(&to_protocol_request(request), None) -} - -/// Extract tool-execution signals with a caller-supplied `recent_*` window. -pub fn extract_tool_signals_with_window( - request: &ChatRequest, - recent_window: usize, -) -> ToolResultSignal { - ToolResultSignal::from_request(&to_protocol_request(request), Some(recent_window)) -} diff --git a/crates/switchyard-components/src/lib.rs b/crates/switchyard-components/src/lib.rs deleted file mode 100644 index e9c6983be..000000000 --- a/crates/switchyard-components/src/lib.rs +++ /dev/null @@ -1,36 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Compatibility backends and processors for the Python Switchyard runtime. -//! -//! New Rust orchestration belongs in libsy algorithms and clients. The contracts -//! here remain only while the Python compatibility runtime uses these components. - -pub mod backends; -mod contracts; -pub mod dimension_collector; -pub mod request_processors; -pub mod response_processors; -pub mod stage_router; -pub mod stats; -mod telemetry; - -pub use backends::{ - AnthropicNativeBackend, BackendSelection, BackendSelectionReason, LlmTargetBackend, - MultiLlmBackend, OpenAiNativeBackend, OpenAiPassthroughBackend, StatsLlmBackend, -}; -pub use contracts::*; -pub use dimension_collector::{ - ResponseFlag, ResponseSignals, ToolResultSignal, extract_tool_signals, -}; -pub use request_processors::{ - DimensionCollector, RandomRoutingDecision, RandomRoutingEngine, RandomRoutingProcessorConfig, - RandomRoutingTier, StatsRequestProcessor, -}; -pub use response_processors::{ResponseSignalCollector, StatsResponseProcessor}; -pub use stats::{ - ClassifierStatsSnapshot, CostBreakdown, CostEstimate, LatencyHistogramSnapshot, - ModelStatsSnapshot, PrefixProbe, StatsAccumulator, StatsBackendLatency, StatsRequestStart, - StatsRouteLabel, StatsSnapshot, TierStatsSnapshot, TokenTotals, TokenUsage, prefix_probe, - tracking_enabled_from_env, -}; diff --git a/crates/switchyard-components/src/request_processors/dimension_collector.rs b/crates/switchyard-components/src/request_processors/dimension_collector.rs deleted file mode 100644 index 41b5c4d62..000000000 --- a/crates/switchyard-components/src/request_processors/dimension_collector.rs +++ /dev/null @@ -1,78 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Request-processor adapter for the tool-signal extractor. -//! -//! Thin wrapper around [`crate::dimension_collector::extract_tool_signals_with_window`]. -//! It walks the request's tool-call history and stamps the resulting -//! [`ToolResultSignal`] into `ProxyContext` for the stage_router picker to read. - -use crate::{ChatRequest, ProxyContext, Result}; - -use crate::dimension_collector::{ - DEFAULT_RECENT_WINDOW, ToolResultSignal, extract_tool_signals_with_window, -}; - -/// Populates `ProxyContext` with a [`ToolResultSignal`] read from the request's -/// tool-call history. The stage_router picker reads it via -/// `ctx.get::()`. -#[derive(Clone, Debug)] -pub struct DimensionCollector { - recent_window: usize, -} - -impl Default for DimensionCollector { - fn default() -> Self { - Self { - recent_window: DEFAULT_RECENT_WINDOW, - } - } -} - -impl DimensionCollector { - /// Construct a collector with a caller-supplied sliding-window size for the - /// `recent_*` signal counts. Smaller windows make the picker more reactive - /// to the latest tool call; larger windows smooth over turn-by-turn noise. - pub fn with_recent_window(recent_window: usize) -> Self { - Self { recent_window } - } - - /// Returns the configured `recent_*` sliding-window size. - pub fn recent_window(&self) -> usize { - self.recent_window - } - - /// Extracts the tool-result signal and stores it on the request context. - pub async fn process( - &self, - ctx: &mut ProxyContext, - request: ChatRequest, - ) -> Result { - let tool_signal = extract_tool_signals_with_window(&request, self.recent_window); - ctx.insert::(tool_signal); - Ok(request) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[tokio::test] - async fn stamps_tool_result_signal_into_proxy_context() { - let collector = DimensionCollector::default(); - let request = ChatRequest::openai_chat(json!({ - "model": "test-model", - "messages": [{"role": "user", "content": "hi"}], - })); - - let mut ctx = ProxyContext::new(); - collector - .process(&mut ctx, request) - .await - .expect("process ok"); - - assert!(ctx.get::().is_some()); - } -} diff --git a/crates/switchyard-components/src/request_processors/mod.rs b/crates/switchyard-components/src/request_processors/mod.rs deleted file mode 100644 index 3855c81c3..000000000 --- a/crates/switchyard-components/src/request_processors/mod.rs +++ /dev/null @@ -1,12 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Built-in request processor implementations. - -pub mod dimension_collector; -pub mod random_routing; -pub mod stats; - -pub use dimension_collector::DimensionCollector; -pub use random_routing::*; -pub use stats::*; diff --git a/crates/switchyard-components/src/request_processors/random_routing.rs b/crates/switchyard-components/src/request_processors/random_routing.rs deleted file mode 100644 index dc53d0828..000000000 --- a/crates/switchyard-components/src/request_processors/random_routing.rs +++ /dev/null @@ -1,172 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Runtime support for random routing. - -use std::fmt; - -use crate::{LlmTarget, LlmTargetId, ModelId, Result, SwitchyardError}; -use parking_lot::Mutex; -use rand::rngs::StdRng; -use rand::{RngExt, SeedableRng}; -use serde::{Deserialize, Serialize}; - -const DEFAULT_STRONG_PROBABILITY: f64 = 0.5; - -/// Named side of a strong/weak random-routing decision. -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum RandomRoutingTier { - Strong, - Weak, -} - -impl RandomRoutingTier { - /// Returns the stable lowercase tier label. - pub fn as_str(self) -> &'static str { - match self { - Self::Strong => "strong", - Self::Weak => "weak", - } - } -} - -/// Runtime config for weighted random routing between two LLM targets. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct RandomRoutingProcessorConfig { - pub strong: LlmTarget, - pub weak: LlmTarget, - pub strong_probability: f64, - pub rng_seed: Option, -} - -impl RandomRoutingProcessorConfig { - /// Creates a config with a 50/50 split and entropy-backed randomness. - pub fn new(strong: LlmTarget, weak: LlmTarget) -> Self { - Self { - strong, - weak, - strong_probability: DEFAULT_STRONG_PROBABILITY, - rng_seed: None, - } - } - - /// Creates a config from just strong and weak model names. - pub fn from_models( - strong_model: impl Into, - weak_model: impl Into, - ) -> Result { - Ok(Self::new( - LlmTarget::new( - LlmTargetId::from_static("strong"), - ModelId::new(strong_model)?, - ), - LlmTarget::new(LlmTargetId::from_static("weak"), ModelId::new(weak_model)?), - )) - } - - /// Sets the probability of selecting the strong tier. - pub fn with_strong_probability(mut self, strong_probability: f64) -> Result { - validate_probability(strong_probability)?; - self.strong_probability = strong_probability; - Ok(self) - } - - /// Sets an optional seed for deterministic routing sequences. - pub fn with_rng_seed(mut self, rng_seed: impl Into>) -> Self { - self.rng_seed = rng_seed.into(); - self - } - - /// Validates the random-routing configuration. - pub fn validate(&self) -> Result<()> { - validate_probability(self.strong_probability) - } -} - -/// Captures the chosen strong/weak side and target for downstream components. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct RandomRoutingDecision { - pub tier: RandomRoutingTier, - pub selected_target: LlmTargetId, - pub selected_model: ModelId, - pub original_model: Option, - pub strong_probability: f64, - pub draw: f64, -} - -/// Pure random-routing engine decoupled from request mutation. -pub struct RandomRoutingEngine { - config: RandomRoutingProcessorConfig, - rng: Mutex, -} - -impl fmt::Debug for RandomRoutingEngine { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("RandomRoutingEngine") - .field("config", &self.config) - .finish_non_exhaustive() - } -} - -impl RandomRoutingEngine { - /// Creates a routing engine and initializes its random number generator. - pub fn new(config: RandomRoutingProcessorConfig) -> Result { - config.validate()?; - let rng = match config.rng_seed { - Some(seed) => StdRng::seed_from_u64(seed), - None => rand::make_rng(), - }; - Ok(Self { - config, - rng: Mutex::new(rng), - }) - } - - /// Returns the immutable routing configuration. - pub fn config(&self) -> &RandomRoutingProcessorConfig { - &self.config - } - - /// Selects a target without mutating a request. - pub fn select(&self, original_model: Option) -> Result { - let draw = self.next_draw(); - let tier = if draw < self.config.strong_probability { - RandomRoutingTier::Strong - } else { - RandomRoutingTier::Weak - }; - let selected = self.tier_config(tier); - Ok(RandomRoutingDecision { - tier, - selected_target: selected.id.clone(), - selected_model: selected.model.clone(), - original_model, - strong_probability: self.config.strong_probability, - draw, - }) - } - - // Returns the configured target for the selected tier. - fn tier_config(&self, tier: RandomRoutingTier) -> &LlmTarget { - match tier { - RandomRoutingTier::Strong => &self.config.strong, - RandomRoutingTier::Weak => &self.config.weak, - } - } - - // Draws the next probability sample. - fn next_draw(&self) -> f64 { - (*self.rng.lock()).random() - } -} - -// Validates the weighted random-routing probability. -fn validate_probability(strong_probability: f64) -> Result<()> { - if strong_probability.is_finite() && (0.0..=1.0).contains(&strong_probability) { - return Ok(()); - } - Err(SwitchyardError::InvalidConfig(format!( - "strong_probability must be finite and in [0.0, 1.0], got {strong_probability:?}" - ))) -} diff --git a/crates/switchyard-components/src/request_processors/stats.rs b/crates/switchyard-components/src/request_processors/stats.rs deleted file mode 100644 index 5bdda7142..000000000 --- a/crates/switchyard-components/src/request_processors/stats.rs +++ /dev/null @@ -1,38 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Request-side stats processor. - -use crate::{ChatRequest, ProxyContext, Result}; - -use crate::stats::{StatsRequestStart, prefix_probe}; - -/// Records request start time, and optionally the prefix fingerprints used for -/// switch-aware theoretical cache eligibility (gated, since fingerprinting hashes -/// the full prompt each turn and the per-model seen-sets grow over a run). -#[derive(Clone, Copy, Debug, Default)] -pub struct StatsRequestProcessor { - track_cache_eligibility: bool, -} - -impl StatsRequestProcessor { - /// Creates a processor; `track_cache_eligibility` gates prefix fingerprinting. - pub fn new(track_cache_eligibility: bool) -> Self { - Self { - track_cache_eligibility, - } - } - - /// Records request-start metadata and returns the request unchanged. - pub async fn process( - &self, - ctx: &mut ProxyContext, - request: ChatRequest, - ) -> Result { - ctx.insert(StatsRequestStart::now()); - if self.track_cache_eligibility { - ctx.insert(prefix_probe(request.body())); - } - Ok(request) - } -} diff --git a/crates/switchyard-components/src/response_processors/mod.rs b/crates/switchyard-components/src/response_processors/mod.rs deleted file mode 100644 index 453e0978f..000000000 --- a/crates/switchyard-components/src/response_processors/mod.rs +++ /dev/null @@ -1,10 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Built-in response processor implementations. - -pub mod response_signals; -pub mod stats; - -pub use response_signals::ResponseSignalCollector; -pub use stats::*; diff --git a/crates/switchyard-components/src/response_processors/response_signals.rs b/crates/switchyard-components/src/response_processors/response_signals.rs deleted file mode 100644 index f546f93c2..000000000 --- a/crates/switchyard-components/src/response_processors/response_signals.rs +++ /dev/null @@ -1,98 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Response-side context-signal collector. -//! -//! Thin adapter around -//! [`crate::dimension_collector::response::extract_response_signals`]. Stamps -//! the resulting [`ResponseSignals`] into `ProxyContext` so downstream -//! stage_router / escalation logic can read structured response-quality -//! flags without re-parsing the wire body. - -use crate::{ChatResponse, ProxyContext, Result}; - -use crate::dimension_collector::response::{ResponseSignals, extract_response_signals}; - -/// Populates `ProxyContext` with [`ResponseSignals`] derived from the -/// buffered response body. -/// -/// Stamps nothing for streaming responses — those can't be introspected -/// without consuming the stream. Consumers that read -/// `ctx.get::()` and find `None` after this processor -/// ran should treat that as "stream / not-yet-checked," not as -/// "response was acceptable." -#[derive(Clone, Copy, Debug, Default)] -pub struct ResponseSignalCollector; - -impl ResponseSignalCollector { - /// Extracts response-side signals from buffered responses and leaves streams untouched. - pub async fn process( - &self, - ctx: &mut ProxyContext, - response: ChatResponse, - ) -> Result { - // Only buffered responses carry an inspectable body. Streams pass - // through unchanged with no signals stamped. - if response.body().is_some() { - let signals = extract_response_signals(&response); - ctx.insert::(signals); - } - Ok(response) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::dimension_collector::response::ResponseFlag; - use serde_json::json; - - use crate::Result; - - #[tokio::test] - async fn stamps_response_signals_for_buffered_response() -> Result<()> { - let collector = ResponseSignalCollector; - let response = ChatResponse::openai_completion(json!({ - "choices": [{ - "message": { - "tool_calls": [{ - "function": { - "name": "get_weather", - "arguments": "{\"city\":" // malformed - } - }] - }, - "finish_reason": "tool_calls" - }] - })); - - let mut ctx = ProxyContext::new(); - let _ = collector.process(&mut ctx, response).await?; - - let Some(signals) = ctx.get::() else { - panic!("ResponseSignalCollector did not stamp ResponseSignals onto ctx"); - }; - assert!(signals.contains(ResponseFlag::MalformedToolCallJson)); - Ok(()) - } - - #[tokio::test] - async fn stamps_empty_signals_for_clean_response() -> Result<()> { - let collector = ResponseSignalCollector; - let response = ChatResponse::openai_completion(json!({ - "choices": [{ - "message": { "content": "looks good" }, - "finish_reason": "stop" - }] - })); - - let mut ctx = ProxyContext::new(); - let _ = collector.process(&mut ctx, response).await?; - - let Some(signals) = ctx.get::() else { - panic!("ResponseSignalCollector did not stamp ResponseSignals onto ctx"); - }; - assert!(!signals.has_failures()); - Ok(()) - } -} diff --git a/crates/switchyard-components/src/response_processors/stats.rs b/crates/switchyard-components/src/response_processors/stats.rs deleted file mode 100644 index e5e4a657c..000000000 --- a/crates/switchyard-components/src/response_processors/stats.rs +++ /dev/null @@ -1,249 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Response-side stats processor. - -use crate::{BoxResponseStream, ChatResponse, ProxyContext, Result}; -use async_stream::try_stream; -use futures_util::StreamExt; - -use crate::stats::{ - AnthropicStreamUsage, PrefixProbe, StatsAccumulator, StatsBackendLatency, StatsRequestStart, - TokenUsage, openai_chat_usage_from_stream_event, openai_responses_usage_from_stream_event, - selected_stats_model, selected_stats_tier, usage_from_body, -}; - -/// Records token usage, total latency, and routing overhead. -#[derive(Clone, Debug)] -pub struct StatsResponseProcessor { - accumulator: StatsAccumulator, -} - -impl StatsResponseProcessor { - /// Creates a response processor sharing the supplied accumulator. - pub fn new(accumulator: StatsAccumulator) -> Self { - Self { accumulator } - } - - /// Returns the shared accumulator. - pub fn accumulator(&self) -> &StatsAccumulator { - &self.accumulator - } - - /// Records response usage and wraps streams so usage is captured on completion. - pub async fn process( - &self, - ctx: &mut ProxyContext, - response: ChatResponse, - ) -> Result { - let model = selected_stats_model(ctx, None); - let tier = selected_stats_tier(ctx); - let started_at = ctx.get::().copied(); - let backend_latency = ctx.get::().copied(); - // Switch-aware: eligible only against the prefixes this model has already seen. - let cache_eligible = ctx - .get::() - .map(|probe| self.accumulator.prefix_eligibility(&model, probe)) - .unwrap_or(0.0); - - match response { - ChatResponse::OpenAiCompletion(response) => { - record_usage( - &self.accumulator, - &model, - usage_from_body(response.body()), - started_at, - backend_latency, - tier.as_deref(), - cache_eligible, - )?; - Ok(ChatResponse::OpenAiCompletion(response)) - } - ChatResponse::OpenAiResponsesCompletion(response) => { - record_usage( - &self.accumulator, - &model, - usage_from_body(response.body()), - started_at, - backend_latency, - tier.as_deref(), - cache_eligible, - )?; - Ok(ChatResponse::OpenAiResponsesCompletion(response)) - } - ChatResponse::AnthropicCompletion(response) => { - record_usage( - &self.accumulator, - &model, - usage_from_body(response.body()), - started_at, - backend_latency, - tier.as_deref(), - cache_eligible, - )?; - Ok(ChatResponse::AnthropicCompletion(response)) - } - ChatResponse::OpenAiStream(stream) => { - Ok(ChatResponse::OpenAiStream(wrap_openai_chat_stream( - stream, - self.accumulator.clone(), - model, - started_at, - backend_latency, - tier, - cache_eligible, - ))) - } - ChatResponse::OpenAiResponsesStream(stream) => Ok(ChatResponse::OpenAiResponsesStream( - wrap_openai_responses_stream( - stream, - self.accumulator.clone(), - model, - started_at, - backend_latency, - tier, - cache_eligible, - ), - )), - ChatResponse::AnthropicStream(stream) => { - Ok(ChatResponse::AnthropicStream(wrap_anthropic_stream( - stream, - self.accumulator.clone(), - model, - started_at, - backend_latency, - tier, - cache_eligible, - ))) - } - } - } -} - -fn wrap_openai_chat_stream( - mut stream: BoxResponseStream, - accumulator: StatsAccumulator, - model: String, - started_at: Option, - backend_latency: Option, - tier: Option, - cache_eligible: f64, -) -> BoxResponseStream { - Box::pin(try_stream! { - let mut committed = false; - while let Some(event) = stream.next().await { - let event = event?; - if !committed - && let Some(usage) = openai_chat_usage_from_stream_event(&event) { - log_stream_record_result(record_usage( - &accumulator, - &model, - usage, - started_at, - backend_latency, - tier.as_deref(), - cache_eligible, - ), &model); - committed = true; - } - yield event; - } - }) -} - -fn wrap_openai_responses_stream( - mut stream: BoxResponseStream, - accumulator: StatsAccumulator, - model: String, - started_at: Option, - backend_latency: Option, - tier: Option, - cache_eligible: f64, -) -> BoxResponseStream { - Box::pin(try_stream! { - let mut committed = false; - while let Some(event) = stream.next().await { - let event = event?; - if !committed - && let Some(usage) = openai_responses_usage_from_stream_event(&event) { - log_stream_record_result(record_usage( - &accumulator, - &model, - usage, - started_at, - backend_latency, - tier.as_deref(), - cache_eligible, - ), &model); - committed = true; - } - yield event; - } - }) -} - -fn wrap_anthropic_stream( - mut stream: BoxResponseStream, - accumulator: StatsAccumulator, - model: String, - started_at: Option, - backend_latency: Option, - tier: Option, - cache_eligible: f64, -) -> BoxResponseStream { - Box::pin(try_stream! { - let mut stream_usage = AnthropicStreamUsage::default(); - while let Some(event) = stream.next().await { - let event = event?; - if let Some(usage) = stream_usage.observe(&event) { - log_stream_record_result(record_usage( - &accumulator, - &model, - usage, - started_at, - backend_latency, - tier.as_deref(), - cache_eligible, - ), &model); - } - yield event; - } - }) -} - -fn log_stream_record_result(result: Result<()>, model: &str) { - if let Err(error) = result { - tracing::warn!( - error = %error, - model = %model, - "failed to record stream usage" - ); - } -} - -fn record_usage( - accumulator: &StatsAccumulator, - model: &str, - mut usage: TokenUsage, - started_at: Option, - backend_latency: Option, - tier: Option<&str>, - cache_eligible: f64, -) -> Result<()> { - usage.cacheable_prompt_tokens = (usage.prompt_tokens as f64 * cache_eligible).round() as u64; - let total_latency_ms = started_at.map(StatsRequestStart::elapsed_ms); - let backend_latency_ms = backend_latency.map(StatsBackendLatency::as_millis_f64); - let routing_overhead_ms = - total_latency_ms - .zip(backend_latency_ms) - .map(|(total_latency_ms, backend_latency_ms)| { - (total_latency_ms - backend_latency_ms).max(0.0) - }); - accumulator.record_usage_after_success_attribution( - model.to_string(), - usage, - total_latency_ms, - routing_overhead_ms, - tier, - ) -} diff --git a/crates/switchyard-components/src/stage_router.rs b/crates/switchyard-components/src/stage_router.rs deleted file mode 100644 index 04a845030..000000000 --- a/crates/switchyard-components/src/stage_router.rs +++ /dev/null @@ -1,15 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Stage-router tier selection — thin re-export of libsy's decision core. -//! -//! The scoring and tier-selection logic lives in [`switchyard_libsy::stage_router`]. -//! Its input, a [`ToolResultSignal`][crate::dimension_collector::ToolResultSignal], -//! is already libsy's `ToolSignals`, so no request adaptation is needed here — this -//! module simply re-exports the API so the crate's processors decide a turn's tier -//! through the same implementation as the libsy algorithm. - -pub use switchyard_libsy::{ - CodingAgentDimensions, DecisionSource, PickOutcome, PickerMode, ScoreResult, StageClassifier, - Tier, dimensions_from_signal, pick_tier, score_signal, -}; diff --git a/crates/switchyard-components/src/stats/accumulator.rs b/crates/switchyard-components/src/stats/accumulator.rs deleted file mode 100644 index 455c8cafe..000000000 --- a/crates/switchyard-components/src/stats/accumulator.rs +++ /dev/null @@ -1,792 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Thread-safe stats accumulator and serializable snapshot schema. - -use std::cmp::{Ordering, Reverse}; -use std::collections::{BTreeMap, BinaryHeap, HashSet, btree_map::Entry}; -use std::sync::Arc; - -use crate::Result; -use parking_lot::{Mutex, MutexGuard}; -use serde::{Deserialize, Serialize}; - -use super::cost::{CostEstimate, estimate_cost}; -use super::{PrefixProbe, TokenUsage}; - -const MAX_LATENCY_SAMPLES: usize = 10_000; - -/// Thread-safe stats store shared by stats processors and backend wrappers. -#[derive(Clone, Debug)] -pub struct StatsAccumulator { - inner: Arc>, -} - -impl Default for StatsAccumulator { - fn default() -> Self { - Self::new() - } -} - -impl StatsAccumulator { - /// Creates an empty stats accumulator. - pub fn new() -> Self { - Self { - inner: Arc::new(Mutex::new(StatsAccumulatorInner::default())), - } - } - - /// Records a successful backend call. - pub fn record_success( - &self, - model: impl Into, - backend_latency_ms: Option, - tier: Option<&str>, - ) -> Result<()> { - let mut inner = self.lock(); - inner.total_requests = inner.total_requests.saturating_add(1); - let model = model.into(); - let tier = tier.map(str::trim).filter(|tier| !tier.is_empty()); - { - let stats = inner.model_stats_mut(model.clone()); - stats.calls = stats.calls.saturating_add(1); - if let Some(tier) = tier { - stats.tier = Some(tier.to_string()); - } - if let Some(latency) = backend_latency_ms { - stats.model_call_latency.record(latency); - } - } - if let Some(tier) = tier { - if let Some(tier_stats) = inner.tier_stats_mut(tier, &model) { - tier_stats.calls = tier_stats.calls.saturating_add(1); - } - } else { - inner.record_untiered_success(&model); - } - Ok(()) - } - - /// Records a backend error. - pub fn record_error(&self, model: impl Into, tier: Option<&str>) -> Result<()> { - let mut inner = self.lock(); - inner.total_requests = inner.total_requests.saturating_add(1); - inner.total_errors = inner.total_errors.saturating_add(1); - let stats = inner.model_stats_mut(model.into()); - stats.errors = stats.errors.saturating_add(1); - if let Some(tier) = tier { - stats.tier = Some(tier.to_string()); - } - Ok(()) - } - - /// Returns the cache-eligible fraction for `model` and records the prefix as seen. - /// - /// Switch-aware: a prefix counts only if this model was previously sent it, so a - /// switch to a cold model yields 0 and a return credits only the shared prefix. - pub fn prefix_eligibility(&self, model: &str, probe: &PrefixProbe) -> f64 { - let mut inner = self.lock(); - let stats = inner.model_stats_mut(model.to_string()); - let fraction = probe.eligible_fraction(&stats.seen_prefixes); - if let Some(hash) = probe.full_hash() { - stats.seen_prefixes.insert(hash); - } - fraction - } - - /// Records token usage and end-to-end latency for a completed response. - pub fn record_usage( - &self, - model: impl Into, - usage: TokenUsage, - total_latency_ms: Option, - routing_overhead_ms: Option, - tier: Option<&str>, - ) -> Result<()> { - self.record_usage_inner( - model, - usage, - total_latency_ms, - routing_overhead_ms, - tier, - TierCallAttribution::LegacyPendingUntiered, - ) - } - - /// Records usage and attaches one previously untiered success to `tier`. - /// - /// This is only for compatibility paths that recorded success before the - /// route label was available. Normal stats backends record success with the - /// tier already present, so their usage events must leave this flag false. - pub fn record_usage_with_success_was_untiered( - &self, - model: impl Into, - usage: TokenUsage, - total_latency_ms: Option, - routing_overhead_ms: Option, - tier: Option<&str>, - ) -> Result<()> { - self.record_usage_inner( - model, - usage, - total_latency_ms, - routing_overhead_ms, - tier, - TierCallAttribution::ExplicitUntiered, - ) - } - - /// Records usage after the corresponding success call was already attributed. - /// - /// `StatsLlmBackend` and routing runtimes record success before the response - /// processor records tokens. Those internal paths must not consume a legacy - /// pending untiered success that belongs to some other direct accumulator caller. - pub fn record_usage_after_success_attribution( - &self, - model: impl Into, - usage: TokenUsage, - total_latency_ms: Option, - routing_overhead_ms: Option, - tier: Option<&str>, - ) -> Result<()> { - self.record_usage_inner( - model, - usage, - total_latency_ms, - routing_overhead_ms, - tier, - TierCallAttribution::AlreadyRecorded, - ) - } - - fn record_usage_inner( - &self, - model: impl Into, - usage: TokenUsage, - total_latency_ms: Option, - routing_overhead_ms: Option, - tier: Option<&str>, - tier_call_attribution: TierCallAttribution, - ) -> Result<()> { - let mut inner = self.lock(); - let model = model.into(); - { - let stats = inner.model_stats_mut(model.clone()); - stats.prompt_tokens = stats.prompt_tokens.saturating_add(usage.prompt_tokens); - stats.max_observed_context_tokens = stats - .max_observed_context_tokens - .max(usage.prompt_tokens.saturating_add(usage.completion_tokens)); - stats.completion_tokens = stats - .completion_tokens - .saturating_add(usage.completion_tokens); - stats.cached_tokens = stats.cached_tokens.saturating_add(usage.cached_tokens); - stats.cache_creation_tokens = stats - .cache_creation_tokens - .saturating_add(usage.cache_creation_tokens); - stats.cacheable_prompt_tokens = stats - .cacheable_prompt_tokens - .saturating_add(usage.cacheable_prompt_tokens); - stats.reasoning_tokens = stats - .reasoning_tokens - .saturating_add(usage.reasoning_tokens); - if let Some(tier) = tier { - stats.tier = Some(tier.to_string()); - } - if let Some(latency) = total_latency_ms { - stats.total_latency.record(latency); - } - } - if let Some(tier) = tier { - let tier = tier.trim(); - if !tier.is_empty() { - let should_attribute_call = match tier_call_attribution { - TierCallAttribution::LegacyPendingUntiered => { - inner.consume_untiered_success(&model) - } - TierCallAttribution::ExplicitUntiered => { - inner.consume_untiered_success(&model); - true - } - TierCallAttribution::AlreadyRecorded => false, - }; - if let Some(tier_stats) = inner.tier_stats_mut(tier, &model) { - if should_attribute_call { - tier_stats.calls = tier_stats.calls.saturating_add(1); - } - tier_stats.prompt_tokens = - tier_stats.prompt_tokens.saturating_add(usage.prompt_tokens); - tier_stats.completion_tokens = tier_stats - .completion_tokens - .saturating_add(usage.completion_tokens); - } - } - } - if let Some(overhead) = routing_overhead_ms { - inner.routing_overhead.record(overhead); - } - Ok(()) - } - - /// Records one LLM-classifier overhead call. - /// - /// The classifier's per-request call is not part of the routed-backend - /// chain and must stay out of `by_model` — otherwise the default TB-lite - /// config (classifier model == efficient-tier model) double-counts the spend. - /// `record_classifier_usage` writes to a dedicated bucket; the snapshot - /// exposes it under `classifier.models` with its own `cost_estimate`. - pub fn record_classifier_usage( - &self, - model: impl Into, - usage: TokenUsage, - latency_ms: Option, - ) -> Result<()> { - let mut inner = self.lock(); - inner.classifier_requests = inner.classifier_requests.saturating_add(1); - let stats = inner.classifier_stats_mut(model.into()); - stats.calls = stats.calls.saturating_add(1); - stats.prompt_tokens = stats.prompt_tokens.saturating_add(usage.prompt_tokens); - stats.max_observed_context_tokens = stats - .max_observed_context_tokens - .max(usage.prompt_tokens.saturating_add(usage.completion_tokens)); - stats.completion_tokens = stats - .completion_tokens - .saturating_add(usage.completion_tokens); - stats.cached_tokens = stats.cached_tokens.saturating_add(usage.cached_tokens); - stats.cache_creation_tokens = stats - .cache_creation_tokens - .saturating_add(usage.cache_creation_tokens); - stats.reasoning_tokens = stats - .reasoning_tokens - .saturating_add(usage.reasoning_tokens); - if let Some(latency) = latency_ms { - stats.model_call_latency.record(latency); - stats.total_latency.record(latency); - } - Ok(()) - } - - /// Records a classifier-call failure. - /// - /// Bumps the classifier `total_requests` (so the failure shows up in - /// the per-request distribution) and `total_errors`, plus the - /// per-model `errors` counter. Does **not** bump `calls` — that - /// field counts completed (token-bearing) calls only, so the - /// `errors / (calls + errors)` ratio is the failure rate. - pub fn record_classifier_error(&self, model: impl Into) -> Result<()> { - let mut inner = self.lock(); - inner.classifier_requests = inner.classifier_requests.saturating_add(1); - inner.classifier_errors = inner.classifier_errors.saturating_add(1); - let stats = inner.classifier_stats_mut(model.into()); - stats.errors = stats.errors.saturating_add(1); - Ok(()) - } - - /// Records one routing decision source for a routing algorithm. - /// - /// This is intentionally separate from model/tier accounting: a stage-router can - /// choose `efficient` because of an override, a dimensions score, an LLM-classifier - /// verdict, or a fail-open default, and those explanations are useful even - /// when they all land on the same backend model. - pub fn record_routing_decision( - &self, - profile_type: impl Into, - source: impl Into, - ) -> Result<()> { - let mut inner = self.lock(); - let sources = inner - .routing_decisions - .entry(profile_type.into()) - .or_default(); - let count = sources.entry(source.into()).or_insert(0); - *count = count.saturating_add(1); - Ok(()) - } - - /// Returns a computed snapshot suitable for JSON serialization. - pub fn snapshot(&self) -> Result { - let inner = self.lock().clone(); - Ok(inner.snapshot()) - } - - /// Clears all counters. - pub fn reset(&self) -> Result<()> { - let mut inner = self.lock(); - *inner = StatsAccumulatorInner::default(); - Ok(()) - } - - fn lock(&self) -> MutexGuard<'_, StatsAccumulatorInner> { - self.inner.lock() - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum TierCallAttribution { - LegacyPendingUntiered, - ExplicitUntiered, - AlreadyRecorded, -} - -#[derive(Clone, Debug, Default)] -struct StatsAccumulatorInner { - by_model: BTreeMap, - model_order: Vec, - untiered_successes_by_model: BTreeMap, - by_tier: BTreeMap, - total_requests: u64, - total_errors: u64, - routing_overhead: LatencyHistogram, - by_classifier: BTreeMap, - classifier_model_order: Vec, - classifier_requests: u64, - classifier_errors: u64, - routing_decisions: BTreeMap>, -} - -impl StatsAccumulatorInner { - fn model_stats_mut(&mut self, model: String) -> &mut ModelStats { - match self.by_model.entry(model) { - Entry::Occupied(entry) => entry.into_mut(), - Entry::Vacant(entry) => { - self.model_order.push(entry.key().clone()); - entry.insert(ModelStats::default()) - } - } - } - - fn classifier_stats_mut(&mut self, model: String) -> &mut ModelStats { - match self.by_classifier.entry(model) { - Entry::Occupied(entry) => entry.into_mut(), - Entry::Vacant(entry) => { - self.classifier_model_order.push(entry.key().clone()); - entry.insert(ModelStats::default()) - } - } - } - - fn tier_stats_mut(&mut self, tier: &str, model: &str) -> Option<&mut TierStats> { - let tier = tier.trim(); - if tier.is_empty() { - return None; - } - Some(match self.by_tier.entry(tier.to_string()) { - Entry::Occupied(entry) => entry.into_mut(), - Entry::Vacant(entry) => entry.insert(TierStats { - model: model.to_string(), - ..TierStats::default() - }), - }) - } - - fn record_untiered_success(&mut self, model: &str) { - let count = self - .untiered_successes_by_model - .entry(model.to_string()) - .or_insert(0); - *count = count.saturating_add(1); - } - - fn consume_untiered_success(&mut self, model: &str) -> bool { - match self.untiered_successes_by_model.entry(model.to_string()) { - Entry::Occupied(mut entry) => { - let count = entry.get_mut(); - if *count > 1 { - *count -= 1; - } else { - entry.remove(); - } - true - } - Entry::Vacant(_) => false, - } - } - - fn snapshot(&self) -> StatsSnapshot { - let (models, totals) = build_model_snapshots(&self.by_model, self.total_requests); - let total_tokens = totals.total; - let mut cost_estimate = estimate_cost(&models); - - let classifier = build_classifier_snapshot( - &self.by_classifier, - self.classifier_requests, - self.classifier_errors, - ); - cost_estimate.classifier_cost = classifier.cost_estimate.total_cost; - - cost_estimate.total_cost = - round6(cost_estimate.backend_cost + cost_estimate.classifier_cost); - - StatsSnapshot { - total_requests: self.total_requests, - total_errors: self.total_errors, - total_tokens: totals, - tiers: tier_snapshots(&self.by_tier, total_tokens, self.total_requests), - cost_estimate, - models, - routing_overhead: self.routing_overhead.snapshot(), - classifier, - routing_decisions: self.routing_decisions.clone(), - } - } -} - -fn build_model_snapshots( - by_model: &BTreeMap, - total_requests: u64, -) -> (BTreeMap, TokenTotals) { - let mut totals = TokenTotals::default(); - for stats in by_model.values() { - totals.prompt = totals.prompt.saturating_add(stats.prompt_tokens); - totals.completion = totals.completion.saturating_add(stats.completion_tokens); - totals.cached = totals.cached.saturating_add(stats.cached_tokens); - totals.cache_creation = totals - .cache_creation - .saturating_add(stats.cache_creation_tokens); - totals.reasoning = totals.reasoning.saturating_add(stats.reasoning_tokens); - } - totals.total = totals.prompt.saturating_add(totals.completion); - - let mut models = BTreeMap::new(); - for (model, stats) in by_model { - let token_total = stats.prompt_tokens.saturating_add(stats.completion_tokens); - let request_pct = if total_requests == 0 { - 0.0 - } else { - round2(stats.calls as f64 / total_requests as f64 * 100.0) - }; - let token_pct = if totals.total == 0 { - 0.0 - } else { - round2(token_total as f64 / totals.total as f64 * 100.0) - }; - let avg_prompt_tokens = if stats.calls == 0 { - 0.0 - } else { - round2(stats.prompt_tokens as f64 / stats.calls as f64) - }; - let avg_completion_tokens = if stats.calls == 0 { - 0.0 - } else { - round2(stats.completion_tokens as f64 / stats.calls as f64) - }; - let cache_hit_rate = if stats.prompt_tokens == 0 { - 0.0 - } else { - round4(stats.cached_tokens as f64 / stats.prompt_tokens as f64) - }; - // Switch-aware ceiling: prefix this model had already seen; gap to actual is the backend. - let theoretical_cache_hit_rate = if stats.prompt_tokens == 0 { - 0.0 - } else { - round4(stats.cacheable_prompt_tokens as f64 / stats.prompt_tokens as f64) - }; - - models.insert( - model.clone(), - ModelStatsSnapshot { - calls: stats.calls, - errors: stats.errors, - request_pct, - prompt_tokens: stats.prompt_tokens, - max_observed_context_tokens: stats.max_observed_context_tokens, - completion_tokens: stats.completion_tokens, - total_tokens: token_total, - token_pct, - cached_tokens: stats.cached_tokens, - cache_creation_tokens: stats.cache_creation_tokens, - reasoning_tokens: stats.reasoning_tokens, - avg_prompt_tokens, - avg_completion_tokens, - cache_hit_rate, - theoretical_cache_hit_rate, - model_call_latency: stats.model_call_latency.snapshot(), - total_latency: stats.total_latency.snapshot(), - tier: stats.tier.clone(), - }, - ); - } - (models, totals) -} - -fn build_classifier_snapshot( - by_classifier: &BTreeMap, - total_requests: u64, - total_errors: u64, -) -> ClassifierStatsSnapshot { - let (models, totals) = build_model_snapshots(by_classifier, total_requests); - let cost_estimate = estimate_cost(&models); - ClassifierStatsSnapshot { - total_requests, - total_errors, - total_tokens: totals, - models, - cost_estimate, - } -} - -#[derive(Clone, Debug, Default)] -struct ModelStats { - calls: u64, - errors: u64, - prompt_tokens: u64, - max_observed_context_tokens: u64, - completion_tokens: u64, - cached_tokens: u64, - cache_creation_tokens: u64, - cacheable_prompt_tokens: u64, - reasoning_tokens: u64, - /// Prefix fingerprints this model has been sent; basis for switch-aware theoretical. - seen_prefixes: HashSet, - model_call_latency: LatencyHistogram, - total_latency: LatencyHistogram, - tier: Option, -} - -#[derive(Clone, Debug, Default)] -struct TierStats { - model: String, - calls: u64, - prompt_tokens: u64, - completion_tokens: u64, -} - -#[derive(Clone, Debug)] -struct LatencyHistogram { - count: u64, - total_ms: f64, - min_ms: f64, - max_ms: f64, - samples: BinaryHeap>, -} - -impl Default for LatencyHistogram { - fn default() -> Self { - Self { - count: 0, - total_ms: 0.0, - min_ms: f64::INFINITY, - max_ms: 0.0, - samples: BinaryHeap::new(), - } - } -} - -impl LatencyHistogram { - fn record(&mut self, latency_ms: f64) { - if !latency_ms.is_finite() { - tracing::debug!(latency_ms, "dropping non-finite latency sample"); - return; - } - let latency_ms = latency_ms.max(0.0); - self.count = self.count.saturating_add(1); - self.total_ms += latency_ms; - self.min_ms = self.min_ms.min(latency_ms); - self.max_ms = self.max_ms.max(latency_ms); - let sample = Reverse(LatencySample(latency_ms)); - if self.samples.len() < MAX_LATENCY_SAMPLES { - self.samples.push(sample); - } else if let Some(smallest_sample) = self.samples.peek() - && latency_ms > smallest_sample.0.value() - && let Some(mut smallest_sample) = self.samples.peek_mut() - { - *smallest_sample = sample; - } - } - - fn snapshot(&self) -> LatencyHistogramSnapshot { - if self.count == 0 { - return LatencyHistogramSnapshot::default(); - } - let mut samples = self - .samples - .iter() - .map(|sample| sample.0.value()) - .collect::>(); - samples.sort_by(f64::total_cmp); - let sample_count = samples.len(); - let p50_ms = samples - .get(sample_count / 2) - .copied() - .map(round2) - .unwrap_or(0.0); - let p99_index = sample_count - .saturating_sub(1) - .min((sample_count as f64 * 0.99) as usize); - let p99_ms = samples.get(p99_index).copied().map(round2).unwrap_or(0.0); - - LatencyHistogramSnapshot { - count: self.count, - total_ms: round2(self.total_ms), - min_ms: round2(self.min_ms), - max_ms: round2(self.max_ms), - avg_ms: round2(self.total_ms / self.count as f64), - p50_ms, - p99_ms, - } - } -} - -#[derive(Clone, Copy, Debug)] -struct LatencySample(f64); - -impl LatencySample { - fn value(self) -> f64 { - self.0 - } -} - -impl PartialEq for LatencySample { - fn eq(&self, other: &Self) -> bool { - self.0.total_cmp(&other.0) == Ordering::Equal - } -} - -impl Eq for LatencySample {} - -impl PartialOrd for LatencySample { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for LatencySample { - fn cmp(&self, other: &Self) -> Ordering { - self.0.total_cmp(&other.0) - } -} - -/// Full stats snapshot. -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] -pub struct StatsSnapshot { - pub total_requests: u64, - pub total_errors: u64, - pub total_tokens: TokenTotals, - pub models: BTreeMap, - pub tiers: BTreeMap, - pub routing_overhead: LatencyHistogramSnapshot, - pub cost_estimate: CostEstimate, - pub classifier: ClassifierStatsSnapshot, - pub routing_decisions: BTreeMap>, -} - -/// LLM-classifier overhead stats, recorded out-of-band from routed-backend -/// traffic so the two never alias on a shared model id. -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] -pub struct ClassifierStatsSnapshot { - pub total_requests: u64, - pub total_errors: u64, - pub total_tokens: TokenTotals, - pub models: BTreeMap, - pub cost_estimate: CostEstimate, -} - -/// Aggregate token totals. -#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] -pub struct TokenTotals { - pub prompt: u64, - pub completion: u64, - pub cached: u64, - pub cache_creation: u64, - pub reasoning: u64, - pub total: u64, -} - -/// Per-model stats in a snapshot. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct ModelStatsSnapshot { - pub calls: u64, - pub errors: u64, - pub request_pct: f64, - pub prompt_tokens: u64, - /// Largest prompt-plus-completion token count observed in one completed response. - pub max_observed_context_tokens: u64, - pub completion_tokens: u64, - pub total_tokens: u64, - pub token_pct: f64, - pub cached_tokens: u64, - pub cache_creation_tokens: u64, - pub reasoning_tokens: u64, - pub avg_prompt_tokens: f64, - pub avg_completion_tokens: f64, - pub cache_hit_rate: f64, - /// Switch-aware ceiling: fraction of the prompt this model had already been sent. - pub theoretical_cache_hit_rate: f64, - pub model_call_latency: LatencyHistogramSnapshot, - pub total_latency: LatencyHistogramSnapshot, - pub tier: Option, -} - -/// Per-tier stats in a snapshot. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct TierStatsSnapshot { - pub model: String, - pub calls: u64, - pub request_pct: f64, - pub prompt_tokens: u64, - pub completion_tokens: u64, - pub total_tokens: u64, - pub token_pct: f64, -} - -/// Latency histogram summary in milliseconds. -/// -/// Non-finite latency samples are ignored and logged at debug level. -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] -pub struct LatencyHistogramSnapshot { - pub count: u64, - pub total_ms: f64, - pub min_ms: f64, - pub max_ms: f64, - pub avg_ms: f64, - pub p50_ms: f64, - pub p99_ms: f64, -} - -fn tier_snapshots( - by_tier: &BTreeMap, - total_tokens: u64, - total_requests: u64, -) -> BTreeMap { - let mut tiers = BTreeMap::new(); - for (tier, stats) in by_tier { - tiers.insert( - tier.clone(), - TierStatsSnapshot { - model: stats.model.clone(), - calls: stats.calls, - request_pct: 0.0, - prompt_tokens: stats.prompt_tokens, - completion_tokens: stats.completion_tokens, - total_tokens: 0, - token_pct: 0.0, - }, - ); - } - - for tier in tiers.values_mut() { - tier.total_tokens = tier.prompt_tokens.saturating_add(tier.completion_tokens); - tier.request_pct = if total_requests == 0 { - 0.0 - } else { - round2(tier.calls as f64 / total_requests as f64 * 100.0) - }; - tier.token_pct = if total_tokens == 0 { - 0.0 - } else { - round2(tier.total_tokens as f64 / total_tokens as f64 * 100.0) - }; - } - tiers -} - -fn round2(value: f64) -> f64 { - (value * 100.0).round() / 100.0 -} - -fn round4(value: f64) -> f64 { - (value * 10_000.0).round() / 10_000.0 -} - -fn round6(value: f64) -> f64 { - (value * 1_000_000.0).round() / 1_000_000.0 -} diff --git a/crates/switchyard-components/src/stats/cache_eligibility.rs b/crates/switchyard-components/src/stats/cache_eligibility.rs deleted file mode 100644 index 9724e04ca..000000000 --- a/crates/switchyard-components/src/stats/cache_eligibility.rs +++ /dev/null @@ -1,164 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Switch-aware cache eligibility: how much of a prompt a model has already been sent. - -use std::collections::HashSet; -use std::collections::hash_map::DefaultHasher; -use std::hash::{Hash, Hasher}; - -use serde_json::Value; - -/// Env var that opts in to per-model theoretical cache-hit tracking. -const TRACK_ENV: &str = "SWITCHYARD_THEORETICAL_CACHE"; - -/// Whether theoretical cache-hit tracking is enabled via the environment. -/// -/// Off by default: prefix fingerprinting and the per-model seen-sets are skipped -/// unless opted in, so the hot path adds nothing and memory stays flat. -pub fn tracking_enabled_from_env() -> bool { - env_opts_in(std::env::var(TRACK_ENV).ok().as_deref()) -} - -fn env_opts_in(value: Option<&str>) -> bool { - matches!( - value.map(|v| v.trim().to_ascii_lowercase()).as_deref(), - Some("1" | "true" | "yes" | "on") - ) -} - -/// Cumulative prefix fingerprints of a request, in message order. -#[derive(Clone, Debug, Default)] -pub struct PrefixProbe { - /// `(cumulative_text_len, rolling_hash)` after the prefix fields and each turn. - boundaries: Vec<(u64, u64)>, - /// Total prompt text length, including the newest turn. - total_len: u64, -} - -impl PrefixProbe { - /// Eligible fraction given the prefix fingerprints a model has already seen. - /// Longest matching boundary over total; the newest turn is unseen, so excluded. - pub fn eligible_fraction(&self, seen: &HashSet) -> f64 { - if self.total_len == 0 { - return 0.0; - } - let eligible = self - .boundaries - .iter() - .filter(|(_, hash)| seen.contains(hash)) - .map(|(len, _)| *len) - .max() - .unwrap_or(0); - eligible as f64 / self.total_len as f64 - } - - /// Fingerprint of the full prompt, recorded once a model has processed it. - pub fn full_hash(&self) -> Option { - self.boundaries.last().map(|(_, hash)| *hash) - } -} - -/// Builds prefix fingerprints from a request body. -/// Format-agnostic: `system`/`instructions`, then each `messages`/`input` turn. -pub fn prefix_probe(body: &Value) -> PrefixProbe { - let mut boundaries = Vec::new(); - let mut acc_len = 0u64; - let mut hasher = DefaultHasher::new(); - - let system = body.get("system"); - let instructions = body.get("instructions"); - let prefix_len = system.map(text_len).unwrap_or(0) + instructions.map(text_len).unwrap_or(0); - if prefix_len > 0 { - acc_len += prefix_len; - for value in [system, instructions].into_iter().flatten() { - hash_text_into(value, &mut hasher); - } - boundaries.push((acc_len, hasher.finish())); - } - - let turns = body - .get("messages") - .or_else(|| body.get("input")) - .and_then(Value::as_array); - if let Some(turns) = turns { - for turn in turns { - acc_len += text_len(turn); - hash_text_into(turn, &mut hasher); - boundaries.push((acc_len, hasher.finish())); - } - } - PrefixProbe { - boundaries, - total_len: acc_len, - } -} - -/// Recursively sums the byte length of every JSON string value. -fn text_len(value: &Value) -> u64 { - match value { - Value::String(s) => s.len() as u64, - Value::Array(items) => items.iter().map(text_len).sum(), - Value::Object(map) => map.values().map(text_len).sum(), - _ => 0, - } -} - -/// Recursively feeds every JSON scalar value into the hasher, in order. -/// Includes numbers and bools so prompts differing only in a scalar don't collide. -fn hash_text_into(value: &Value, hasher: &mut DefaultHasher) { - match value { - Value::String(s) => s.hash(hasher), - Value::Number(n) => n.to_string().hash(hasher), - Value::Bool(b) => b.hash(hasher), - Value::Array(items) => items.iter().for_each(|item| hash_text_into(item, hasher)), - Value::Object(map) => map.values().for_each(|val| hash_text_into(val, hasher)), - Value::Null => {} - } -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn env_opt_in_parsing() { - assert!(env_opts_in(Some("1"))); - assert!(env_opts_in(Some(" TRUE "))); - assert!(env_opts_in(Some("on"))); - assert!(!env_opts_in(Some("0"))); - assert!(!env_opts_in(Some("false"))); - assert!(!env_opts_in(None)); - } - - #[test] - fn empty_body_is_zero() { - assert_eq!( - prefix_probe(&json!({})).eligible_fraction(&HashSet::new()), - 0.0 - ); - } - - #[test] - fn unseen_prefix_is_not_eligible() { - let probe = prefix_probe(&json!({"messages": [{"role": "user", "content": "aaaa"}]})); - assert_eq!(probe.eligible_fraction(&HashSet::new()), 0.0); - } - - #[test] - fn previously_seen_prefix_is_eligible_newest_turn_is_not() { - let turn1 = prefix_probe(&json!({"messages": [{"role": "user", "content": "aaaa"}]})); - let mut seen = HashSet::new(); - seen.insert(turn1.full_hash().unwrap()); - - // Same first turn plus an equal-length newest turn -> half is re-presentable. - let turn2 = prefix_probe(&json!({ - "messages": [ - {"role": "user", "content": "aaaa"}, - {"role": "user", "content": "bbbb"}, - ], - })); - assert_eq!(turn2.eligible_fraction(&seen), 0.5); - } -} diff --git a/crates/switchyard-components/src/stats/context.rs b/crates/switchyard-components/src/stats/context.rs deleted file mode 100644 index 6d397eb97..000000000 --- a/crates/switchyard-components/src/stats/context.rs +++ /dev/null @@ -1,72 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Typed context markers shared by stats components. - -use std::time::{Duration, Instant}; - -use crate::ProxyContext; -use serde::{Deserialize, Serialize}; - -use crate::backends::BackendSelection; -use crate::request_processors::RandomRoutingDecision; - -/// Request start time captured by `StatsRequestProcessor`. -#[derive(Clone, Copy, Debug)] -pub struct StatsRequestStart(Instant); - -impl StatsRequestStart { - /// Captures the current monotonic clock instant. - pub fn now() -> Self { - Self(Instant::now()) - } - - /// Returns elapsed milliseconds since the captured start. - pub fn elapsed_ms(self) -> f64 { - self.0.elapsed().as_secs_f64() * 1000.0 - } -} - -/// Backend-call duration captured by `StatsLlmBackend`. -#[derive(Clone, Copy, Debug)] -pub struct StatsBackendLatency(pub Duration); - -impl StatsBackendLatency { - /// Converts the duration to milliseconds. - pub fn as_millis_f64(self) -> f64 { - self.0.as_secs_f64() * 1000.0 - } -} - -/// Optional generic tier label for non-random routing stats. -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] -pub struct StatsRouteLabel(pub String); - -impl StatsRouteLabel { - /// Creates a route label, preserving caller-provided naming. - pub fn new(label: impl Into) -> Self { - Self(label.into()) - } -} - -/// Returns the selected model label for stats attribution. -pub fn selected_stats_model(ctx: &ProxyContext, fallback: Option<&str>) -> String { - ctx.get::() - .map(|selection| selection.model.as_str().to_string()) - .or_else(|| fallback.map(str::to_string)) - .unwrap_or_else(|| "".to_string()) -} - -/// Returns a tier label when a routing component supplied one. -pub fn selected_stats_tier(ctx: &ProxyContext) -> Option { - if let Some(label) = ctx.get::() { - return Some(label.0.clone()); - } - if let Some(decision) = ctx.get::() { - return Some(decision.tier.as_str().to_string()); - } - // Fallback: Python-based pickers (e.g. stage_router) stamp selected_target but - // don't write a typed marker — use it as the tier label so /v1/routing/stats - // populates the `tiers` field for those routes too. - ctx.selected_target().map(|t| t.as_str().to_string()) -} diff --git a/crates/switchyard-components/src/stats/cost.rs b/crates/switchyard-components/src/stats/cost.rs deleted file mode 100644 index 8d1ba43ce..000000000 --- a/crates/switchyard-components/src/stats/cost.rs +++ /dev/null @@ -1,269 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Best-effort token cost estimation for stats snapshots. - -use std::collections::BTreeMap; - -use serde::{Deserialize, Serialize}; - -use super::accumulator::ModelStatsSnapshot; - -/// Cost estimate for all recorded models. -/// -/// `total_cost` is the grand total spend including overhead calls; -/// `backend_cost` is the portion attributed to routed-backend traffic; -/// `classifier_cost` is the LLM-classifier-overhead portion (zero unless -/// `record_classifier_usage` has been called). The split exists because the -/// default TB-lite configs can use the same model id for the classifier and -/// backend buckets, and a single-row aggregation cannot distinguish them. -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] -pub struct CostEstimate { - pub models: BTreeMap, - pub total_cost: f64, - pub backend_cost: f64, - pub classifier_cost: f64, -} - -/// Per-model cost breakdown. -#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] -pub struct CostBreakdown { - pub base_input_cost: f64, - pub cached_input_cost: f64, - pub cache_write_cost: f64, - pub input_cost: f64, - pub output_cost: f64, - pub total_cost: f64, -} - -#[derive(Clone, Copy, Debug)] -struct ModelPrice { - input: f64, - output: f64, - cached: f64, - cache_write: f64, -} - -/// Estimates token cost from per-model stats. -pub fn estimate_cost(models: &BTreeMap) -> CostEstimate { - let mut estimated = BTreeMap::new(); - let mut total_cost = 0.0; - - for (model, stats) in models { - let breakdown = estimate_model_cost( - model, - stats.prompt_tokens, - stats.completion_tokens, - stats.cached_tokens, - stats.cache_creation_tokens, - ); - total_cost += breakdown.total_cost; - estimated.insert(model.clone(), breakdown); - } - - let total = round6(total_cost); - CostEstimate { - models: estimated, - total_cost: total, - backend_cost: total, - classifier_cost: 0.0, - } -} - -pub fn estimate_model_cost( - model: &str, - prompt_tokens: u64, - completion_tokens: u64, - cached_tokens: u64, - cache_creation_tokens: u64, -) -> CostBreakdown { - let prices = raw_model_price(model).unwrap_or(ModelPrice { - input: 0.0, - output: 0.0, - cached: 0.0, - cache_write: 0.0, - }); - let base_input = prompt_tokens - .saturating_sub(cached_tokens) - .saturating_sub(cache_creation_tokens); - let base_input_cost = base_input as f64 / 1e6 * prices.input; - let cached_input_cost = cached_tokens as f64 / 1e6 * prices.cached; - let cache_write_cost = cache_creation_tokens as f64 / 1e6 * prices.cache_write; - let input_cost = base_input_cost + cached_input_cost + cache_write_cost; - let output_cost = completion_tokens as f64 / 1e6 * prices.output; - CostBreakdown { - base_input_cost: round6(base_input_cost), - cached_input_cost: round6(cached_input_cost), - cache_write_cost: round6(cache_write_cost), - input_cost: round6(input_cost), - output_cost: round6(output_cost), - total_cost: round6(input_cost + output_cost), - } -} - -pub fn has_model_price(model: &str) -> bool { - raw_model_price(model).is_some() -} - -fn raw_model_price(model: &str) -> Option { - let price = match model { - "openai/openai/gpt-5.2" | "openai/openai/openai/gpt-5.2" => ModelPrice { - input: 1.75, - output: 14.00, - cached: 0.175, - cache_write: 1.75, - }, - "nvidia/nvidia/nemotron-3-super-v3" | "openai/nvidia/nvidia/nemotron-3-super-v3" => { - ModelPrice { - input: 0.10, - output: 0.50, - cached: 0.01, - cache_write: 0.10, - } - } - // Moonshot Kimi K2.6 — official platform.kimi.ai pricing (May 2026). - // OpenAI wire format on NVIDIA Inference Hub; no cache_write - // premium (cache_write equals input). - "nvidia/moonshotai/kimi-k2.6" | "openai/nvidia/moonshotai/kimi-k2.6" => ModelPrice { - input: 0.95, - output: 4.00, - cached: 0.16, - cache_write: 0.95, - }, - // Moonshot Kimi K2 / K2.5 — platform.kimi.ai (k2-thinking variant - // pricing, the closest standard tier; NVIDIA hub serves this as - // ``kimi-k2.5``). Same no-cache-write-premium posture. - "nvidia/moonshotai/kimi-k2.5" | "openai/nvidia/moonshotai/kimi-k2.5" => ModelPrice { - input: 0.60, - output: 2.50, - cached: 0.15, - cache_write: 0.60, - }, - // DeepSeek V4 Flash — official api-docs.deepseek.com standard list - // price (post-promo). 284B total / 13B active, 1M-token context - // window. Aggressive cache discount (98% off on hits). OpenAI - // wire format on NVIDIA hub; no cache_write premium. - "nvidia/deepseek-ai/deepseek-v4-flash" - | "openai/nvidia/deepseek-ai/deepseek-v4-flash" - | "deepseek-v4-flash" => ModelPrice { - input: 0.14, - output: 0.28, - cached: 0.0028, - cache_write: 0.14, - }, - // DeepSeek V4 Pro — official api-docs.deepseek.com standard list - // price (post-promo). 1.6T total / 49B active, 1M-token context - // window. Pro tier is currently under a 75% promotional discount - // through 2026-05-31 UTC ($0.435 / $0.87 effective); we price at - // the standard rate for stable cost-model comparisons that - // outlive the promo window. ``evals-`` prefix variant is the - // same model exposed on NVIDIA Inference Hub's benchmarking - // gateway (paired with the ``X-Inference-Priority: batch`` - // header) — it bypasses the regular gateway's 6-min timeout - // that under high concurrency manifests as cascading 504s on - // V4-class models. Same per-token pricing. - "nvidia/deepseek-ai/deepseek-v4-pro" - | "openai/nvidia/deepseek-ai/deepseek-v4-pro" - | "nvidia/deepseek-ai/evals-deepseek-v4-pro" - | "openai/nvidia/deepseek-ai/evals-deepseek-v4-pro" - | "deepseek-v4-pro" => ModelPrice { - input: 1.74, - output: 3.48, - cached: 0.0145, - cache_write: 1.74, - }, - // Gemini 3.5 Flash — Google Vertex global list price (ai.google.dev): - // input $1.50, output $9.00, cache-read $0.15 (90% off input). The - // default LLM-classifier model (gcp wire). No per-token cache-write - // premium (cache storage is billed per-hour, not per-token), so - // cache_write = input. - "gcp/google/gemini-3.5-flash" - | "openai/gcp/google/gemini-3.5-flash" - | "gemini-3.5-flash" => ModelPrice { - input: 1.50, - output: 9.00, - cached: 0.15, - cache_write: 1.50, - }, - // Nemotron 3 Ultra (550B/55B MoE) — OpenRouter reference list - // price, June 2026. evals-N ids are parallel benchmarking-gateway - // deployments of the same model. - "nvidia/nvidia/nemotron-3-ultra" - | "openai/nvidia/nvidia/nemotron-3-ultra" - | "nvidia/nvidia/evals-nemotron-ultra" - | "nvidia/nvidia/evals-nemotron-ultra-2" - | "nvidia/nvidia/evals-nemotron-ultra-3" - | "nvidia/nvidia/evals-nemotron-ultra-4" => ModelPrice { - input: 0.50, - output: 2.20, - cached: 0.05, - cache_write: 0.50, - }, - "aws/anthropic/bedrock-claude-opus-4-8" - | "aws/anthropic/bedrock-claude-opus-4-7" - | "aws/anthropic/bedrock-claude-opus-4-6" - | "aws/anthropic/bedrock-claude-opus-4-5" - | "azure/anthropic/claude-opus-4-8" - | "azure/anthropic/claude-opus-4-7" - | "azure/anthropic/claude-opus-4-6" - | "claude-opus-4-8" - | "claude-opus-4-7" - | "claude-opus-4-6" - | "claude-opus-4-5" => ModelPrice { - input: 5.00, - output: 25.00, - cached: 0.50, - cache_write: 6.25, - }, - "aws/anthropic/bedrock-claude-sonnet-4-6" - | "aws/anthropic/bedrock-claude-sonnet-4-5" - | "azure/anthropic/claude-sonnet-4-5" - | "claude-sonnet-4-6" - | "claude-sonnet-4-5" => ModelPrice { - input: 3.00, - output: 15.00, - cached: 0.30, - cache_write: 3.75, - }, - "aws/anthropic/bedrock-claude-haiku-4-5" | "claude-haiku-4-5" => ModelPrice { - input: 1.00, - output: 5.00, - cached: 0.10, - cache_write: 1.25, - }, - _ => return None, - }; - Some(price) -} - -fn round6(value: f64) -> f64 { - (value * 1_000_000.0).round() / 1_000_000.0 -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn gemini_3_5_flash_is_priced() { - // The default LLM-classifier model must not silently cost $0. - for model in [ - "gcp/google/gemini-3.5-flash", - "openai/gcp/google/gemini-3.5-flash", - "gemini-3.5-flash", - ] { - assert!(has_model_price(model), "{model} should be priced"); - let cost = estimate_model_cost(model, 1_000_000, 1_000_000, 0, 0); - assert_eq!(cost.base_input_cost, 1.50); - assert_eq!(cost.output_cost, 9.00); - assert_eq!(cost.total_cost, 10.50); - } - } - - #[test] - fn unknown_model_defaults_to_zero() { - assert!(!has_model_price("nvidia/qwen/qwen3.6-35b-a3b")); - let cost = estimate_model_cost("nvidia/qwen/qwen3.6-35b-a3b", 1_000_000, 1_000_000, 0, 0); - assert_eq!(cost.total_cost, 0.0); - } -} diff --git a/crates/switchyard-components/src/stats/mod.rs b/crates/switchyard-components/src/stats/mod.rs deleted file mode 100644 index e88efac23..000000000 --- a/crates/switchyard-components/src/stats/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Shared stats accounting used by stats processors and backend wrappers. - -mod accumulator; -mod cache_eligibility; -mod context; -mod cost; -mod usage; - -pub use accumulator::{ - ClassifierStatsSnapshot, LatencyHistogramSnapshot, ModelStatsSnapshot, StatsAccumulator, - StatsSnapshot, TierStatsSnapshot, TokenTotals, -}; -pub use cache_eligibility::{PrefixProbe, prefix_probe, tracking_enabled_from_env}; -pub use context::{ - StatsBackendLatency, StatsRequestStart, StatsRouteLabel, selected_stats_model, - selected_stats_tier, -}; -pub use cost::{CostBreakdown, CostEstimate, estimate_model_cost, has_model_price}; -pub use usage::{ - AnthropicStreamUsage, TokenUsage, openai_chat_usage_from_stream_event, - openai_responses_usage_from_stream_event, usage_from_body, -}; diff --git a/crates/switchyard-components/src/stats/usage.rs b/crates/switchyard-components/src/stats/usage.rs deleted file mode 100644 index 10b939a30..000000000 --- a/crates/switchyard-components/src/stats/usage.rs +++ /dev/null @@ -1,221 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Provider usage extraction for buffered and streaming responses. - -use crate::StreamEvent; -use serde::{Deserialize, Serialize}; -use serde_json::Value; - -/// Normalized token usage counters. -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] -pub struct TokenUsage { - pub prompt_tokens: u64, - pub completion_tokens: u64, - pub cached_tokens: u64, - pub cache_creation_tokens: u64, - pub reasoning_tokens: u64, - /// Set by the stats response processor, not by provider usage extraction. - #[serde(default)] - pub cacheable_prompt_tokens: u64, -} - -impl TokenUsage { - /// Returns whether all counters are zero. - pub fn is_zero(self) -> bool { - self.prompt_tokens == 0 - && self.completion_tokens == 0 - && self.cached_tokens == 0 - && self.cache_creation_tokens == 0 - && self.reasoning_tokens == 0 - } -} - -/// Extracts usage from a buffered response body. -pub fn usage_from_body(body: &Value) -> TokenUsage { - body.get("usage") - .and_then(usage_from_candidate) - .unwrap_or_default() -} - -/// Extracts OpenAI Chat streaming usage from an event. -pub fn openai_chat_usage_from_stream_event(event: &StreamEvent) -> Option { - let StreamEvent::Json(value) = event else { - return None; - }; - value.get("usage").and_then(usage_from_candidate) -} - -/// Extracts OpenAI Responses streaming usage from an event. -/// -/// Fidelity-preserving backends yield raw SSE frame *strings* -/// (`StreamEvent::Text`) instead of decoded JSON events; those frames are -/// parsed here so usage accounting survives verbatim passthrough. -pub fn openai_responses_usage_from_stream_event(event: &StreamEvent) -> Option { - match event { - StreamEvent::Json(value) => usage_from_responses_value(value), - StreamEvent::Text(text) => sse_data_payloads(text) - .iter() - .find_map(usage_from_responses_value), - } -} - -/// Reads `response.usage` from one decoded Responses stream event. -fn usage_from_responses_value(value: &Value) -> Option { - value - .get("response") - .and_then(|response| response.get("usage")) - .and_then(usage_from_candidate) -} - -/// Parses the JSON `data:` payload(s) out of a raw SSE frame string. -/// -/// Per the SSE contract, a frame's `data:` lines join with newlines to form -/// one payload; a single leading space after the colon is stripped. Comment -/// frames, `[DONE]` sentinels, and non-JSON payloads yield nothing. -fn sse_data_payloads(text: &str) -> Vec { - let mut payloads = Vec::new(); - for block in text.split("\n\n") { - let data_lines: Vec<&str> = block - .split('\n') - .filter_map(|line| line.strip_prefix("data:")) - .map(|value| value.strip_prefix(' ').unwrap_or(value)) - .collect(); - if data_lines.is_empty() { - continue; - } - let data = data_lines.join("\n"); - if data.trim() == "[DONE]" { - continue; - } - if let Ok(parsed) = serde_json::from_str::(&data) { - payloads.push(parsed); - } - } - payloads -} - -/// Accumulates Anthropic streaming usage and commits once at `message_stop`. -#[derive(Clone, Copy, Debug, Default)] -pub struct AnthropicStreamUsage { - input_tokens: u64, - output_tokens: u64, - cache_read_input_tokens: u64, - cache_creation_input_tokens: u64, - saw_usage: bool, - committed: bool, -} - -impl AnthropicStreamUsage { - /// Observes one stream event and returns usage exactly once at `message_stop`. - /// A stop event before any usage frame is a known no-op, matching Python stream taps. - pub fn observe(&mut self, event: &StreamEvent) -> Option { - let StreamEvent::Json(value) = event else { - return None; - }; - match value.get("type").and_then(Value::as_str) { - Some("message_start") => { - if let Some(usage) = value - .get("message") - .and_then(|message| message.get("usage")) - { - self.merge(usage); - } - None - } - Some("message_delta") => { - if let Some(usage) = value - .get("usage") - .or_else(|| value.get("delta").and_then(|delta| delta.get("usage"))) - { - self.merge(usage); - } - None - } - Some("message_stop") if self.saw_usage && !self.committed => { - self.committed = true; - Some(TokenUsage { - prompt_tokens: self - .input_tokens - .saturating_add(self.cache_read_input_tokens) - .saturating_add(self.cache_creation_input_tokens), - completion_tokens: self.output_tokens, - cached_tokens: self.cache_read_input_tokens, - cache_creation_tokens: self.cache_creation_input_tokens, - reasoning_tokens: 0, - cacheable_prompt_tokens: 0, - }) - } - _ => None, - } - } - - fn merge(&mut self, usage: &Value) { - if !usage.is_object() { - return; - } - self.saw_usage = true; - if let Some(value) = int_field(usage, "input_tokens") { - self.input_tokens = value; - } - if let Some(value) = int_field(usage, "output_tokens") { - self.output_tokens = value; - } - if let Some(value) = int_field(usage, "cache_read_input_tokens") { - self.cache_read_input_tokens = value; - } - if let Some(value) = int_field(usage, "cache_creation_input_tokens") { - self.cache_creation_input_tokens = value; - } - } -} - -fn usage_from_candidate(usage: &Value) -> Option { - usage.is_object().then(|| usage_from_value(usage)) -} - -fn usage_from_value(usage: &Value) -> TokenUsage { - let completion_tokens = int_field(usage, "completion_tokens") - .unwrap_or_else(|| int_field(usage, "output_tokens").unwrap_or(0)); - let mut output = TokenUsage { - completion_tokens, - ..TokenUsage::default() - }; - - if let Some(prompt_tokens) = int_field(usage, "prompt_tokens") { - output.prompt_tokens = prompt_tokens; - if let Some(details) = usage.get("prompt_tokens_details") { - output.cached_tokens = int_field(details, "cached_tokens").unwrap_or(0); - output.cache_creation_tokens = int_field(details, "cache_creation_tokens").unwrap_or(0); - } - } else { - let base = int_field(usage, "input_tokens").unwrap_or(0); - if let Some(details) = usage.get("input_tokens_details") { - output.cached_tokens = int_field(details, "cached_tokens").unwrap_or(0); - } - let cache_read = int_field(usage, "cache_read_input_tokens").unwrap_or(0); - let cache_creation = int_field(usage, "cache_creation_input_tokens").unwrap_or(0); - if output.cached_tokens == 0 { - output.cached_tokens = cache_read; - } - output.cache_creation_tokens = cache_creation; - output.prompt_tokens = base - .saturating_add(cache_read) - .saturating_add(cache_creation); - } - - output.reasoning_tokens = usage - .get("completion_tokens_details") - .and_then(|details| int_field(details, "reasoning_tokens")) - .or_else(|| { - usage - .get("output_tokens_details") - .and_then(|details| int_field(details, "reasoning_tokens")) - }) - .unwrap_or(0); - output -} - -fn int_field(value: &Value, name: &str) -> Option { - value.get(name).and_then(Value::as_u64) -} diff --git a/crates/switchyard-components/src/telemetry.rs b/crates/switchyard-components/src/telemetry.rs deleted file mode 100644 index 10b7dfd66..000000000 --- a/crates/switchyard-components/src/telemetry.rs +++ /dev/null @@ -1,97 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Shared version and telemetry helpers for Rust-owned components. - -use std::env; - -pub(crate) const SWITCHYARD_VERSION_HEADER: &str = "X-Switchyard-Version"; - -const SWITCHYARD_VERSION_ENV: &str = "SWITCHYARD_VERSION"; -const SWITCHYARD_TELEMETRY_OPT_OUT_ENV: &str = "SWITCHYARD_TELEMETRY_OPT_OUT"; -const NEMO_SWITCHYARD_TELEMETRY_OPT_OUT_ENV: &str = "NEMO_SWITCHYARD_TELEMETRY_OPT_OUT"; - -pub(crate) fn telemetry_header_value() -> Option { - telemetry_header_value_from_values( - env::var(SWITCHYARD_TELEMETRY_OPT_OUT_ENV).ok().as_deref(), - env::var(NEMO_SWITCHYARD_TELEMETRY_OPT_OUT_ENV) - .ok() - .as_deref(), - env::var(SWITCHYARD_VERSION_ENV).ok().as_deref(), - ) -} - -fn telemetry_header_value_from_values( - opt_out: Option<&str>, - legacy_opt_out: Option<&str>, - version: Option<&str>, -) -> Option { - if env_value_opts_out(opt_out) || env_value_opts_out(legacy_opt_out) { - return None; - } - Some(configured_version(version).unwrap_or_else(|| env!("CARGO_PKG_VERSION").to_string())) -} - -fn configured_version(value: Option<&str>) -> Option { - value - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_string) -} - -fn env_value_opts_out(value: Option<&str>) -> bool { - let Some(value) = value.map(str::trim) else { - return false; - }; - !matches!( - value.to_ascii_lowercase().as_str(), - "" | "0" | "false" | "no" - ) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn telemetry_header_uses_configured_version_when_not_opted_out() { - assert_eq!( - telemetry_header_value_from_values(None, None, Some(" 1.2.3 ")), - Some("1.2.3".to_string()) - ); - } - - #[test] - fn telemetry_header_falls_back_to_crate_version() { - assert_eq!( - telemetry_header_value_from_values(None, None, None), - Some(env!("CARGO_PKG_VERSION").to_string()) - ); - assert_eq!( - telemetry_header_value_from_values(None, None, Some(" ")), - Some(env!("CARGO_PKG_VERSION").to_string()) - ); - } - - #[test] - fn telemetry_header_respects_current_and_legacy_opt_out_env_values() { - assert_eq!( - telemetry_header_value_from_values(Some("true"), None, Some("1.2.3")), - None - ); - assert_eq!( - telemetry_header_value_from_values(None, Some("yes"), Some("1.2.3")), - None - ); - } - - #[test] - fn telemetry_header_ignores_falsey_opt_out_values() { - for value in ["", "0", "false", "FALSE", "no", " No "] { - assert_eq!( - telemetry_header_value_from_values(Some(value), None, Some("1.2.3")), - Some("1.2.3".to_string()) - ); - } - } -} diff --git a/crates/switchyard-components/tests/adversarial_multi_llm_backend.rs b/crates/switchyard-components/tests/adversarial_multi_llm_backend.rs deleted file mode 100644 index d17772575..000000000 --- a/crates/switchyard-components/tests/adversarial_multi_llm_backend.rs +++ /dev/null @@ -1,1268 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Adversarial tests for the Rust multi-target LLM backend. - -use std::sync::Arc; - -use async_trait::async_trait; -use parking_lot::Mutex; -use serde_json::{Value, json}; -use switchyard_components::{ - BackendFormat, ChatRequest, ChatRequestType, ChatResponse, LlmBackend, LlmTarget, LlmTargetId, - ModelId, ProxyContext, Result, SwitchyardError, -}; -use switchyard_components::{ - BackendSelection, BackendSelectionReason, LlmTargetBackend, MultiLlmBackend, -}; - -/// One backend call observed by the recording backend. -#[derive(Clone, Debug, PartialEq)] -struct ObservedCall { - /// Name of the backend that handled the request. - backend_name: &'static str, - /// Request wire type received by the child backend. - request_type: ChatRequestType, - /// Request model visible to the child backend. - model: Option, - /// Full request body visible to the child backend. - body: Value, - /// Context-selected target visible at delegation time. - ctx_selected_target: Option, - /// Context-selected model visible at delegation time. - ctx_selected_model: Option, -} - -/// Mutex-backed shared vector used for test observations. -#[derive(Clone)] -struct Shared(Arc>>); - -impl Default for Shared { - fn default() -> Self { - Self(Arc::new(Mutex::new(Vec::new()))) - } -} - -impl Shared { - /// Appends one observation. - fn push(&self, value: T) -> Result<()> { - self.0.lock().push(value); - Ok(()) - } - - /// Returns a cloned copy of all observations. - fn values(&self) -> Result> { - Ok(self.0.lock().clone()) - } -} - -/// Backend fixture that records calls and optional lifecycle failures. -struct RecordingBackend { - /// Stable backend name for assertions. - name: &'static str, - /// Captured backend calls. - calls: Shared, - /// Captured startup/shutdown events. - events: Shared, - /// Request types this fixture advertises. - supported_request_types: &'static [ChatRequestType], - /// Optional call error. - call_error: Option<&'static str>, - /// Optional startup error. - startup_error: Option<&'static str>, - /// Optional shutdown error. - shutdown_error: Option<&'static str>, -} - -impl RecordingBackend { - /// Creates a recording backend with all request types enabled. - fn new(name: &'static str, calls: Shared, events: Shared) -> Self { - Self { - name, - calls, - events, - supported_request_types: &ALL_REQUEST_TYPES, - call_error: None, - startup_error: None, - shutdown_error: None, - } - } - - /// Overrides the advertised request types. - fn with_supported_request_types( - mut self, - supported_request_types: &'static [ChatRequestType], - ) -> Self { - self.supported_request_types = supported_request_types; - self - } - - /// Configures a call-time backend error. - fn with_call_error(mut self, message: &'static str) -> Self { - self.call_error = Some(message); - self - } - - /// Configures a startup failure. - fn with_startup_error(mut self, message: &'static str) -> Self { - self.startup_error = Some(message); - self - } - - /// Configures a shutdown failure. - fn with_shutdown_error(mut self, message: &'static str) -> Self { - self.shutdown_error = Some(message); - self - } -} - -static ALL_REQUEST_TYPES: [ChatRequestType; 3] = [ - ChatRequestType::OpenAiChat, - ChatRequestType::OpenAiResponses, - ChatRequestType::Anthropic, -]; -static OPENAI_CHAT_ONLY: [ChatRequestType; 1] = [ChatRequestType::OpenAiChat]; - -#[async_trait] -impl LlmBackend for RecordingBackend { - fn supported_request_types(&self) -> &[ChatRequestType] { - self.supported_request_types - } - - async fn call(&self, ctx: &mut ProxyContext, request: &ChatRequest) -> Result { - self.calls.push(ObservedCall { - backend_name: self.name, - request_type: request.request_type(), - model: request.model().map(str::to_string), - body: request.body().clone(), - ctx_selected_target: selected_target(ctx).cloned(), - ctx_selected_model: selected_model(ctx).cloned(), - })?; - if let Some(message) = self.call_error { - return Err(SwitchyardError::Backend(message.to_string())); - } - Ok(ChatResponse::openai_completion(json!({ - "backend": self.name, - "model": request.model(), - }))) - } - - async fn startup(&self) -> Result<()> { - self.events.push(format!("{}:startup", self.name))?; - if let Some(message) = self.startup_error { - return Err(SwitchyardError::Backend(message.to_string())); - } - Ok(()) - } - - async fn shutdown(&self) -> Result<()> { - self.events.push(format!("{}:shutdown", self.name))?; - if let Some(message) = self.shutdown_error { - return Err(SwitchyardError::Backend(message.to_string())); - } - Ok(()) - } -} - -/// Builds an OpenAI-format test target. -fn target(id: &'static str, model: &'static str) -> LlmTarget { - let mut target = LlmTarget::new(LlmTargetId::from_static(id), ModelId::from_static(model)); - target.format = BackendFormat::OpenAi; - target -} - -/// Builds a target/backend pair for `MultiLlmBackend`. -fn target_backend( - id: &'static str, - model: &'static str, - backend: RecordingBackend, -) -> LlmTargetBackend { - LlmTargetBackend::new(target(id, model), Arc::new(backend)) -} - -/// Builds an OpenAI Chat request fixture. -fn request(model: &str) -> ChatRequest { - ChatRequest::openai_chat(json!({ - "model": model, - "messages": [{"role": "user", "content": "preserve me"}], - "temperature": 0.2 - })) -} - -/// Builds an Anthropic request fixture. -fn anthropic_request(model: &str) -> ChatRequest { - ChatRequest::anthropic(json!({ - "model": model, - "max_tokens": 128, - "messages": [{"role": "user", "content": "preserve anthropic"}], - "metadata": {"kept": true} - })) -} - -/// Builds a Responses API request fixture. -fn responses_request(model: &str) -> ChatRequest { - ChatRequest::openai_responses(json!({ - "model": model, - "input": "preserve responses", - "metadata": {"kept": true} - })) -} - -/// Returns the backend selection stamped by `MultiLlmBackend`. -fn selection(ctx: &ProxyContext) -> Result<&BackendSelection> { - ctx.get::() - .ok_or_else(|| SwitchyardError::Other("multi-LLM selection should be recorded".to_string())) -} - -/// Returns the selected model stamped in context. -fn selected_model(ctx: &ProxyContext) -> Option<&ModelId> { - ctx.get::() - .map(|selection| &selection.model) -} - -/// Returns the selected target stamped in context. -fn selected_target(ctx: &ProxyContext) -> Option<&LlmTargetId> { - ctx.get::() - .and_then(|selection| selection.target_id.as_ref()) -} - -// Default support should cover all inbound wire formats in stable order. -#[test] -fn default_supported_request_types_are_all_wire_formats_in_stable_order() -> Result<()> { - let calls = Shared::default(); - let events = Shared::default(); - let backend = MultiLlmBackend::new([target_backend( - "only-target", - "served-model", - RecordingBackend::new("only", calls, events), - )])?; - - assert_eq!( - backend.supported_request_types(), - &[ - ChatRequestType::OpenAiChat, - ChatRequestType::OpenAiResponses, - ChatRequestType::Anthropic, - ] - ); - Ok(()) -} - -// Custom request type support should de-dupe without reordering caller input. -#[test] -fn custom_supported_request_types_are_deduped_without_reordering() -> Result<()> { - let calls = Shared::default(); - let events = Shared::default(); - let backend = MultiLlmBackend::new([target_backend( - "only-target", - "served-model", - RecordingBackend::new("only", calls, events), - )])? - .with_supported_request_types([ - ChatRequestType::Anthropic, - ChatRequestType::OpenAiChat, - ChatRequestType::Anthropic, - ChatRequestType::OpenAiChat, - ChatRequestType::OpenAiResponses, - ])?; - - assert_eq!( - backend.supported_request_types(), - &[ - ChatRequestType::Anthropic, - ChatRequestType::OpenAiChat, - ChatRequestType::OpenAiResponses, - ] - ); - Ok(()) -} - -// Context-selected targets should win and cloned requests should keep caller state intact. -#[tokio::test] -async fn context_selected_target_wins_and_request_is_cloned() -> Result<()> { - let strong_calls = Shared::default(); - let weak_calls = Shared::default(); - let events = Shared::default(); - let backend = MultiLlmBackend::new([ - target_backend( - "strong-target", - "strong-model", - RecordingBackend::new("strong", strong_calls.clone(), events.clone()), - ), - target_backend( - "weak-target", - "weak-model", - RecordingBackend::new("weak", weak_calls.clone(), events.clone()), - ), - ])?; - let original = request("client-model"); - let mut ctx = ProxyContext::new(); - ctx.set_selected_target(LlmTargetId::from_static("weak-target")); - - let response = backend.call(&mut ctx, &original).await?; - - assert_eq!( - response.body(), - Some(&json!({"backend": "weak", "model": "weak-model"})) - ); - assert_eq!(original.model(), Some("client-model")); - assert_eq!( - selected_target(&ctx), - Some(&LlmTargetId::from_static("weak-target")) - ); - assert_eq!( - selected_model(&ctx), - Some(&ModelId::from_static("weak-model")) - ); - assert_eq!( - selection(&ctx)?.reason, - BackendSelectionReason::ContextTarget - ); - assert_eq!( - selection(&ctx)?.original_model.as_deref(), - Some("client-model") - ); - assert!(strong_calls.values()?.is_empty()); - - let weak = weak_calls.values()?; - assert_eq!(weak.len(), 1); - assert_eq!(weak[0].backend_name, "weak"); - assert_eq!(weak[0].request_type, ChatRequestType::OpenAiChat); - assert_eq!(weak[0].model.as_deref(), Some("weak-model")); - assert_eq!(weak[0].body["messages"][0]["content"], "preserve me"); - assert_eq!( - weak[0].ctx_selected_target, - Some(LlmTargetId::from_static("weak-target")) - ); - assert_eq!( - weak[0].ctx_selected_model, - Some(ModelId::from_static("weak-model")) - ); - Ok(()) -} - -// Explicit context target selection should override request model matches. -#[tokio::test] -async fn context_selected_target_wins_even_when_request_model_matches_another_target() -> Result<()> -{ - let strong_calls = Shared::default(); - let weak_calls = Shared::default(); - let events = Shared::default(); - let backend = MultiLlmBackend::new([ - target_backend( - "strong-target", - "strong-model", - RecordingBackend::new("strong", strong_calls.clone(), events.clone()), - ), - target_backend( - "weak-target", - "weak-model", - RecordingBackend::new("weak", weak_calls.clone(), events), - ), - ])?; - let mut ctx = ProxyContext::new(); - ctx.set_selected_target(LlmTargetId::from_static("weak-target")); - - backend.call(&mut ctx, &request("strong-model")).await?; - - assert!(strong_calls.values()?.is_empty()); - let weak = weak_calls.values()?; - assert_eq!(weak.len(), 1); - assert_eq!(weak[0].model.as_deref(), Some("weak-model")); - assert_eq!( - selection(&ctx)?.reason, - BackendSelectionReason::ContextTarget - ); - assert_eq!( - selection(&ctx)?.original_model.as_deref(), - Some("strong-model") - ); - Ok(()) -} - -// A single configured target should route without any selector processor. -#[tokio::test] -async fn single_target_fallback_routes_without_selector() -> Result<()> { - let calls = Shared::default(); - let events = Shared::default(); - let backend = MultiLlmBackend::new([target_backend( - "only-target", - "served-model", - RecordingBackend::new("only", calls.clone(), events), - )])?; - let mut ctx = ProxyContext::new(); - - backend.call(&mut ctx, &request("client-model")).await?; - - assert_eq!( - selected_target(&ctx), - Some(&LlmTargetId::from_static("only-target")) - ); - assert_eq!( - selected_model(&ctx), - Some(&ModelId::from_static("served-model")) - ); - assert_eq!( - selection(&ctx)?.reason, - BackendSelectionReason::SingleTarget - ); - let calls = calls.values()?; - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].model.as_deref(), Some("served-model")); - Ok(()) -} - -// Non-object request bodies should be repaired only in the delegated clone. -#[tokio::test] -async fn single_target_fallback_recovers_non_object_request_body_without_mutating_original() --> Result<()> { - let calls = Shared::default(); - let events = Shared::default(); - let backend = MultiLlmBackend::new([target_backend( - "only-target", - "served-model", - RecordingBackend::new("only", calls.clone(), events), - )])?; - let original = ChatRequest::openai_chat(json!("not an object")); - let mut ctx = ProxyContext::new(); - - backend.call(&mut ctx, &original).await?; - - assert_eq!(original.body(), &json!("not an object")); - let calls = calls.values()?; - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].body, json!({"model": "served-model"})); - assert_eq!(selection(&ctx)?.original_model, None); - Ok(()) -} - -// Anthropic request bodies should keep their wire shape while the model is rewritten. -#[tokio::test] -async fn routing_preserves_anthropic_payload_shape_while_rewriting_model() -> Result<()> { - let calls = Shared::default(); - let events = Shared::default(); - let backend = MultiLlmBackend::new([target_backend( - "anthropic-target", - "served-claude", - RecordingBackend::new("anthropic", calls.clone(), events), - )])?; - let original = anthropic_request("client-claude"); - let mut ctx = ProxyContext::new(); - - backend.call(&mut ctx, &original).await?; - - assert_eq!(original.model(), Some("client-claude")); - let calls = calls.values()?; - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].request_type, ChatRequestType::Anthropic); - assert_eq!(calls[0].model.as_deref(), Some("served-claude")); - assert_eq!(calls[0].body["max_tokens"], 128); - assert_eq!( - calls[0].body["messages"][0]["content"], - "preserve anthropic" - ); - assert_eq!(calls[0].body["metadata"], json!({"kept": true})); - Ok(()) -} - -// Responses request bodies should keep their wire shape while the model is rewritten. -#[tokio::test] -async fn routing_preserves_responses_payload_shape_while_rewriting_model() -> Result<()> { - let calls = Shared::default(); - let events = Shared::default(); - let backend = MultiLlmBackend::new([target_backend( - "responses-target", - "served-responses", - RecordingBackend::new("responses", calls.clone(), events), - )])?; - let original = responses_request("client-responses"); - let mut ctx = ProxyContext::new(); - - backend.call(&mut ctx, &original).await?; - - assert_eq!(original.model(), Some("client-responses")); - let calls = calls.values()?; - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].request_type, ChatRequestType::OpenAiResponses); - assert_eq!(calls[0].model.as_deref(), Some("served-responses")); - assert_eq!(calls[0].body["input"], "preserve responses"); - assert_eq!(calls[0].body["metadata"], json!({"kept": true})); - Ok(()) -} - -// Multi-LLM dispatch should not reject a selected child based on its native format list. -#[tokio::test] -async fn selected_child_backend_receives_request_even_when_its_direct_formats_do_not_match() --> Result<()> { - let calls = Shared::default(); - let events = Shared::default(); - let backend = MultiLlmBackend::new([target_backend( - "translated-target", - "served-model", - RecordingBackend::new("translated", calls.clone(), events) - .with_supported_request_types(&OPENAI_CHAT_ONLY), - )])?; - let mut ctx = ProxyContext::new(); - - backend - .call(&mut ctx, &anthropic_request("client-claude")) - .await?; - - let calls = calls.values()?; - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].request_type, ChatRequestType::Anthropic); - assert_eq!(calls[0].model.as_deref(), Some("served-model")); - assert_eq!( - calls[0].ctx_selected_target, - Some(LlmTargetId::from_static("translated-target")) - ); - Ok(()) -} - -// Request model should select a unique matching target when context is empty. -#[tokio::test] -async fn request_model_selects_unique_target_when_context_is_empty() -> Result<()> { - let strong_calls = Shared::default(); - let weak_calls = Shared::default(); - let events = Shared::default(); - let backend = MultiLlmBackend::new([ - target_backend( - "strong-target", - "strong-model", - RecordingBackend::new("strong", strong_calls.clone(), events.clone()), - ), - target_backend( - "weak-target", - "weak-model", - RecordingBackend::new("weak", weak_calls.clone(), events), - ), - ])?; - let mut ctx = ProxyContext::new(); - - backend.call(&mut ctx, &request("strong-model")).await?; - - assert_eq!( - selected_target(&ctx), - Some(&LlmTargetId::from_static("strong-target")) - ); - assert_eq!( - selection(&ctx)?.reason, - BackendSelectionReason::RequestModel - ); - assert_eq!(strong_calls.values()?.len(), 1); - assert!(weak_calls.values()?.is_empty()); - Ok(()) -} - -// Configured default targets should handle model-less requests. -#[tokio::test] -async fn configured_default_target_routes_when_no_selector_ran() -> Result<()> { - let strong_calls = Shared::default(); - let weak_calls = Shared::default(); - let events = Shared::default(); - let backend = MultiLlmBackend::new([ - target_backend( - "strong-target", - "strong-model", - RecordingBackend::new("strong", strong_calls.clone(), events.clone()), - ), - target_backend( - "weak-target", - "weak-model", - RecordingBackend::new("weak", weak_calls.clone(), events), - ), - ])? - .with_default_target(LlmTargetId::from_static("strong-target"))?; - let mut ctx = ProxyContext::new(); - let request = ChatRequest::openai_chat(json!({ - "messages": [{"role": "user", "content": "no model from client"}] - })); - - backend.call(&mut ctx, &request).await?; - - assert_eq!( - selected_target(&ctx), - Some(&LlmTargetId::from_static("strong-target")) - ); - assert_eq!( - selected_model(&ctx), - Some(&ModelId::from_static("strong-model")) - ); - assert_eq!( - selection(&ctx)?.reason, - BackendSelectionReason::DefaultTarget - ); - assert_eq!(strong_calls.values()?.len(), 1); - assert!(weak_calls.values()?.is_empty()); - Ok(()) -} - -// Configured default targets intentionally override request-model selection. -#[tokio::test] -async fn configured_default_target_overrides_request_model_selection() -> Result<()> { - let strong_calls = Shared::default(); - let weak_calls = Shared::default(); - let events = Shared::default(); - let backend = MultiLlmBackend::new([ - target_backend( - "strong-target", - "strong-model", - RecordingBackend::new("strong", strong_calls.clone(), events.clone()), - ), - target_backend( - "weak-target", - "weak-model", - RecordingBackend::new("weak", weak_calls.clone(), events), - ), - ])? - .with_default_target(LlmTargetId::from_static("strong-target"))?; - let mut ctx = ProxyContext::new(); - - backend.call(&mut ctx, &request("weak-model")).await?; - - assert_eq!( - selected_target(&ctx), - Some(&LlmTargetId::from_static("strong-target")) - ); - assert_eq!( - selection(&ctx)?.reason, - BackendSelectionReason::DefaultTarget - ); - assert_eq!( - selection(&ctx)?.original_model.as_deref(), - Some("weak-model") - ); - assert_eq!(strong_calls.values()?.len(), 1); - assert!(weak_calls.values()?.is_empty()); - Ok(()) -} - -// Context-selected targets still have the highest routing priority. -#[tokio::test] -async fn explicit_context_target_still_wins_over_configured_default_target() -> Result<()> { - let strong_calls = Shared::default(); - let weak_calls = Shared::default(); - let events = Shared::default(); - let backend = MultiLlmBackend::new([ - target_backend( - "strong-target", - "strong-model", - RecordingBackend::new("strong", strong_calls.clone(), events.clone()), - ), - target_backend( - "weak-target", - "weak-model", - RecordingBackend::new("weak", weak_calls.clone(), events), - ), - ])? - .with_default_target(LlmTargetId::from_static("strong-target"))?; - let mut ctx = ProxyContext::new(); - ctx.set_selected_target(LlmTargetId::from_static("weak-target")); - - backend.call(&mut ctx, &request("client-model")).await?; - - assert_eq!( - selection(&ctx)?.reason, - BackendSelectionReason::ContextTarget - ); - assert!(strong_calls.values()?.is_empty()); - assert_eq!(weak_calls.values()?.len(), 1); - Ok(()) -} - -// Ambiguous model-less requests should fail before any backend sees them. -#[tokio::test] -async fn model_less_request_with_multiple_targets_fails_without_mutating_context() -> Result<()> { - let calls = Shared::default(); - let events = Shared::default(); - let backend = MultiLlmBackend::new([ - target_backend( - "left-target", - "left-model", - RecordingBackend::new("left", calls.clone(), events.clone()), - ), - target_backend( - "right-target", - "right-model", - RecordingBackend::new("right", calls.clone(), events), - ), - ])?; - let mut ctx = ProxyContext::new(); - let request = ChatRequest::openai_chat(json!({ - "messages": [{"role": "user", "content": "no model"}] - })); - - let Err(error) = backend.call(&mut ctx, &request).await else { - return Err(SwitchyardError::Other( - "model-less multi-target request should fail".to_string(), - )); - }; - - assert!(matches!(error, SwitchyardError::InvalidConfig(_))); - assert_eq!(selected_target(&ctx), None); - assert_eq!(selected_model(&ctx), None); - assert!(ctx.get::().is_none()); - assert!(calls.values()?.is_empty()); - Ok(()) -} - -// Duplicate model names are legal when an explicit target disambiguates them. -#[tokio::test] -async fn duplicate_models_route_successfully_when_target_is_explicit() -> Result<()> { - let left_calls = Shared::default(); - let right_calls = Shared::default(); - let events = Shared::default(); - let backend = MultiLlmBackend::new([ - target_backend( - "left-target", - "shared-model", - RecordingBackend::new("left", left_calls.clone(), events.clone()), - ), - target_backend( - "right-target", - "shared-model", - RecordingBackend::new("right", right_calls.clone(), events), - ), - ])?; - let mut ctx = ProxyContext::new(); - ctx.set_selected_target(LlmTargetId::from_static("right-target")); - - backend.call(&mut ctx, &request("shared-model")).await?; - - assert!(left_calls.values()?.is_empty()); - let right = right_calls.values()?; - assert_eq!(right.len(), 1); - assert_eq!(right[0].backend_name, "right"); - assert_eq!( - selection(&ctx)?.reason, - BackendSelectionReason::ContextTarget - ); - assert_eq!( - selection(&ctx)?.target_id, - Some(LlmTargetId::from_static("right-target")) - ); - Ok(()) -} - -// Successful explicit routing should replace stale context selection before delegation. -#[tokio::test] -async fn explicit_target_overwrites_stale_selected_model_and_selection_before_delegation() --> Result<()> { - let calls = Shared::default(); - let events = Shared::default(); - let backend = MultiLlmBackend::new([target_backend( - "fresh-target", - "fresh-model", - RecordingBackend::new("fresh", calls.clone(), events), - )])?; - let mut ctx = ProxyContext::new(); - ctx.set_selected_target(LlmTargetId::from_static("fresh-target")); - let _ = ctx.insert(BackendSelection { - target_id: Some(LlmTargetId::from_static("stale-target")), - model: ModelId::from_static("stale-model"), - original_model: Some("stale-client".to_string()), - reason: BackendSelectionReason::RequestModel, - }); - - backend.call(&mut ctx, &request("client-model")).await?; - - assert_eq!( - selected_model(&ctx), - Some(&ModelId::from_static("fresh-model")) - ); - let calls = calls.values()?; - assert_eq!(calls.len(), 1); - assert_eq!( - calls[0].ctx_selected_model, - Some(ModelId::from_static("fresh-model")) - ); - let selection = selection(&ctx)?; - assert_eq!( - selection.target_id, - Some(LlmTargetId::from_static("fresh-target")) - ); - assert_eq!(selection.model, ModelId::from_static("fresh-model")); - assert_eq!(selection.original_model.as_deref(), Some("client-model")); - assert_eq!(selection.reason, BackendSelectionReason::ContextTarget); - Ok(()) -} - -// Unknown selected targets should fail before delegation. -#[tokio::test] -async fn unknown_selected_target_fails_before_any_backend_call() -> Result<()> { - let calls = Shared::default(); - let events = Shared::default(); - let backend = MultiLlmBackend::new([target_backend( - "known-target", - "known-model", - RecordingBackend::new("known", calls.clone(), events), - )])?; - let mut ctx = ProxyContext::new(); - ctx.set_selected_target(LlmTargetId::from_static("missing-target")); - - let Err(error) = backend.call(&mut ctx, &request("known-model")).await else { - return Err(SwitchyardError::Other( - "unknown selected target should fail".to_string(), - )); - }; - - assert!(matches!(error, SwitchyardError::InvalidConfig(_))); - assert!(calls.values()?.is_empty()); - Ok(()) -} - -// Failed routing should leave any existing context selection untouched. -#[tokio::test] -async fn failed_target_selection_does_not_replace_existing_context_selection() -> Result<()> { - let calls = Shared::default(); - let events = Shared::default(); - let backend = MultiLlmBackend::new([target_backend( - "known-target", - "known-model", - RecordingBackend::new("known", calls.clone(), events), - )])?; - let mut ctx = ProxyContext::new(); - ctx.set_selected_target(LlmTargetId::from_static("missing-target")); - let stale = BackendSelection { - target_id: Some(LlmTargetId::from_static("stale-target")), - model: ModelId::from_static("stale-model"), - original_model: Some("stale-client-model".to_string()), - reason: BackendSelectionReason::RequestModel, - }; - let _ = ctx.insert(stale.clone()); - - let Err(error) = backend.call(&mut ctx, &request("known-model")).await else { - return Err(SwitchyardError::Other( - "unknown selected target should fail".to_string(), - )); - }; - - assert!(matches!(error, SwitchyardError::InvalidConfig(_))); - assert_eq!(ctx.get::(), Some(&stale)); - assert!(calls.values()?.is_empty()); - Ok(()) -} - -// Multiple possible targets without a unique selector should be rejected. -#[tokio::test] -async fn multiple_targets_without_a_unique_selection_are_rejected() -> Result<()> { - let calls = Shared::default(); - let events = Shared::default(); - let backend = MultiLlmBackend::new([ - target_backend( - "left-target", - "left-model", - RecordingBackend::new("left", calls.clone(), events.clone()), - ), - target_backend( - "right-target", - "right-model", - RecordingBackend::new("right", calls.clone(), events), - ), - ])?; - let mut ctx = ProxyContext::new(); - - let Err(error) = backend.call(&mut ctx, &request("client-model")).await else { - return Err(SwitchyardError::Other( - "ambiguous request should fail".to_string(), - )); - }; - - assert!(matches!(error, SwitchyardError::InvalidConfig(_))); - assert!(calls.values()?.is_empty()); - Ok(()) -} - -// Duplicate model names require target-level disambiguation. -#[tokio::test] -async fn duplicate_models_require_explicit_target_selection() -> Result<()> { - let calls = Shared::default(); - let events = Shared::default(); - let backend = MultiLlmBackend::new([ - target_backend( - "left-target", - "shared-model", - RecordingBackend::new("left", calls.clone(), events.clone()), - ), - target_backend( - "right-target", - "shared-model", - RecordingBackend::new("right", calls.clone(), events), - ), - ])?; - let mut ctx = ProxyContext::new(); - - let Err(error) = backend.call(&mut ctx, &request("shared-model")).await else { - return Err(SwitchyardError::Other( - "duplicate model selection should fail".to_string(), - )); - }; - - assert!(matches!(error, SwitchyardError::InvalidConfig(_))); - assert!(calls.values()?.is_empty()); - Ok(()) -} - -// Successful calls should replace stale typed backend-selection metadata. -#[tokio::test] -async fn successful_call_replaces_stale_typed_selection_extension() -> Result<()> { - let calls = Shared::default(); - let events = Shared::default(); - let backend = MultiLlmBackend::new([target_backend( - "only-target", - "fresh-model", - RecordingBackend::new("only", calls, events), - )])?; - let mut ctx = ProxyContext::new(); - let _ = ctx.insert(BackendSelection { - target_id: Some(LlmTargetId::from_static("stale-target")), - model: ModelId::from_static("stale-model"), - original_model: None, - reason: BackendSelectionReason::RequestModel, - }); - - backend.call(&mut ctx, &request("client-model")).await?; - - let selection = selection(&ctx)?; - assert_eq!( - selection.target_id, - Some(LlmTargetId::from_static("only-target")) - ); - assert_eq!(selection.model, ModelId::from_static("fresh-model")); - assert_eq!(selection.original_model.as_deref(), Some("client-model")); - assert_eq!(selection.reason, BackendSelectionReason::SingleTarget); - Ok(()) -} - -// Unsupported request types should fail before child backend delegation. -#[tokio::test] -async fn unsupported_request_type_is_rejected_before_delegation() -> Result<()> { - let calls = Shared::default(); - let events = Shared::default(); - let backend = MultiLlmBackend::new([target_backend( - "only-target", - "served-model", - RecordingBackend::new("only", calls.clone(), events), - )])? - .with_supported_request_types([ChatRequestType::OpenAiChat])?; - let mut ctx = ProxyContext::new(); - let request = ChatRequest::anthropic(json!({ - "model": "served-model", - "max_tokens": 128, - "messages": [{"role": "user", "content": "hello"}] - })); - - let Err(error) = backend.call(&mut ctx, &request).await else { - return Err(SwitchyardError::Other( - "unsupported request type should fail".to_string(), - )); - }; - - assert!(matches!( - error, - SwitchyardError::UnsupportedRequestType { .. } - )); - assert!(calls.values()?.is_empty()); - Ok(()) -} - -// Child backend errors should propagate without trying other targets. -#[tokio::test] -async fn selected_backend_error_propagates_and_does_not_try_fallback_target() -> Result<()> { - let first_calls = Shared::default(); - let second_calls = Shared::default(); - let events = Shared::default(); - let backend = MultiLlmBackend::new([ - target_backend( - "first-target", - "first-model", - RecordingBackend::new("first", first_calls.clone(), events.clone()) - .with_call_error("upstream exploded"), - ), - target_backend( - "second-target", - "second-model", - RecordingBackend::new("second", second_calls.clone(), events), - ), - ])?; - let mut ctx = ProxyContext::new(); - ctx.set_selected_target(LlmTargetId::from_static("first-target")); - - let Err(error) = backend.call(&mut ctx, &request("client-model")).await else { - return Err(SwitchyardError::Other( - "selected backend error should propagate".to_string(), - )); - }; - - assert!(matches!(error, SwitchyardError::Backend(message) if message == "upstream exploded")); - let first_calls = first_calls.values()?; - assert_eq!(first_calls.len(), 1); - assert_eq!(first_calls[0].model.as_deref(), Some("first-model")); - assert!(second_calls.values()?.is_empty()); - assert_eq!( - selected_target(&ctx), - Some(&LlmTargetId::from_static("first-target")) - ); - assert_eq!( - selected_model(&ctx), - Some(&ModelId::from_static("first-model")) - ); - assert_eq!( - selection(&ctx)?.reason, - BackendSelectionReason::ContextTarget - ); - Ok(()) -} - -// Startup rollback should keep shutting down started backends even if rollback fails. -#[tokio::test] -async fn startup_failure_rolls_back_all_started_backends_even_when_shutdowns_fail() -> Result<()> { - let calls = Shared::default(); - let events = Shared::default(); - let backend = MultiLlmBackend::new([ - target_backend( - "first-target", - "first-model", - RecordingBackend::new("first", calls.clone(), events.clone()) - .with_shutdown_error("first rollback failed"), - ), - target_backend( - "second-target", - "second-model", - RecordingBackend::new("second", calls.clone(), events.clone()) - .with_shutdown_error("second rollback failed"), - ), - target_backend( - "third-target", - "third-model", - RecordingBackend::new("third", calls, events.clone()) - .with_startup_error("third startup failed"), - ), - ])?; - - let Err(error) = backend.startup().await else { - return Err(SwitchyardError::Other("startup should fail".to_string())); - }; - - assert!( - matches!(error, SwitchyardError::Backend(message) if message == "third startup failed") - ); - assert_eq!( - events.values()?, - vec![ - "first:startup".to_string(), - "second:startup".to_string(), - "third:startup".to_string(), - "second:shutdown".to_string(), - "first:shutdown".to_string(), - ] - ); - Ok(()) -} - -// Normal lifecycle should start forward and shut down in reverse order. -#[tokio::test] -async fn successful_startup_and_shutdown_use_forward_then_reverse_order() -> Result<()> { - let calls = Shared::default(); - let events = Shared::default(); - let backend = MultiLlmBackend::new([ - target_backend( - "first-target", - "first-model", - RecordingBackend::new("first", calls.clone(), events.clone()), - ), - target_backend( - "second-target", - "second-model", - RecordingBackend::new("second", calls, events.clone()), - ), - ])?; - - backend.startup().await?; - backend.shutdown().await?; - - assert_eq!( - events.values()?, - vec![ - "first:startup".to_string(), - "second:startup".to_string(), - "second:shutdown".to_string(), - "first:shutdown".to_string(), - ] - ); - Ok(()) -} - -// Startup failure should roll back only the backends that actually started. -#[tokio::test] -async fn startup_rolls_back_already_started_backends() -> Result<()> { - let calls = Shared::default(); - let events = Shared::default(); - let backend = MultiLlmBackend::new([ - target_backend( - "first-target", - "first-model", - RecordingBackend::new("first", calls.clone(), events.clone()), - ), - target_backend( - "second-target", - "second-model", - RecordingBackend::new("second", calls.clone(), events.clone()) - .with_startup_error("startup failed"), - ), - target_backend( - "third-target", - "third-model", - RecordingBackend::new("third", calls, events.clone()), - ), - ])?; - - let Err(error) = backend.startup().await else { - return Err(SwitchyardError::Other("startup should fail".to_string())); - }; - - assert!(matches!(error, SwitchyardError::Backend(_))); - assert_eq!( - events.values()?, - vec![ - "first:startup".to_string(), - "second:startup".to_string(), - "first:shutdown".to_string(), - ] - ); - Ok(()) -} - -// Shutdown should run every backend in reverse and return the first observed failure. -#[tokio::test] -async fn shutdown_runs_all_backends_in_reverse_and_returns_first_error() -> Result<()> { - let calls = Shared::default(); - let events = Shared::default(); - let backend = MultiLlmBackend::new([ - target_backend( - "first-target", - "first-model", - RecordingBackend::new("first", calls.clone(), events.clone()) - .with_shutdown_error("first shutdown failed"), - ), - target_backend( - "second-target", - "second-model", - RecordingBackend::new("second", calls, events.clone()) - .with_shutdown_error("second shutdown failed"), - ), - ])?; - - let Err(error) = backend.shutdown().await else { - return Err(SwitchyardError::Other("shutdown should fail".to_string())); - }; - - assert!( - matches!(error, SwitchyardError::Backend(message) if message == "second shutdown failed") - ); - assert_eq!( - events.values()?, - vec!["second:shutdown".to_string(), "first:shutdown".to_string()] - ); - Ok(()) -} - -// Public accessors should expose targets without coupling tests to storage internals. -#[test] -fn public_accessors_return_targets_and_backends_without_leaking_storage_shape() -> Result<()> { - let calls = Shared::default(); - let events = Shared::default(); - let backend = MultiLlmBackend::new([ - target_backend( - "first-target", - "first-model", - RecordingBackend::new("first", calls.clone(), events.clone()), - ), - target_backend( - "second-target", - "second-model", - RecordingBackend::new("second", calls, events), - ), - ])?; - - assert_eq!(backend.targets().len(), 2); - assert_eq!( - backend.targets()[0].target().id, - LlmTargetId::from_static("first-target") - ); - assert_eq!( - backend - .target(&LlmTargetId::from_static("second-target")) - .ok_or_else(|| SwitchyardError::Other("second target should exist".to_string()))? - .target() - .model, - ModelId::from_static("second-model") - ); - assert!( - backend - .target(&LlmTargetId::from_static("missing-target")) - .is_none() - ); - assert_eq!( - backend.targets()[0].backend().supported_request_types(), - &ALL_REQUEST_TYPES - ); - Ok(()) -} - -// Constructor validation should reject empty target lists and duplicate IDs. -#[test] -fn configuration_rejects_empty_targets_duplicate_ids_and_empty_supported_types() -> Result<()> { - let calls = Shared::default(); - let events = Shared::default(); - - let Err(empty_targets) = MultiLlmBackend::new([]) else { - return Err(SwitchyardError::Other( - "empty target list should fail".to_string(), - )); - }; - assert!(matches!(empty_targets, SwitchyardError::InvalidConfig(_))); - - let Err(duplicate_ids) = MultiLlmBackend::new([ - target_backend( - "same-target", - "left-model", - RecordingBackend::new("left", calls.clone(), events.clone()), - ), - target_backend( - "same-target", - "right-model", - RecordingBackend::new("right", calls.clone(), events.clone()), - ), - ]) else { - return Err(SwitchyardError::Other( - "duplicate target IDs should fail".to_string(), - )); - }; - assert!(matches!(duplicate_ids, SwitchyardError::InvalidConfig(_))); - - let backend = MultiLlmBackend::new([target_backend( - "only-target", - "only-model", - RecordingBackend::new("only", calls, events), - )])?; - let Err(invalid_default) = backend - .clone() - .with_default_target(LlmTargetId::from_static("missing-target")) - else { - return Err(SwitchyardError::Other( - "unknown default target should fail".to_string(), - )); - }; - assert!(matches!(invalid_default, SwitchyardError::InvalidConfig(_))); - let Err(empty_supported_types) = backend.with_supported_request_types([]) else { - return Err(SwitchyardError::Other( - "empty supported request types should fail".to_string(), - )); - }; - assert!(matches!( - empty_supported_types, - SwitchyardError::InvalidConfig(_) - )); - Ok(()) -} diff --git a/crates/switchyard-components/tests/adversarial_native_backends.rs b/crates/switchyard-components/tests/adversarial_native_backends.rs deleted file mode 100644 index 3cc581389..000000000 --- a/crates/switchyard-components/tests/adversarial_native_backends.rs +++ /dev/null @@ -1,1160 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Adversarial integration tests for native LLM backends. - -mod support; - -use futures_util::StreamExt; -use serde_json::{Value, json}; -use switchyard_components::{ - AnthropicNativeBackend, BackendSelection, OpenAiNativeBackend, OpenAiPassthroughBackend, -}; -use switchyard_components::{ - BackendFormat, ChatRequest, ChatRequestType, ChatResponse, ChatResponseType, EndpointConfig, - LlmBackend, LlmTarget, LlmTargetId, ModelId, ProxyContext, Result, StreamEvent, - SwitchyardError, -}; - -use support::{CapturedRequest, OneShotServer}; - -// Reads the selected model stamped by the backend into context. -fn selected_model(ctx: &ProxyContext) -> Option<&ModelId> { - ctx.get::() - .map(|selection| &selection.model) -} - -// Reads the selected target stamped by native backends into context. -fn selected_target(ctx: &ProxyContext) -> Option<&LlmTargetId> { - ctx.get::() - .and_then(|selection| selection.target_id.as_ref()) -} - -// Mirrors runtime telemetry header resolution for assertions. -fn expected_switchyard_version_header() -> Option { - if env_value_opts_out( - std::env::var("SWITCHYARD_TELEMETRY_OPT_OUT") - .ok() - .as_deref(), - ) || env_value_opts_out( - std::env::var("NEMO_SWITCHYARD_TELEMETRY_OPT_OUT") - .ok() - .as_deref(), - ) { - return None; - } - Some( - std::env::var("SWITCHYARD_VERSION") - .ok() - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()) - .unwrap_or_else(|| env!("CARGO_PKG_VERSION").to_string()), - ) -} - -// Parses telemetry opt-out environment values like the production helper. -fn env_value_opts_out(value: Option<&str>) -> bool { - let Some(value) = value.map(str::trim) else { - return false; - }; - !matches!( - value.to_ascii_lowercase().as_str(), - "" | "0" | "false" | "no" - ) -} - -// Builds a target with explicit endpoint credentials for native backend tests. -fn target( - format: BackendFormat, - base_url: String, - api_key: &str, - model: &str, -) -> Result { - let mut target = LlmTarget::new( - LlmTargetId::from_static("primary"), - ModelId::new(model).map_err(|error| SwitchyardError::Other(error.to_string()))?, - ); - target.format = format; - target.endpoint = EndpointConfig { - base_url: Some(base_url), - api_key: Some(api_key.to_string()), - timeout_secs: None, - }; - Ok(target) -} - -// Builds an OpenAI target fixture. -fn openai_target(base_url: String) -> Result { - target( - BackendFormat::OpenAi, - base_url, - "openai-secret", - "target-gpt", - ) -} - -// Builds an OpenAI Responses target fixture. -fn responses_target(base_url: String) -> Result { - target( - BackendFormat::Responses, - base_url, - "openai-secret", - "target-responses", - ) -} - -// Builds an Anthropic target fixture. -fn anthropic_target(base_url: String) -> Result { - target( - BackendFormat::Anthropic, - base_url, - "anthropic-secret", - "target-claude", - ) -} - -// Builds a passthrough OpenAI endpoint fixture. -fn openai_endpoint(base_url: String) -> EndpointConfig { - EndpointConfig { - base_url: Some(base_url), - api_key: Some("openai-secret".to_string()), - timeout_secs: None, - } -} - -// Calls the OpenAI backend against a one-shot mock server. -async fn openai_call(body: Value) -> Result<(ChatResponse, CapturedRequest, ProxyContext)> { - let server = OneShotServer::json( - 200, - json!({ - "id": "chatcmpl-test", - "object": "chat.completion", - "choices": [] - }), - )?; - let backend = OpenAiNativeBackend::new(openai_target(format!("{}/v1", server.base_url()))?)?; - let mut ctx = ProxyContext::new(); - let response = backend - .call(&mut ctx, &ChatRequest::openai_chat(body)) - .await?; - Ok((response, server.captured()?, ctx)) -} - -// Drains JSON stream events and rejects unexpected text frames. -async fn collect_json_events(response: ChatResponse) -> Result> { - let mut stream = match response { - ChatResponse::OpenAiStream(stream) - | ChatResponse::OpenAiResponsesStream(stream) - | ChatResponse::AnthropicStream(stream) => stream, - other => { - return Err(SwitchyardError::Other(format!( - "expected streaming response, got {:?}", - other.response_type() - ))); - } - }; - let mut events = Vec::new(); - while let Some(event) = stream.next().await { - match event? { - StreamEvent::Json(value) => events.push(value), - StreamEvent::Text(text) => { - return Err(SwitchyardError::Other(format!( - "unexpected text stream event: {text}" - ))); - } - } - } - Ok(events) -} - -// OpenAI native backends should only advertise OpenAI Chat input. -#[test] -fn openai_backend_is_openai_chat_only() -> Result<()> { - let backend = OpenAiNativeBackend::new(openai_target("http://127.0.0.1:1/v1".to_string())?)?; - - assert_eq!( - backend.supported_request_types(), - &[ChatRequestType::OpenAiChat] - ); - Ok(()) -} - -// Responses-format OpenAI targets should advertise OpenAI Responses input. -#[test] -fn openai_backend_can_be_responses_only() -> Result<()> { - let backend = OpenAiNativeBackend::new(responses_target("http://127.0.0.1:1/v1".to_string())?)?; - - assert_eq!( - backend.supported_request_types(), - &[ChatRequestType::OpenAiResponses] - ); - Ok(()) -} - -// OpenAI passthrough keeps the same OpenAI Chat-only role contract. -#[test] -fn openai_passthrough_backend_is_openai_chat_only() -> Result<()> { - let backend = - OpenAiPassthroughBackend::new(openai_endpoint("http://127.0.0.1:1/v1".to_string()))?; - - assert_eq!( - backend.supported_request_types(), - &[ChatRequestType::OpenAiChat] - ); - Ok(()) -} - -// Non-streaming OpenAI calls should preserve body fields and stamp context. -#[tokio::test] -async fn openai_non_streaming_posts_configured_body_and_records_context() -> Result<()> { - let (response, request, ctx) = openai_call(json!({ - "model": "client-gpt", - "messages": [{"role": "user", "content": "hello"}], - "temperature": 0.2, - "made_up_beta_field": {"kept": true}, - "stream": false - })) - .await?; - - assert_eq!(response.response_type(), ChatResponseType::OpenAiCompletion); - assert_eq!( - response - .body() - .ok_or_else(|| SwitchyardError::Other("buffered response".to_string()))?["id"], - "chatcmpl-test" - ); - assert_eq!(ctx.inbound_format, Some(ChatRequestType::OpenAiChat)); - assert_eq!( - selected_model(&ctx), - Some(&ModelId::from_static("target-gpt")) - ); - assert_eq!( - selected_target(&ctx), - Some(&LlmTargetId::from_static("primary")) - ); - - assert_eq!(request.method, "POST"); - assert_eq!(request.path, "/v1/chat/completions"); - assert_eq!( - request.header("authorization"), - Some("Bearer openai-secret") - ); - let expected_version = expected_switchyard_version_header(); - assert_eq!( - request.header("x-switchyard-version"), - expected_version.as_deref() - ); - assert_eq!(request.body["model"], "target-gpt"); - assert_eq!(request.body["messages"][0]["content"], "hello"); - assert_eq!(request.body["temperature"], 0.2); - assert_eq!(request.body["made_up_beta_field"], json!({"kept": true})); - assert!(request.body.get("stream_options").is_none()); - Ok(()) -} - -// Responses-format OpenAI targets should call /v1/responses without translating through Chat. -#[tokio::test] -async fn openai_responses_target_posts_responses_body_and_records_context() -> Result<()> { - let server = OneShotServer::json( - 200, - json!({ - "id": "resp-test", - "object": "response", - "output": [] - }), - )?; - let backend = OpenAiNativeBackend::new(responses_target(format!("{}/v1", server.base_url()))?)?; - let mut ctx = ProxyContext::new(); - - let response = backend - .call( - &mut ctx, - &ChatRequest::openai_responses(json!({ - "model": "client-gpt", - "input": "hello", - "stream": false - })), - ) - .await?; - let request = server.captured()?; - - assert_eq!( - response.response_type(), - ChatResponseType::OpenAiResponsesCompletion - ); - assert_eq!(ctx.inbound_format, Some(ChatRequestType::OpenAiResponses)); - assert_eq!( - selected_model(&ctx), - Some(&ModelId::from_static("target-responses")) - ); - assert_eq!(request.method, "POST"); - assert_eq!(request.path, "/v1/responses"); - assert_eq!(request.body["model"], "target-responses"); - assert_eq!(request.body["input"], "hello"); - assert!(request.body.get("messages").is_none()); - assert!(request.body.get("stream_options").is_none()); - Ok(()) -} - -// Endpoint-specific base URLs should normalize to the endpoint selected by target format. -#[tokio::test] -async fn openai_specific_base_url_uses_selected_endpoint() -> Result<()> { - let chat_server = OneShotServer::json(200, json!({"id": "chatcmpl-test", "choices": []}))?; - let chat_backend = OpenAiNativeBackend::new(openai_target(format!( - "{}/v1/responses", - chat_server.base_url() - ))?)?; - let mut chat_ctx = ProxyContext::new(); - - chat_backend - .call( - &mut chat_ctx, - &ChatRequest::openai_chat(json!({ - "model": "client-gpt", - "messages": [{"role": "user", "content": "hello"}] - })), - ) - .await?; - assert_eq!(chat_server.captured()?.path, "/v1/chat/completions"); - - let responses_server = OneShotServer::json(200, json!({"id": "resp-test", "output": []}))?; - let responses_backend = OpenAiNativeBackend::new(responses_target(format!( - "{}/v1/chat/completions", - responses_server.base_url() - ))?)?; - let mut responses_ctx = ProxyContext::new(); - - responses_backend - .call( - &mut responses_ctx, - &ChatRequest::openai_responses(json!({ - "model": "client-gpt", - "input": "hello" - })), - ) - .await?; - assert_eq!(responses_server.captured()?.path, "/v1/responses"); - Ok(()) -} - -// Chat-format OpenAI targets should keep translating Responses requests to Chat fallback. -#[tokio::test] -async fn openai_chat_target_translates_responses_to_chat_fallback() -> Result<()> { - let server = OneShotServer::json(200, json!({"id": "chatcmpl-test", "choices": []}))?; - let backend = OpenAiNativeBackend::new(openai_target(format!("{}/v1", server.base_url()))?)?; - let mut ctx = ProxyContext::new(); - - let response = backend - .call( - &mut ctx, - &ChatRequest::openai_responses(json!({ - "model": "client-gpt", - "input": "translate me" - })), - ) - .await?; - let request = server.captured()?; - - assert_eq!(response.response_type(), ChatResponseType::OpenAiCompletion); - assert_eq!(ctx.inbound_format, Some(ChatRequestType::OpenAiResponses)); - assert_eq!(request.path, "/v1/chat/completions"); - assert_eq!(request.body["model"], "target-gpt"); - assert_eq!(request.body["messages"][0]["content"], "translate me"); - Ok(()) -} - -// Native OpenAI targets should apply per-target body and header overrides. -#[tokio::test] -async fn openai_native_applies_target_extra_body_and_headers() -> Result<()> { - let server = OneShotServer::json( - 200, - json!({ - "id": "chatcmpl-test", - "object": "chat.completion", - "choices": [] - }), - )?; - let mut target = openai_target(format!("{}/v1", server.base_url()))?; - target.extra_body = Some(json!({ - "chat_template_kwargs": {"enable_thinking": false} - })); - target - .extra_headers - .insert("X-Inference-Priority".to_string(), "batch".to_string()); - let backend = OpenAiNativeBackend::new(target)?; - let mut ctx = ProxyContext::new(); - - backend - .call( - &mut ctx, - &ChatRequest::openai_chat(json!({ - "model": "client-gpt", - "messages": [{"role": "user", "content": "hello"}], - "stream": false - })), - ) - .await?; - let request = server.captured()?; - - assert_eq!( - request.body["chat_template_kwargs"], - json!({"enable_thinking": false}) - ); - assert_eq!(request.header("x-inference-priority"), Some("batch")); - Ok(()) -} - -// Passthrough OpenAI calls should not rewrite caller model names. -#[tokio::test] -async fn openai_passthrough_preserves_client_model_and_records_context() -> Result<()> { - let server = OneShotServer::json( - 200, - json!({ - "id": "chatcmpl-test", - "object": "chat.completion", - "choices": [] - }), - )?; - let backend = - OpenAiPassthroughBackend::new(openai_endpoint(format!("{}/v1", server.base_url())))?; - let mut ctx = ProxyContext::new(); - - let response = backend - .call( - &mut ctx, - &ChatRequest::openai_chat(json!({ - "model": "client-gpt", - "messages": [{"role": "user", "content": "hello"}], - "stream": false - })), - ) - .await?; - let request = server.captured()?; - - assert_eq!(response.response_type(), ChatResponseType::OpenAiCompletion); - assert_eq!(ctx.inbound_format, Some(ChatRequestType::OpenAiChat)); - assert_eq!( - selected_model(&ctx), - Some(&ModelId::from_static("client-gpt")) - ); - assert!(selected_target(&ctx).is_none()); - assert_eq!(request.path, "/v1/chat/completions"); - assert_eq!( - request.header("authorization"), - Some("Bearer openai-secret") - ); - assert_eq!(request.body["model"], "client-gpt"); - Ok(()) -} - -// OpenAI streams should request usage and stop cleanly on `[DONE]`. -#[tokio::test] -async fn openai_streaming_injects_usage_opt_in_and_parses_done() -> Result<()> { - let server = OneShotServer::sse( - "data: {\"choices\":[{\"delta\":{\"content\":\"hi\"}}]}\n\n\ - data: {\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":2,\"total_tokens\":3}}\n\n\ - data: [DONE]\n\n", - )?; - let backend = OpenAiNativeBackend::new(openai_target(format!("{}/v1", server.base_url()))?)?; - let mut ctx = ProxyContext::new(); - - let response = backend - .call( - &mut ctx, - &ChatRequest::openai_chat(json!({ - "model": "client-gpt", - "messages": [{"role": "user", "content": "stream"}], - "stream": true - })), - ) - .await?; - assert_eq!(response.response_type(), ChatResponseType::OpenAiStream); - let events = collect_json_events(response).await?; - let request = server.captured()?; - - assert_eq!(events.len(), 2); - assert_eq!(events[0]["choices"][0]["delta"]["content"], "hi"); - assert_eq!(events[1]["usage"]["total_tokens"], 3); - assert_eq!( - request.body["stream_options"], - json!({"include_usage": true}) - ); - Ok(()) -} - -// Existing stream options should win over backend usage defaults. -#[tokio::test] -async fn openai_streaming_respects_usage_opt_out_and_preserves_other_options() -> Result<()> { - let server = OneShotServer::sse("data: [DONE]\n\n")?; - let backend = OpenAiNativeBackend::new(openai_target(format!("{}/v1", server.base_url()))?)?; - let mut ctx = ProxyContext::new(); - - let response = backend - .call( - &mut ctx, - &ChatRequest::openai_chat(json!({ - "model": "client-gpt", - "messages": [{"role": "user", "content": "stream"}], - "stream": true, - "stream_options": { - "include_usage": false, - "continuous_usage_stats": true - } - })), - ) - .await?; - let events = collect_json_events(response).await?; - let request = server.captured()?; - - assert!(events.is_empty()); - assert_eq!( - request.body["stream_options"], - json!({ - "include_usage": false, - "continuous_usage_stats": true - }) - ); - Ok(()) -} - -// OpenAI native should translate Anthropic requests before upstream dispatch. -#[tokio::test] -async fn openai_translates_anthropic_requests_before_native_call() -> Result<()> { - let server = OneShotServer::json(200, json!({"id": "chatcmpl-test", "choices": []}))?; - let backend = OpenAiNativeBackend::new(openai_target(format!("{}/v1", server.base_url()))?)?; - let mut ctx = ProxyContext::new(); - - backend - .call( - &mut ctx, - &ChatRequest::anthropic(json!({ - "model": "client-claude", - "max_tokens": 128, - "messages": [{"role": "user", "content": "translate me"}] - })), - ) - .await?; - let request = server.captured()?; - - assert_eq!(ctx.inbound_format, Some(ChatRequestType::Anthropic)); - assert_eq!(request.body["model"], "target-gpt"); - assert_eq!(request.body["messages"][0]["role"], "user"); - assert_eq!(request.body["messages"][0]["content"], "translate me"); - Ok(()) -} - -// OpenAI passthrough should translate Anthropic shape without model rewriting. -#[tokio::test] -async fn openai_passthrough_translates_anthropic_without_rewriting_model() -> Result<()> { - let server = OneShotServer::json(200, json!({"id": "chatcmpl-test", "choices": []}))?; - let backend = - OpenAiPassthroughBackend::new(openai_endpoint(format!("{}/v1", server.base_url())))?; - let mut ctx = ProxyContext::new(); - - backend - .call( - &mut ctx, - &ChatRequest::anthropic(json!({ - "model": "client-claude", - "max_tokens": 128, - "messages": [{"role": "user", "content": "translate me"}] - })), - ) - .await?; - let request = server.captured()?; - - assert_eq!(ctx.inbound_format, Some(ChatRequestType::Anthropic)); - assert_eq!( - selected_model(&ctx), - Some(&ModelId::from_static("client-claude")) - ); - assert_eq!(request.body["model"], "client-claude"); - assert_eq!(request.body["messages"][0]["content"], "translate me"); - Ok(()) -} - -// Responses streams should call /responses and return Responses stream variants. -#[tokio::test] -async fn openai_responses_streaming_uses_responses_stream_variant() -> Result<()> { - let server = OneShotServer::sse( - "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp-stream\"}}\n\n", - )?; - let backend = OpenAiNativeBackend::new(responses_target(format!("{}/v1", server.base_url()))?)?; - let mut ctx = ProxyContext::new(); - - let response = backend - .call( - &mut ctx, - &ChatRequest::openai_responses(json!({ - "model": "client-gpt", - "input": "stream", - "stream": true - })), - ) - .await?; - assert_eq!( - response.response_type(), - ChatResponseType::OpenAiResponsesStream - ); - let events = collect_json_events(response).await?; - let request = server.captured()?; - - assert_eq!(events.len(), 1); - assert_eq!(events[0]["type"], "response.created"); - assert_eq!(request.path, "/v1/responses"); - assert!(request.body.get("stream_options").is_none()); - Ok(()) -} - -// OpenAI native must reject targets configured for Anthropic format. -#[test] -fn openai_backend_rejects_anthropic_targets() -> Result<()> { - let Err(error) = OpenAiNativeBackend::new(target( - BackendFormat::Anthropic, - "http://127.0.0.1:1".to_string(), - "secret", - "claude", - )?) else { - return Err(SwitchyardError::Other( - "OpenAI backend should reject Anthropic targets".to_string(), - )); - }; - - assert!(matches!(error, SwitchyardError::InvalidConfig(_))); - Ok(()) -} - -// OpenAI upstream error responses should preserve status and body in the error. -#[tokio::test] -async fn openai_error_status_includes_status_and_body() -> Result<()> { - let server = OneShotServer::json( - 429, - json!({"error": {"message": "rate limited", "type": "rate_limit"}}), - )?; - let backend = OpenAiNativeBackend::new(openai_target(format!("{}/v1", server.base_url()))?)?; - let mut ctx = ProxyContext::new(); - - let Err(error) = backend - .call( - &mut ctx, - &ChatRequest::openai_chat(json!({ - "model": "client-gpt", - "messages": [{"role": "user", "content": "hello"}] - })), - ) - .await - else { - return Err(SwitchyardError::Other( - "OpenAI backend should propagate HTTP errors".to_string(), - )); - }; - let captured = server.captured()?; - - assert!(matches!(error, SwitchyardError::UpstreamHttp { .. })); - assert!(error.to_string().contains("HTTP 429")); - assert!(error.to_string().contains("rate limited")); - assert_eq!(captured.path, "/v1/chat/completions"); - Ok(()) -} - -// Anthropic native backends should only advertise Anthropic input. -#[test] -fn anthropic_backend_is_anthropic_only() -> Result<()> { - let backend = AnthropicNativeBackend::new(anthropic_target("http://127.0.0.1:1".to_string())?)?; - - assert_eq!( - backend.supported_request_types(), - &[ChatRequestType::Anthropic] - ); - Ok(()) -} - -// Non-streaming Anthropic calls should strip incompatible fields and stamp context. -#[tokio::test] -async fn anthropic_non_streaming_strips_incompatible_fields_and_records_context() -> Result<()> { - let server = OneShotServer::json( - 200, - json!({ - "id": "msg-test", - "type": "message", - "content": [] - }), - )?; - let backend = AnthropicNativeBackend::new(anthropic_target(server.base_url().to_string())?)?; - let mut ctx = ProxyContext::new(); - - let response = backend - .call( - &mut ctx, - &ChatRequest::anthropic(json!({ - "model": "client-claude", - "max_tokens": 128, - "messages": [{"role": "user", "content": "hello"}], - "reasoning_effort": "high", - "context_management": {"strategy": "auto"}, - "made_up_beta_field": {"kept": true}, - "extra_body": {"caller": "value"}, - "stream": false - })), - ) - .await?; - let request = server.captured()?; - - assert_eq!( - response.response_type(), - ChatResponseType::AnthropicCompletion - ); - assert_eq!( - response - .body() - .ok_or_else(|| SwitchyardError::Other("buffered response".to_string()))?["id"], - "msg-test" - ); - assert_eq!(ctx.inbound_format, Some(ChatRequestType::Anthropic)); - assert_eq!( - selected_model(&ctx), - Some(&ModelId::from_static("target-claude")) - ); - assert_eq!( - selected_target(&ctx), - Some(&LlmTargetId::from_static("primary")) - ); - - assert_eq!(request.method, "POST"); - assert_eq!(request.path, "/v1/messages"); - assert_eq!(request.header("x-api-key"), Some("anthropic-secret")); - assert_eq!(request.header("anthropic-version"), Some("2023-06-01")); - let expected_version = expected_switchyard_version_header(); - assert_eq!( - request.header("x-switchyard-version"), - expected_version.as_deref() - ); - assert_eq!(request.body["model"], "target-claude"); - assert_eq!(request.body["messages"][0]["content"], "hello"); - assert!(request.body.get("reasoning_effort").is_none()); - assert!(request.body.get("context_management").is_none()); - assert_eq!(request.body["made_up_beta_field"], json!({"kept": true})); - assert_eq!(request.body["extra_body"], json!({"caller": "value"})); - Ok(()) -} - -// Anthropic-native calls should downgrade Opus-4.8-style system turns for legacy targets. -#[tokio::test] -async fn anthropic_lifts_message_level_system_roles_before_native_call() -> Result<()> { - let server = OneShotServer::json(200, json!({"id": "msg-test", "content": []}))?; - let backend = AnthropicNativeBackend::new(anthropic_target(server.base_url().to_string())?)?; - let mut ctx = ProxyContext::new(); - - backend - .call( - &mut ctx, - &ChatRequest::anthropic(json!({ - "model": "client-claude", - "max_tokens": 128, - "messages": [ - {"role": "system", "content": "System rules."}, - {"role": "user", "content": "hello"}, - { - "role": "developer", - "content": [ - {"type": "text", "text": "Developer rules."} - ] - }, - {"role": "assistant", "content": "ready"} - ] - })), - ) - .await?; - let request = server.captured()?; - - assert_eq!(request.body["system"], "System rules.\n\nDeveloper rules."); - let messages = request.body["messages"] - .as_array() - .ok_or_else(|| SwitchyardError::Other("messages should be an array".to_string()))?; - let roles = messages - .iter() - .map(|message| { - message - .get("role") - .and_then(Value::as_str) - .unwrap_or("") - }) - .collect::>(); - assert_eq!(roles, vec!["user", "assistant"]); - assert_eq!(messages[0]["content"], "hello"); - assert_eq!(messages[1]["content"], "ready"); - Ok(()) -} - -// Interleaved system turns should preserve encounter order after lifting. -#[tokio::test] -async fn anthropic_lifts_multiple_interleaved_system_messages_in_order() -> Result<()> { - let server = OneShotServer::json(200, json!({"id": "msg-test", "content": []}))?; - let backend = AnthropicNativeBackend::new(anthropic_target(server.base_url().to_string())?)?; - let mut ctx = ProxyContext::new(); - - backend - .call( - &mut ctx, - &ChatRequest::anthropic(json!({ - "model": "client-claude", - "max_tokens": 128, - "system": "Top-level rules.", - "messages": [ - {"role": "system", "content": "First lifted system."}, - {"role": "user", "content": "first user"}, - {"role": "system", "content": "Second lifted system."}, - {"role": "assistant", "content": "assistant reply"}, - {"role": "developer", "content": "Developer lifted system."}, - {"role": "user", "content": "second user"} - ] - })), - ) - .await?; - let request = server.captured()?; - - assert_eq!( - request.body["system"], - "Top-level rules.\n\nFirst lifted system.\n\nSecond lifted system.\n\nDeveloper lifted system." - ); - assert_eq!( - request.body["messages"], - json!([ - {"role": "user", "content": "first user"}, - {"role": "assistant", "content": "assistant reply"}, - {"role": "user", "content": "second user"} - ]) - ); - Ok(()) -} - -// Existing structured Anthropic system prompts should keep their shape when lifted text is added. -#[tokio::test] -async fn anthropic_lifts_message_level_system_into_existing_system_blocks() -> Result<()> { - let server = OneShotServer::json(200, json!({"id": "msg-test", "content": []}))?; - let backend = AnthropicNativeBackend::new(anthropic_target(server.base_url().to_string())?)?; - let mut ctx = ProxyContext::new(); - - backend - .call( - &mut ctx, - &ChatRequest::anthropic(json!({ - "model": "client-claude", - "max_tokens": 128, - "system": [{"type": "text", "text": "Existing system."}], - "messages": [ - { - "role": "system", - "content": [ - {"type": "text", "text": "Lifted system."}, - {"type": "image", "source": {"type": "url", "url": "https://example.test/a.png"}}, - {"type": "input_text", "text": "Lifted input text."} - ] - }, - {"role": "user", "content": "hello"} - ] - })), - ) - .await?; - let request = server.captured()?; - - assert_eq!( - request.body["system"], - json!([ - {"type": "text", "text": "Existing system."}, - {"type": "text", "text": "Lifted system.\n\nLifted input text."} - ]) - ); - assert_eq!( - request.body["messages"], - json!([{"role": "user", "content": "hello"}]) - ); - Ok(()) -} - -// Responses requests should translate into Anthropic Messages with default max_tokens. -#[tokio::test] -async fn anthropic_translates_responses_requests_with_default_max_tokens() -> Result<()> { - let server = OneShotServer::json(200, json!({"id": "msg-test", "content": []}))?; - let backend = AnthropicNativeBackend::new(anthropic_target(server.base_url().to_string())?)?; - let mut ctx = ProxyContext::new(); - - backend - .call( - &mut ctx, - &ChatRequest::openai_responses(json!({ - "model": "client-gpt", - "input": "translate me" - })), - ) - .await?; - let request = server.captured()?; - - assert_eq!(ctx.inbound_format, Some(ChatRequestType::OpenAiResponses)); - assert_eq!(request.body["model"], "target-claude"); - assert_eq!(request.body["max_tokens"], 64000); - assert_eq!(request.body["messages"][0]["role"], "user"); - assert_eq!(request.body["messages"][0]["content"], "translate me"); - Ok(()) -} - -// Invalid Anthropic tool-use IDs should be sanitized consistently with results. -#[tokio::test] -async fn anthropic_sanitizes_invalid_tool_use_ids_and_matching_results() -> Result<()> { - let server = OneShotServer::json(200, json!({"id": "msg-test", "content": []}))?; - let backend = AnthropicNativeBackend::new(anthropic_target(server.base_url().to_string())?)?; - let mut ctx = ProxyContext::new(); - - backend - .call( - &mut ctx, - &ChatRequest::anthropic(json!({ - "model": "client-claude", - "max_tokens": 128, - "messages": [ - {"role": "user", "content": "use the tool"}, - { - "role": "assistant", - "content": [{ - "type": "tool_use", - "id": "toolu_01*bad:id", - "name": "lookup", - "input": {} - }] - }, - { - "role": "user", - "content": [{ - "type": "tool_result", - "tool_use_id": "toolu_01*bad:id", - "content": "done" - }] - } - ] - })), - ) - .await?; - let request = server.captured()?; - - let tool_use_id = &request.body["messages"][1]["content"][0]["id"]; - assert_eq!(tool_use_id, "toolu_01_bad_id"); - assert_eq!( - &request.body["messages"][2]["content"][0]["tool_use_id"], - tool_use_id - ); - Ok(()) -} - -// Unsigned synthetic thinking blocks should be removed before Anthropic replay. -#[tokio::test] -async fn anthropic_strips_unsigned_thinking_blocks_before_native_call() -> Result<()> { - let server = OneShotServer::json(200, json!({"id": "msg-test", "content": []}))?; - let backend = AnthropicNativeBackend::new(anthropic_target(server.base_url().to_string())?)?; - let mut ctx = ProxyContext::new(); - - backend - .call( - &mut ctx, - &ChatRequest::anthropic(json!({ - "model": "client-claude", - "max_tokens": 128, - "messages": [ - { - "role": "assistant", - "content": [ - {"type": "thinking", "thinking": "synthetic", "signature": ""}, - {"type": "tool_use", "id": "toolu_ok", "name": "lookup", "input": {}} - ] - }, - { - "role": "assistant", - "content": [ - {"type": "thinking", "thinking": "real", "signature": "signed"}, - {"type": "text", "text": "visible"} - ] - }, - { - "role": "assistant", - "content": [ - {"type": "thinking", "thinking": "only synthetic"} - ] - } - ] - })), - ) - .await?; - let request = server.captured()?; - - assert_eq!( - request.body["messages"][0]["content"] - .as_array() - .ok_or_else(|| SwitchyardError::Other("content should be an array".to_string()))? - .len(), - 1 - ); - assert_eq!( - request.body["messages"][0]["content"][0]["type"], - "tool_use" - ); - assert_eq!( - request.body["messages"][1]["content"][0]["type"], - "thinking" - ); - assert_eq!( - request.body["messages"][1]["content"][0]["thinking"], - "real" - ); - assert_eq!(request.body["messages"][2]["content"], ""); - Ok(()) -} - -// Anthropic SSE should produce JSON stream events. -#[tokio::test] -async fn anthropic_streaming_returns_stream_events() -> Result<()> { - let server = OneShotServer::sse( - "event: message_start\n\ - data: {\"type\":\"message_start\",\"message\":{\"id\":\"msg-stream\"}}\n\n", - )?; - let backend = AnthropicNativeBackend::new(anthropic_target(server.base_url().to_string())?)?; - let mut ctx = ProxyContext::new(); - - let response = backend - .call( - &mut ctx, - &ChatRequest::anthropic(json!({ - "model": "client-claude", - "max_tokens": 128, - "messages": [{"role": "user", "content": "stream"}], - "stream": true - })), - ) - .await?; - assert_eq!(response.response_type(), ChatResponseType::AnthropicStream); - let events = collect_json_events(response).await?; - let request = server.captured()?; - - assert_eq!(events.len(), 1); - assert_eq!(events[0]["type"], "message_start"); - assert_eq!(events[0]["message"]["id"], "msg-stream"); - assert_eq!(request.path, "/v1/messages"); - Ok(()) -} - -#[tokio::test] -async fn anthropic_streaming_stops_at_done_marker() -> Result<()> { - let server = OneShotServer::sse( - "event: message_start\n\ - data: {\"type\":\"message_start\",\"message\":{\"id\":\"msg-stream\"}}\n\n\ - data: [DONE]\n\n\ - event: message_stop\n\ - data: {\"type\":\"message_stop\"}\n\n", - )?; - let backend = AnthropicNativeBackend::new(anthropic_target(server.base_url().to_string())?)?; - let mut ctx = ProxyContext::new(); - - let response = backend - .call( - &mut ctx, - &ChatRequest::anthropic(json!({ - "model": "client-claude", - "max_tokens": 128, - "messages": [{"role": "user", "content": "stream"}], - "stream": true - })), - ) - .await?; - let events = collect_json_events(response).await?; - server.captured()?; - - assert_eq!( - events, - vec![json!({ - "type": "message_start", - "message": {"id": "msg-stream"} - })] - ); - Ok(()) -} - -// Anthropic native must reject targets configured for OpenAI format. -#[test] -fn anthropic_backend_rejects_openai_targets() -> Result<()> { - let Err(error) = AnthropicNativeBackend::new(target( - BackendFormat::OpenAi, - "http://127.0.0.1:1".to_string(), - "secret", - "gpt", - )?) else { - return Err(SwitchyardError::Other( - "Anthropic backend should reject OpenAI targets".to_string(), - )); - }; - - assert!(matches!(error, SwitchyardError::InvalidConfig(_))); - Ok(()) -} - -// Anthropic upstream error responses should preserve status and body in the error. -#[tokio::test] -async fn anthropic_error_status_includes_status_and_body() -> Result<()> { - let server = OneShotServer::json( - 400, - json!({"error": {"message": "invalid request", "type": "bad_request"}}), - )?; - let backend = AnthropicNativeBackend::new(anthropic_target(server.base_url().to_string())?)?; - let mut ctx = ProxyContext::new(); - - let Err(error) = backend - .call( - &mut ctx, - &ChatRequest::anthropic(json!({ - "model": "client-claude", - "max_tokens": 128, - "messages": [{"role": "user", "content": "hello"}] - })), - ) - .await - else { - return Err(SwitchyardError::Other( - "Anthropic backend should propagate HTTP errors".to_string(), - )); - }; - let captured = server.captured()?; - - assert!(matches!(error, SwitchyardError::UpstreamHttp { .. })); - assert!(error.to_string().contains("HTTP 400")); - assert!(error.to_string().contains("invalid request")); - assert_eq!(captured.path, "/v1/messages"); - Ok(()) -} - -// Native backends should reject unresolved Auto formats until config resolves them. -#[test] -fn native_backends_reject_unresolved_auto_targets() -> Result<()> { - let auto_openai = target( - BackendFormat::Auto, - "http://127.0.0.1:1".to_string(), - "secret", - "gpt", - )?; - let Err(openai_error) = OpenAiNativeBackend::new(auto_openai) else { - return Err(SwitchyardError::Other( - "OpenAI backend should reject Auto targets".to_string(), - )); - }; - assert!(matches!(openai_error, SwitchyardError::InvalidConfig(_))); - - let auto_anthropic = target( - BackendFormat::Auto, - "http://127.0.0.1:1".to_string(), - "secret", - "claude", - )?; - let Err(anthropic_error) = AnthropicNativeBackend::new(auto_anthropic) else { - return Err(SwitchyardError::Other( - "Anthropic backend should reject Auto targets".to_string(), - )); - }; - assert!(matches!(anthropic_error, SwitchyardError::InvalidConfig(_))); - Ok(()) -} diff --git a/crates/switchyard-components/tests/adversarial_random_routing.rs b/crates/switchyard-components/tests/adversarial_random_routing.rs deleted file mode 100644 index 3cd0fcaed..000000000 --- a/crates/switchyard-components/tests/adversarial_random_routing.rs +++ /dev/null @@ -1,209 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Adversarial tests for the random routing engine. - -use serde_json::json; -use switchyard_components::{ - BackendFormat, ChatRequest, EndpointConfig, LlmTarget, LlmTargetId, ModelId, Result, - SwitchyardError, -}; -use switchyard_components::{RandomRoutingEngine, RandomRoutingProcessorConfig, RandomRoutingTier}; - -// Builds a deterministic strong/weak config for routing tests. -fn config(strong_probability: f64, rng_seed: u64) -> Result { - Ok(RandomRoutingProcessorConfig::new( - LlmTarget::new( - LlmTargetId::from_static("strong-target"), - ModelId::from_static("strong-model"), - ), - LlmTarget::new( - LlmTargetId::from_static("weak-target"), - ModelId::from_static("weak-model"), - ), - ) - .with_strong_probability(strong_probability)? - .with_rng_seed(Some(rng_seed))) -} - -// Builds an OpenAI Chat request whose non-model fields must be preserved. -fn openai_request(model: &str) -> ChatRequest { - ChatRequest::openai_chat(json!({ - "model": model, - "messages": [{"role": "user", "content": "keep me"}], - "temperature": 0.3, - })) -} - -// Runs one request through the engine and applies the route-local model rewrite. -fn route_once(engine: &RandomRoutingEngine, mut request: ChatRequest) -> Result { - let decision = engine.select(request.model().map(str::to_owned))?; - request.set_model(decision.selected_model.as_str()); - Ok(request) -} - -// Probability zero is a hard weak route and should preserve request payload fields. -#[test] -fn probability_zero_always_routes_to_weak_and_preserves_body_fields() -> Result<()> { - let engine = RandomRoutingEngine::new(config(0.0, 7)?)?; - let request = route_once(&engine, openai_request("client-model"))?; - let decision = engine.select(Some("client-model".to_string()))?; - - assert_eq!(request.model(), Some("weak-model")); - assert_eq!(request.body()["messages"][0]["content"], "keep me"); - assert_eq!(request.body()["temperature"], 0.3); - assert_eq!(decision.tier, RandomRoutingTier::Weak); - assert_eq!(decision.selected_model, ModelId::from_static("weak-model")); - assert_eq!(decision.original_model.as_deref(), Some("client-model")); - assert_eq!(decision.strong_probability, 0.0); - assert!((0.0..1.0).contains(&decision.draw)); - Ok(()) -} - -// Probability one is a hard strong route. -#[test] -fn probability_one_always_routes_to_strong() -> Result<()> { - let engine = RandomRoutingEngine::new(config(1.0, 7)?)?; - let request = route_once(&engine, openai_request("client-model"))?; - let decision = engine.select(Some("client-model".to_string()))?; - - assert_eq!(request.model(), Some("strong-model")); - assert_eq!( - decision.selected_target, - LlmTargetId::from_static("strong-target") - ); - assert_eq!(decision.tier, RandomRoutingTier::Strong); - Ok(()) -} - -// Equal seeds should produce identical routing sequences across processors. -#[test] -fn seeded_engines_produce_the_same_routing_sequence() -> Result<()> { - let left = RandomRoutingEngine::new(config(0.5, 42)?)?; - let right = RandomRoutingEngine::new(config(0.5, 42)?)?; - - let mut left_sequence = Vec::new(); - let mut right_sequence = Vec::new(); - for _ in 0..32 { - left_sequence.push(left.select(Some("client-model".to_string()))?.tier); - right_sequence.push(right.select(Some("client-model".to_string()))?.tier); - } - - assert_eq!(left_sequence, right_sequence); - assert!(left_sequence.contains(&RandomRoutingTier::Strong)); - assert!(left_sequence.contains(&RandomRoutingTier::Weak)); - Ok(()) -} - -// Malformed request bodies should be repaired into an object with the selected model. -#[test] -fn malformed_non_object_request_body_is_recovered_with_selected_model() -> Result<()> { - let engine = RandomRoutingEngine::new(config(1.0, 11)?)?; - let request = route_once(&engine, ChatRequest::openai_chat(json!("bad")))?; - - assert_eq!(request.body(), &json!({"model": "strong-model"})); - Ok(()) -} - -// Invalid probabilities must fail during engine construction, not at call time. -#[test] -fn invalid_probability_is_rejected_before_routing_requests() -> Result<()> { - for value in [-0.1, 1.1, f64::NAN, f64::INFINITY] { - let Err(error) = RandomRoutingEngine::new(RandomRoutingProcessorConfig { - strong: LlmTarget::new( - LlmTargetId::from_static("strong"), - ModelId::from_static("strong-model"), - ), - weak: LlmTarget::new( - LlmTargetId::from_static("weak"), - ModelId::from_static("weak-model"), - ), - strong_probability: value, - rng_seed: Some(1), - }) else { - return Err(SwitchyardError::Other( - "invalid probability should be rejected".to_string(), - )); - }; - - assert!(matches!(error, SwitchyardError::InvalidConfig(_))); - } - Ok(()) -} - -// LLM target serde remains compatible with Python-authored configs. -#[test] -fn llm_target_format_wire_values_match_python_config_contract() -> Result<()> { - assert_eq!( - serde_json::to_value(BackendFormat::Auto) - .map_err(|error| SwitchyardError::Other(error.to_string()))?, - json!("auto") - ); - assert_eq!( - serde_json::to_value(BackendFormat::OpenAi) - .map_err(|error| SwitchyardError::Other(error.to_string()))?, - json!("openai") - ); - assert_eq!( - serde_json::to_value(BackendFormat::Responses) - .map_err(|error| SwitchyardError::Other(error.to_string()))?, - json!("responses") - ); - assert_eq!( - serde_json::to_value(BackendFormat::Anthropic) - .map_err(|error| SwitchyardError::Other(error.to_string()))?, - json!("anthropic") - ); - assert!(serde_json::from_value::(json!("unknown")).is_err()); - - let minimal = LlmTarget::new( - LlmTargetId::from_static("minimal"), - ModelId::from_static("model"), - ); - assert_eq!(minimal.format, BackendFormat::Auto); - assert_eq!(minimal.endpoint, EndpointConfig::default()); - - let explicit: LlmTarget = serde_json::from_value(json!({ - "id": "explicit", - "model": "model", - "format": "openai", - "endpoint": { - "base_url": "https://example.test/v1", - "api_key": "secret", - "timeout_secs": 2.5 - } - })) - .map_err(|error| SwitchyardError::Other(error.to_string()))?; - assert_eq!(explicit.format, BackendFormat::OpenAi); - assert_eq!( - explicit.endpoint.base_url.as_deref(), - Some("https://example.test/v1") - ); - assert_eq!(explicit.endpoint.api_key.as_deref(), Some("secret")); - assert_eq!(explicit.endpoint.timeout_secs, Some(2.5)); - Ok(()) -} - -// Random routing config defaults remain stable and accept inclusive boundaries. -#[test] -fn random_routing_config_defaults_and_accepts_boundary_probabilities() -> Result<()> { - let base = RandomRoutingProcessorConfig::new( - LlmTarget::new( - LlmTargetId::from_static("strong"), - ModelId::from_static("strong-model"), - ), - LlmTarget::new( - LlmTargetId::from_static("weak"), - ModelId::from_static("weak-model"), - ), - ); - - assert_eq!(base.strong_probability, 0.5); - assert_eq!(base.rng_seed, None); - - for probability in [0.0, 0.25, 0.5, 0.75, 1.0] { - let configured = base.clone().with_strong_probability(probability)?; - assert_eq!(configured.strong_probability, probability); - } - Ok(()) -} diff --git a/crates/switchyard-components/tests/contracts.rs b/crates/switchyard-components/tests/contracts.rs deleted file mode 100644 index b9955c874..000000000 --- a/crates/switchyard-components/tests/contracts.rs +++ /dev/null @@ -1,186 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Adversarial tests for compatibility identifiers, context storage, and wire wrappers. - -use std::pin::Pin; -use std::task::{Context, Poll}; - -use futures_core::Stream; -use serde_json::json; -use switchyard_components::{ - BackendFormat, ChatRequest, ChatRequestType, ChatResponse, ChatResponseType, LlmTarget, - LlmTargetId, ModelId, ProxyContext, StreamEvent, -}; - -type TestResult = std::result::Result<(), Box>; - -#[derive(Debug, Eq, PartialEq)] -struct ContextMarker(&'static str); - -// Empty stream fixture used to prove streams do not expose buffered JSON bodies. -struct EmptyStream; - -impl Stream for EmptyStream { - type Item = switchyard_components::Result; - - fn poll_next(self: Pin<&mut Self>, _ctx: &mut Context<'_>) -> Poll> { - Poll::Ready(None) - } -} - -// Verifies every ID constructor and serde path rejects empty identifiers. -#[test] -fn ids_reject_empty_and_whitespace_values_from_constructors_and_serde() -> TestResult { - assert!(ModelId::new("").is_err()); - assert!(ModelId::new(" ").is_err()); - assert!(serde_json::from_value::(json!("")).is_err()); - assert!(serde_json::from_value::(json!(" ")).is_err()); - - let parsed = serde_json::from_value::(json!("real-model"))?; - assert_eq!(parsed.as_str(), "real-model"); - Ok(()) -} - -// Verifies typed context extensions are keyed by type, not string names. -#[test] -fn typed_context_extensions_do_not_collide_or_require_string_keys() -> TestResult { - let mut ctx = ProxyContext::new(); - - assert!(ctx.insert(ContextMarker("first")).is_none()); - assert_eq!(ctx.insert(String::from("unrelated")), None); - assert_eq!( - ctx.insert(ContextMarker("second")), - Some(ContextMarker("first")) - ); - - match ctx.get::() { - Some(marker) => assert_eq!(marker, &ContextMarker("second")), - None => panic!("marker should be present"), - } - assert_eq!(ctx.get::().map(String::as_str), Some("unrelated")); - assert_eq!(ctx.remove::(), Some(ContextMarker("second"))); - assert!(ctx.get::().is_none()); - assert_eq!(ctx.get::().map(String::as_str), Some("unrelated")); - Ok(()) -} - -// Verifies model rewriting can recover malformed request bodies. -#[test] -fn set_model_recovers_from_malformed_non_object_request_bodies() { - let mut request = ChatRequest::openai_chat(json!("not-an-object")); - - assert_eq!(request.model(), None); - request.set_model("recovered-model"); - - assert_eq!(request.request_type(), ChatRequestType::OpenAiChat); - assert_eq!(request.model(), Some("recovered-model")); - assert_eq!(request.body(), &json!({"model": "recovered-model"})); -} - -// Verifies serialized wire values stay compatible with the Python side. -#[test] -fn request_and_response_wire_enum_serialization_stays_stable() -> TestResult { - assert_eq!( - serde_json::to_value(ChatRequestType::OpenAiResponses)?, - json!("openai_responses") - ); - assert_eq!( - serde_json::to_value(ChatResponseType::AnthropicStream)?, - json!("anthropic_stream") - ); - - let request = ChatRequest::anthropic(json!({ - "model": "claude", - "messages": [], - })); - assert_eq!( - serde_json::to_value(request)?, - json!({ - "request_type": "anthropic", - "request": { - "body": { - "model": "claude", - "messages": [], - }, - }, - }) - ); - Ok(()) -} - -// Verifies OpenAI backend format remains `openai`, not Rust enum snake_case. -#[test] -fn backend_format_stays_wire_compatible_with_python_configs() -> TestResult { - assert_eq!( - serde_json::to_value(BackendFormat::OpenAi)?, - json!("openai") - ); - assert_eq!( - serde_json::to_value(BackendFormat::Responses)?, - json!("responses") - ); - assert_eq!( - serde_json::from_value::(json!("openai"))?, - BackendFormat::OpenAi - ); - assert_eq!( - serde_json::from_value::(json!("responses"))?, - BackendFormat::Responses - ); - assert!(serde_json::from_value::(json!("open_ai")).is_err()); - Ok(()) -} - -// Verifies Rust LLM targets reject stale provider tuning fields instead of dropping typos. -#[test] -fn llm_target_rejects_provider_tuning_fields() -> TestResult { - assert!( - serde_json::from_value::(json!({ - "id": "primary", - "model": "gpt-5", - "format": "openai", - "endpoint": { - "base_url": "https://example.test/v1", - "api_key": null, - "timeout_secs": 30 - }, - "tuning": { - "max_output_tokens": 4096, - "reasoning_effort": "xhigh" - } - })) - .is_err() - ); - - let target = serde_json::from_value::(json!({ - "id": "primary", - "model": "gpt-5", - "format": "openai", - "endpoint": { - "base_url": "https://example.test/v1", - "api_key": null, - "timeout_secs": 30 - } - }))?; - assert_eq!(target.id, LlmTargetId::from_static("primary")); - assert_eq!(target.model, ModelId::from_static("gpt-5")); - assert_eq!(target.format, BackendFormat::OpenAi); - let serialized = serde_json::to_value(target)?; - assert_eq!( - serialized["endpoint"]["base_url"], - "https://example.test/v1" - ); - assert!(serialized.get("tuning").is_none()); - Ok(()) -} - -// Verifies streaming responses are distinguishable from buffered JSON responses. -#[test] -fn streaming_responses_are_not_mistaken_for_buffered_json_bodies() { - let response = ChatResponse::OpenAiStream(Box::pin(EmptyStream)); - - assert_eq!(response.response_type(), ChatResponseType::OpenAiStream); - assert!(response.body().is_none()); - assert!(format!("{response:?}").contains("")); -} diff --git a/crates/switchyard-components/tests/stats_accumulator.rs b/crates/switchyard-components/tests/stats_accumulator.rs deleted file mode 100644 index e018c4c52..000000000 --- a/crates/switchyard-components/tests/stats_accumulator.rs +++ /dev/null @@ -1,789 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -use std::thread; - -use serde_json::json; -use switchyard_components::{ - ModelStatsSnapshot, StatsAccumulator, StatsSnapshot, TokenUsage, prefix_probe, -}; -use switchyard_components::{Result, SwitchyardError}; - -#[test] -fn accumulator_snapshot_starts_zero_and_reset_clears_all_state() -> Result<()> { - let accumulator = StatsAccumulator::new(); - let initial = accumulator.snapshot()?; - assert_eq!(initial.total_requests, 0); - assert_eq!(initial.total_tokens.total, 0); - assert!(initial.models.is_empty()); - - accumulator.record_success("model-a", Some(12.5), Some("strong"))?; - accumulator.record_usage( - "model-a", - TokenUsage { - prompt_tokens: 5, - completion_tokens: 7, - cached_tokens: 2, - cache_creation_tokens: 1, - reasoning_tokens: 3, - cacheable_prompt_tokens: 0, - }, - Some(20.0), - Some(7.5), - Some("strong"), - )?; - assert_eq!(accumulator.snapshot()?.total_requests, 1); - - accumulator.reset()?; - let reset = accumulator.snapshot()?; - assert_eq!(reset.total_requests, 0); - assert_eq!(reset.total_tokens.total, 0); - assert!(reset.models.is_empty()); - Ok(()) -} - -#[test] -fn accumulator_matches_python_two_tier_snapshot_contract() -> Result<()> { - let accumulator = StatsAccumulator::new(); - accumulator.record_success("strong/model", Some(12.0), Some("strong"))?; - accumulator.record_usage( - "strong/model", - TokenUsage { - prompt_tokens: 100, - completion_tokens: 25, - cached_tokens: 10, - cache_creation_tokens: 5, - reasoning_tokens: 3, - cacheable_prompt_tokens: 90, - }, - Some(20.0), - Some(8.0), - Some("strong"), - )?; - accumulator.record_success("weak/model", None, Some("weak"))?; - accumulator.record_usage( - "weak/model", - TokenUsage { - prompt_tokens: 40, - completion_tokens: 5, - ..TokenUsage::default() - }, - None, - None, - Some("weak"), - )?; - - let snapshot = accumulator.snapshot()?; - - assert_eq!(snapshot.total_requests, 2); - assert_eq!(snapshot.total_tokens.prompt, 140); - assert_eq!(snapshot.total_tokens.completion, 30); - assert_eq!(snapshot.total_tokens.cached, 10); - assert_eq!(snapshot.total_tokens.cache_creation, 5); - assert_eq!(snapshot.total_tokens.reasoning, 3); - assert_eq!(snapshot.total_tokens.total, 170); - - let strong = model_stats(&snapshot, "strong/model")?; - assert_eq!(strong.tier.as_deref(), Some("strong")); - assert_eq!(strong.request_pct, 50.0); - assert_eq!(strong.token_pct, 73.53); - assert_eq!(strong.max_observed_context_tokens, 125); - assert_eq!(strong.avg_prompt_tokens, 100.0); - assert_eq!(strong.avg_completion_tokens, 25.0); - assert_eq!(strong.cache_hit_rate, 0.1); - assert_eq!(strong.theoretical_cache_hit_rate, 0.9); - - let strong_tier = snapshot - .tiers - .get("strong") - .ok_or_else(|| SwitchyardError::Other("strong tier should exist".to_string()))?; - assert_eq!(strong_tier.model, "strong/model"); - assert_eq!(strong_tier.prompt_tokens, 100); - assert_eq!(strong_tier.completion_tokens, 25); - - let weak_tier = snapshot - .tiers - .get("weak") - .ok_or_else(|| SwitchyardError::Other("weak tier should exist".to_string()))?; - assert_eq!(weak_tier.model, "weak/model"); - assert_eq!(weak_tier.prompt_tokens, 40); - assert_eq!(weak_tier.completion_tokens, 5); - Ok(()) -} - -#[test] -fn accumulator_tracks_max_observed_context_tokens_per_model() -> Result<()> { - let accumulator = StatsAccumulator::new(); - for (prompt_tokens, completion_tokens) in [(100, 10), (90, 50), (120, 5)] { - accumulator.record_success("model-a", None, None)?; - accumulator.record_usage( - "model-a", - TokenUsage { - prompt_tokens, - completion_tokens, - ..TokenUsage::default() - }, - None, - None, - None, - )?; - } - - let snapshot = accumulator.snapshot()?; - let model = model_stats(&snapshot, "model-a")?; - assert_eq!(model.prompt_tokens, 310); - assert_eq!(model.completion_tokens, 65); - assert_eq!(model.max_observed_context_tokens, 140); - Ok(()) -} - -// Switch-aware theoretical: cold on switch-in, partial credit on switching back. -#[test] -fn theoretical_is_switch_aware_across_a_model_switch_and_back() -> Result<()> { - let acc = StatsAccumulator::new(); - // One nested conversation; equal-length turns so fractions are exact. - let turn = |content: &[&str]| { - let messages: Vec<_> = content - .iter() - .map(|c| json!({"role": "user", "content": c})) - .collect(); - prefix_probe(&json!({ "messages": messages })) - }; - let p1 = turn(&["aaaa"]); - let p2 = turn(&["aaaa", "bbbb"]); - let p3 = turn(&["aaaa", "bbbb", "cccc"]); - let p4 = turn(&["aaaa", "bbbb", "cccc", "dddd"]); - - // strong serves turns 1-2: cold first sight, then the turn-1 prefix is eligible. - assert_eq!(acc.prefix_eligibility("strong", &p1), 0.0); - assert_eq!(acc.prefix_eligibility("strong", &p2), 0.5); - - // switch to weak at turn 3: weak's cache is empty. - assert_eq!(acc.prefix_eligibility("weak", &p3), 0.0); - - // back to strong at turn 4: it saw turns 1-2 but not turn 3 (weak's) -> 2/4. - assert_eq!(acc.prefix_eligibility("strong", &p4), 0.5); - Ok(()) -} - -// Cache stats are attributed per model across an N-model rotation. -#[test] -fn cache_stats_are_per_model_across_multi_model_rotation() -> Result<()> { - let accumulator = StatsAccumulator::new(); - - // Three models each entered cold (full re-warm). - for (model, tier) in [ - ("model/a", "strong"), - ("model/b", "weak"), - ("model/c", "third"), - ] { - accumulator.record_usage( - model, - TokenUsage { - prompt_tokens: 100, - cache_creation_tokens: 100, - ..TokenUsage::default() - }, - None, - None, - Some(tier), - )?; - } - // a and b take a second, warm turn served from cache; c stays cold. - for model in ["model/a", "model/b"] { - accumulator.record_usage( - model, - TokenUsage { - prompt_tokens: 100, - cached_tokens: 100, - ..TokenUsage::default() - }, - None, - None, - None, - )?; - } - - let snapshot = accumulator.snapshot()?; - let a = model_stats(&snapshot, "model/a")?; - assert_eq!(a.cache_hit_rate, 0.5); - assert_eq!(a.cache_creation_tokens, 100); - let c = model_stats(&snapshot, "model/c")?; - assert_eq!(c.cache_hit_rate, 0.0); - assert_eq!(c.cache_creation_tokens, 100); - Ok(()) -} - -#[test] -fn accumulator_is_thread_safe_under_concurrent_recording() -> Result<()> { - let accumulator = StatsAccumulator::new(); - let n_threads = 16_u64; - let calls_per_thread = 250_u64; - let mut handles = Vec::new(); - - for _ in 0..n_threads { - let accumulator = accumulator.clone(); - handles.push(thread::spawn(move || -> Result<()> { - for _ in 0..calls_per_thread { - accumulator.record_success("threaded/model", None, Some("strong"))?; - accumulator.record_usage( - "threaded/model", - TokenUsage { - prompt_tokens: 1, - completion_tokens: 1, - ..TokenUsage::default() - }, - None, - None, - Some("strong"), - )?; - } - Ok(()) - })); - } - - for handle in handles { - match handle.join() { - Ok(result) => result?, - Err(_) => { - return Err(SwitchyardError::Other( - "stats worker thread panicked".to_string(), - )); - } - } - } - - let expected = n_threads * calls_per_thread; - let snapshot = accumulator.snapshot()?; - let model = model_stats(&snapshot, "threaded/model")?; - assert_eq!(snapshot.total_requests, expected); - assert_eq!(model.calls, expected); - assert_eq!(model.prompt_tokens, expected); - assert_eq!(model.completion_tokens, expected); - Ok(()) -} - -#[test] -fn latency_reservoir_uses_replacement_path_after_saturation() -> Result<()> { - let accumulator = StatsAccumulator::new(); - for latency_ms in 0..10_050 { - accumulator.record_success("reservoir/model", Some(latency_ms as f64), Some("strong"))?; - } - - let snapshot = accumulator.snapshot()?; - let model = model_stats(&snapshot, "reservoir/model")?; - - assert_eq!(model.model_call_latency.count, 10_050); - assert_eq!(model.model_call_latency.min_ms, 0.0); - assert_eq!(model.model_call_latency.max_ms, 10_049.0); - assert_eq!(model.model_call_latency.p50_ms, 5_050.0); - assert_eq!(model.model_call_latency.p99_ms, 9_950.0); - Ok(()) -} - -#[test] -fn cost_estimate_matches_python_known_model_and_unknown_model_behavior() -> Result<()> { - let accumulator = StatsAccumulator::new(); - accumulator.record_usage( - "claude-sonnet-4-6", - TokenUsage { - prompt_tokens: 1_000_000, - ..TokenUsage::default() - }, - None, - None, - None, - )?; - accumulator.record_usage( - "claude-sonnet-4-6-20251022", - TokenUsage { - prompt_tokens: 1_000_000, - ..TokenUsage::default() - }, - None, - None, - None, - )?; - accumulator.record_usage( - "unknown/model", - TokenUsage { - prompt_tokens: 100, - completion_tokens: 50, - ..TokenUsage::default() - }, - None, - None, - None, - )?; - - let snapshot = accumulator.snapshot()?; - let sonnet = snapshot - .cost_estimate - .models - .get("claude-sonnet-4-6") - .ok_or_else(|| SwitchyardError::Other("sonnet cost should exist".to_string()))?; - let dated_sonnet = snapshot - .cost_estimate - .models - .get("claude-sonnet-4-6-20251022") - .ok_or_else(|| SwitchyardError::Other("dated sonnet cost should exist".to_string()))?; - let unknown = snapshot - .cost_estimate - .models - .get("unknown/model") - .ok_or_else(|| SwitchyardError::Other("unknown cost should exist".to_string()))?; - - assert_eq!(sonnet.total_cost, 3.0); - assert_eq!(dated_sonnet.total_cost, 0.0); - assert_eq!(unknown.total_cost, 0.0); - assert_eq!(snapshot.cost_estimate.total_cost, 3.0); - Ok(()) -} - -#[test] -fn tier_rollup_aggregates_shared_tier_under_first_model() -> Result<()> { - let accumulator = StatsAccumulator::new(); - accumulator.record_success("strong/first", None, Some("strong"))?; - accumulator.record_usage( - "strong/first", - TokenUsage { - prompt_tokens: 2, - completion_tokens: 3, - ..TokenUsage::default() - }, - None, - None, - Some("strong"), - )?; - accumulator.record_success("strong/second", None, Some("strong"))?; - accumulator.record_usage( - "strong/second", - TokenUsage { - prompt_tokens: 5, - completion_tokens: 7, - ..TokenUsage::default() - }, - None, - None, - Some("strong"), - )?; - - let snapshot = accumulator.snapshot()?; - let tier = snapshot - .tiers - .get("strong") - .ok_or_else(|| SwitchyardError::Other("strong tier should exist".to_string()))?; - - assert_eq!(tier.model, "strong/first"); - assert_eq!(tier.calls, 2); - assert_eq!(tier.prompt_tokens, 7); - assert_eq!(tier.completion_tokens, 10); - assert_eq!(tier.total_tokens, 17); - Ok(()) -} - -#[test] -fn tier_rollup_keeps_distinct_tiers_for_shared_model() -> Result<()> { - let accumulator = StatsAccumulator::new(); - accumulator.record_success("shared/model", None, Some("weak"))?; - accumulator.record_usage( - "shared/model", - TokenUsage { - prompt_tokens: 2, - completion_tokens: 3, - ..TokenUsage::default() - }, - None, - None, - Some("weak"), - )?; - accumulator.record_success("shared/model", None, Some("primary"))?; - accumulator.record_usage( - "shared/model", - TokenUsage { - prompt_tokens: 5, - completion_tokens: 7, - ..TokenUsage::default() - }, - None, - None, - Some("primary"), - )?; - - let snapshot = accumulator.snapshot()?; - let shared = model_stats(&snapshot, "shared/model")?; - assert_eq!(shared.calls, 2); - assert_eq!(shared.prompt_tokens, 7); - assert_eq!(shared.completion_tokens, 10); - - let weak = snapshot - .tiers - .get("weak") - .ok_or_else(|| SwitchyardError::Other("weak tier should exist".to_string()))?; - assert_eq!(weak.model, "shared/model"); - assert_eq!(weak.calls, 1); - assert_eq!(weak.prompt_tokens, 2); - assert_eq!(weak.completion_tokens, 3); - - let primary = snapshot - .tiers - .get("primary") - .ok_or_else(|| SwitchyardError::Other("primary tier should exist".to_string()))?; - assert_eq!(primary.model, "shared/model"); - assert_eq!(primary.calls, 1); - assert_eq!(primary.prompt_tokens, 5); - assert_eq!(primary.completion_tokens, 7); - Ok(()) -} - -#[test] -fn tier_usage_can_attach_explicit_untiered_success() -> Result<()> { - let accumulator = StatsAccumulator::new(); - accumulator.record_success("shared/model", None, None)?; - accumulator.record_success("shared/model", None, None)?; - accumulator.record_usage_with_success_was_untiered( - "shared/model", - TokenUsage { - prompt_tokens: 2, - completion_tokens: 3, - ..TokenUsage::default() - }, - None, - None, - Some("weak"), - )?; - accumulator.record_usage_with_success_was_untiered( - "shared/model", - TokenUsage { - prompt_tokens: 5, - completion_tokens: 7, - ..TokenUsage::default() - }, - None, - None, - Some("primary"), - )?; - - let snapshot = accumulator.snapshot()?; - let shared = model_stats(&snapshot, "shared/model")?; - assert_eq!(shared.calls, 2); - assert_eq!(shared.prompt_tokens, 7); - assert_eq!(shared.completion_tokens, 10); - - let weak = snapshot - .tiers - .get("weak") - .ok_or_else(|| SwitchyardError::Other("weak tier should exist".to_string()))?; - assert_eq!(weak.calls, 1); - assert_eq!(weak.prompt_tokens, 2); - assert_eq!(weak.completion_tokens, 3); - - let primary = snapshot - .tiers - .get("primary") - .ok_or_else(|| SwitchyardError::Other("primary tier should exist".to_string()))?; - assert_eq!(primary.calls, 1); - assert_eq!(primary.prompt_tokens, 5); - assert_eq!(primary.completion_tokens, 7); - Ok(()) -} - -#[test] -fn tier_usage_preserves_legacy_untiered_success_sequence() -> Result<()> { - let accumulator = StatsAccumulator::new(); - accumulator.record_success("shared/model", None, None)?; - accumulator.record_usage( - "shared/model", - TokenUsage { - prompt_tokens: 2, - completion_tokens: 3, - ..TokenUsage::default() - }, - None, - None, - Some("weak"), - )?; - - let snapshot = accumulator.snapshot()?; - let shared = model_stats(&snapshot, "shared/model")?; - assert_eq!(shared.calls, 1); - assert_eq!(shared.prompt_tokens, 2); - assert_eq!(shared.completion_tokens, 3); - - let weak = snapshot - .tiers - .get("weak") - .ok_or_else(|| SwitchyardError::Other("weak tier should exist".to_string()))?; - assert_eq!(weak.calls, 1); - assert_eq!(weak.prompt_tokens, 2); - assert_eq!(weak.completion_tokens, 3); - Ok(()) -} - -#[test] -fn already_attributed_usage_does_not_consume_legacy_pending_success() -> Result<()> { - let accumulator = StatsAccumulator::new(); - accumulator.record_success("shared/model", None, None)?; - accumulator.record_success("shared/model", None, Some("weak"))?; - accumulator.record_usage_after_success_attribution( - "shared/model", - TokenUsage { - prompt_tokens: 2, - completion_tokens: 3, - ..TokenUsage::default() - }, - None, - None, - Some("weak"), - )?; - accumulator.record_usage( - "shared/model", - TokenUsage { - prompt_tokens: 5, - completion_tokens: 7, - ..TokenUsage::default() - }, - None, - None, - Some("primary"), - )?; - - let snapshot = accumulator.snapshot()?; - let shared = model_stats(&snapshot, "shared/model")?; - assert_eq!(shared.calls, 2); - assert_eq!(shared.prompt_tokens, 7); - assert_eq!(shared.completion_tokens, 10); - - let weak = snapshot - .tiers - .get("weak") - .ok_or_else(|| SwitchyardError::Other("weak tier should exist".to_string()))?; - assert_eq!(weak.calls, 1); - assert_eq!(weak.prompt_tokens, 2); - assert_eq!(weak.completion_tokens, 3); - - let primary = snapshot - .tiers - .get("primary") - .ok_or_else(|| SwitchyardError::Other("primary tier should exist".to_string()))?; - assert_eq!(primary.calls, 1); - assert_eq!(primary.prompt_tokens, 5); - assert_eq!(primary.completion_tokens, 7); - Ok(()) -} - -#[test] -fn generic_tier_labels_are_included_in_tier_rollups() -> Result<()> { - let accumulator = StatsAccumulator::new(); - accumulator.record_success("plugin-a", None, Some("plugin"))?; - accumulator.record_usage( - "plugin-a", - TokenUsage { - prompt_tokens: 2, - completion_tokens: 3, - ..TokenUsage::default() - }, - None, - None, - Some("plugin"), - )?; - accumulator.record_success("plugin-b", None, Some("plugin"))?; - accumulator.record_usage( - "plugin-b", - TokenUsage { - prompt_tokens: 5, - completion_tokens: 7, - ..TokenUsage::default() - }, - None, - None, - Some("plugin"), - )?; - - let snapshot = accumulator.snapshot()?; - let plugin_a = model_stats(&snapshot, "plugin-a")?; - let plugin_b = model_stats(&snapshot, "plugin-b")?; - - assert_eq!(plugin_a.tier.as_deref(), Some("plugin")); - assert_eq!(plugin_b.tier.as_deref(), Some("plugin")); - let plugin_tier = snapshot - .tiers - .get("plugin") - .ok_or_else(|| SwitchyardError::Other("plugin tier should exist".to_string()))?; - assert_eq!(plugin_tier.model, "plugin-a"); - assert_eq!(plugin_tier.calls, 2); - assert_eq!(plugin_tier.prompt_tokens, 7); - assert_eq!(plugin_tier.completion_tokens, 10); - assert_eq!(plugin_tier.total_tokens, 17); - assert_eq!(snapshot.total_tokens.prompt, 7); - assert_eq!(snapshot.total_tokens.completion, 10); - Ok(()) -} - -#[test] -fn classifier_bucket_is_empty_by_default() -> Result<()> { - let accumulator = StatsAccumulator::new(); - accumulator.record_usage( - "claude-sonnet-4-6", - TokenUsage { - prompt_tokens: 1_000_000, - ..TokenUsage::default() - }, - None, - None, - Some("strong"), - )?; - let snapshot = accumulator.snapshot()?; - assert!(snapshot.classifier.models.is_empty()); - assert_eq!(snapshot.classifier.total_requests, 0); - assert_eq!(snapshot.cost_estimate.classifier_cost, 0.0); - // backend_cost == total_cost when classifier is empty. - assert_eq!( - snapshot.cost_estimate.backend_cost, - snapshot.cost_estimate.total_cost - ); - Ok(()) -} - -#[test] -fn classifier_bucket_keeps_same_model_separate_from_routed_traffic() -> Result<()> { - // Default TB-lite config: classifier_model == weak_model. Both record - // against the same model id but must land in distinct buckets so spend - // is plainly attributable. - let accumulator = StatsAccumulator::new(); - let model = "claude-sonnet-4-6"; - accumulator.record_success(model, None, Some("weak"))?; - accumulator.record_usage( - model, - TokenUsage { - prompt_tokens: 1_000_000, - ..TokenUsage::default() - }, - None, - None, - Some("weak"), - )?; - accumulator.record_classifier_usage( - model, - TokenUsage { - prompt_tokens: 500_000, - ..TokenUsage::default() - }, - Some(42.0), - )?; - - let snapshot = accumulator.snapshot()?; - - // Backend bucket: one row, original 1M prompt tokens. - let backend = snapshot - .models - .get(model) - .ok_or_else(|| SwitchyardError::Other("backend row missing".to_string()))?; - assert_eq!(backend.prompt_tokens, 1_000_000); - assert_eq!(backend.max_observed_context_tokens, 1_000_000); - assert_eq!(backend.calls, 1); - - // Classifier bucket: same model id, separate row, separate counts. - let classifier = snapshot - .classifier - .models - .get(model) - .ok_or_else(|| SwitchyardError::Other("classifier row missing".to_string()))?; - assert_eq!(classifier.prompt_tokens, 500_000); - assert_eq!(classifier.max_observed_context_tokens, 500_000); - assert_eq!(classifier.calls, 1); - assert_eq!(snapshot.classifier.total_requests, 1); - - // Cost split: backend = 3.0 (1M @ $3/M); classifier = 1.5 (500k @ $3/M). - // total_cost rolls both in. - assert_eq!(snapshot.cost_estimate.backend_cost, 3.0); - assert_eq!(snapshot.cost_estimate.classifier_cost, 1.5); - assert_eq!(snapshot.cost_estimate.total_cost, 4.5); - Ok(()) -} - -#[test] -fn classifier_latency_lands_on_classifier_model_call_latency() -> Result<()> { - let accumulator = StatsAccumulator::new(); - accumulator.record_classifier_usage( - "claude-sonnet-4-6", - TokenUsage { - prompt_tokens: 10, - completion_tokens: 5, - ..TokenUsage::default() - }, - Some(15.0), - )?; - let snapshot = accumulator.snapshot()?; - let row = snapshot - .classifier - .models - .get("claude-sonnet-4-6") - .ok_or_else(|| SwitchyardError::Other("classifier row missing".to_string()))?; - assert_eq!(row.max_observed_context_tokens, 15); - assert_eq!(row.model_call_latency.count, 1); - assert_eq!(row.model_call_latency.total_ms, 15.0); - Ok(()) -} - -#[test] -fn reset_clears_classifier_bucket() -> Result<()> { - let accumulator = StatsAccumulator::new(); - accumulator.record_classifier_usage( - "claude-sonnet-4-6", - TokenUsage { - prompt_tokens: 100, - ..TokenUsage::default() - }, - None, - )?; - accumulator.reset()?; - let snapshot = accumulator.snapshot()?; - assert!(snapshot.classifier.models.is_empty()); - assert_eq!(snapshot.classifier.total_requests, 0); - assert_eq!(snapshot.cost_estimate.classifier_cost, 0.0); - Ok(()) -} - -#[test] -fn routing_decision_counts_are_grouped_by_profile_type() -> Result<()> { - let accumulator = StatsAccumulator::new(); - accumulator.record_routing_decision("stage_router", "dimensions")?; - accumulator.record_routing_decision("stage_router", "dimensions")?; - accumulator.record_routing_decision("stage_router", "llm-classifier")?; - accumulator.record_routing_decision("escalation_router", "judge")?; - - let snapshot = accumulator.snapshot()?; - assert_eq!( - snapshot - .routing_decisions - .get("stage_router") - .and_then(|sources| sources.get("dimensions")), - Some(&2) - ); - assert_eq!( - snapshot - .routing_decisions - .get("stage_router") - .and_then(|sources| sources.get("llm-classifier")), - Some(&1) - ); - assert_eq!( - snapshot - .routing_decisions - .get("escalation_router") - .and_then(|sources| sources.get("judge")), - Some(&1) - ); - - accumulator.reset()?; - assert!(accumulator.snapshot()?.routing_decisions.is_empty()); - Ok(()) -} - -fn model_stats<'a>(snapshot: &'a StatsSnapshot, model: &str) -> Result<&'a ModelStatsSnapshot> { - snapshot - .models - .get(model) - .ok_or_else(|| SwitchyardError::Other(format!("model stats missing for {model}"))) -} diff --git a/crates/switchyard-components/tests/stats_processors.rs b/crates/switchyard-components/tests/stats_processors.rs deleted file mode 100644 index 434b7f0e9..000000000 --- a/crates/switchyard-components/tests/stats_processors.rs +++ /dev/null @@ -1,838 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Stats processor tests covering request stamps, backend wrappers, and stream usage. - -use std::sync::Arc; -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::time::Duration; - -use async_trait::async_trait; -use futures_util::StreamExt; -use parking_lot::Mutex; -use serde_json::json; -use switchyard_components::{ - BackendSelection, BackendSelectionReason, RandomRoutingDecision, RandomRoutingTier, - StatsAccumulator, StatsBackendLatency, StatsLlmBackend, StatsRequestProcessor, - StatsRequestStart, StatsResponseProcessor, StatsRouteLabel, -}; -use switchyard_components::{ - ChatRequest, ChatRequestType, ChatResponse, LlmBackend, LlmTargetId, ModelId, ProxyContext, - Result, StreamEvent, SwitchyardError, -}; - -static SUPPORTED_OPENAI_CHAT: [ChatRequestType; 1] = [ChatRequestType::OpenAiChat]; - -// Stamps a served model into context the same way native backends do. -fn record_backend_selection(ctx: &mut ProxyContext, model: ModelId) { - let _ = ctx.insert(BackendSelection::for_model( - model, - None, - BackendSelectionReason::PassthroughModel, - )); -} - -// Fake backend records lifecycle and request observations for stats tests. -struct FakeBackend { - response: Mutex>>, - calls: Mutex>>, - selected_model: Option, - tier: Option, - startup_count: AtomicUsize, - shutdown_count: AtomicUsize, -} - -impl FakeBackend { - // Creates a fake backend that returns one successful response. - fn success(response: ChatResponse) -> Self { - Self { - response: Mutex::new(Some(Ok(response))), - calls: Mutex::new(Vec::new()), - selected_model: None, - tier: None, - startup_count: AtomicUsize::new(0), - shutdown_count: AtomicUsize::new(0), - } - } - - // Creates a fake backend that returns one error. - fn error(error: SwitchyardError) -> Self { - Self { - response: Mutex::new(Some(Err(error))), - calls: Mutex::new(Vec::new()), - selected_model: None, - tier: None, - startup_count: AtomicUsize::new(0), - shutdown_count: AtomicUsize::new(0), - } - } - - // Configures the model that the fake backend records in context. - fn with_selected_model(mut self, model: ModelId) -> Self { - self.selected_model = Some(model); - self - } - - // Configures the route label that the fake backend records in context. - fn with_tier(mut self, tier: impl Into) -> Self { - self.tier = Some(tier.into()); - self - } - - // Returns every request model observed by the fake backend. - fn calls(&self) -> Result>> { - Ok(self.calls.lock().clone()) - } -} - -#[async_trait] -impl LlmBackend for FakeBackend { - fn supported_request_types(&self) -> &[ChatRequestType] { - &SUPPORTED_OPENAI_CHAT - } - - async fn call(&self, ctx: &mut ProxyContext, request: &ChatRequest) -> Result { - self.calls.lock().push(request.model().map(str::to_string)); - if let Some(model) = &self.selected_model { - record_backend_selection(ctx, model.clone()); - } - if let Some(tier) = &self.tier { - ctx.insert(StatsRouteLabel::new(tier.clone())); - } - self.response - .lock() - .take() - .ok_or_else(|| SwitchyardError::Other("fake response already consumed".to_string()))? - } - - async fn startup(&self) -> Result<()> { - self.startup_count.fetch_add(1, Ordering::SeqCst); - Ok(()) - } - - async fn shutdown(&self) -> Result<()> { - self.shutdown_count.fetch_add(1, Ordering::SeqCst); - Ok(()) - } -} - -// Request stats should add timing state without changing the request. -#[tokio::test] -async fn request_processor_stamps_start_without_mutating_request() -> Result<()> { - let processor = StatsRequestProcessor::default(); - let request = ChatRequest::openai_chat(json!({"model": "client", "messages": []})); - let mut ctx = ProxyContext::new(); - - let processed = processor.process(&mut ctx, request.clone()).await?; - - assert_eq!(processed, request); - if ctx.get::().is_none() { - return Err(SwitchyardError::Other( - "stats request start should be stamped".to_string(), - )); - } - Ok(()) -} - -// End-to-end stats components should share one accumulator across the chain. -#[tokio::test] -async fn full_stats_chain_shares_one_accumulator_across_all_components() -> Result<()> { - let accumulator = StatsAccumulator::new(); - let request_processor = StatsRequestProcessor::default(); - let backend = StatsLlmBackend::new( - Arc::new( - FakeBackend::success(ChatResponse::openai_completion(json!({ - "usage": {"prompt_tokens": 12, "completion_tokens": 8} - }))) - .with_selected_model(ModelId::new("served-chain-model")?) - .with_tier("strong"), - ), - accumulator.clone(), - ); - let response_processor = StatsResponseProcessor::new(accumulator.clone()); - let mut ctx = ProxyContext::new(); - - let request = request_processor - .process( - &mut ctx, - ChatRequest::openai_chat(json!({"model": "client-model", "messages": []})), - ) - .await?; - let response = backend.call(&mut ctx, &request).await?; - response_processor.process(&mut ctx, response).await?; - - let snapshot = accumulator.snapshot()?; - assert_eq!(snapshot.total_requests, 1); - assert_eq!(snapshot.total_tokens.prompt, 12); - assert_eq!(snapshot.total_tokens.completion, 8); - assert_eq!(snapshot.routing_overhead.count, 1); - let model = model_stats(&snapshot, "served-chain-model")?; - assert_eq!(model.calls, 1); - assert_eq!(model.model_call_latency.count, 1); - assert_eq!(model.total_latency.count, 1); - Ok(()) -} - -// Theoretical flows request -> ctx -> response -> snapshot, and is switch-aware: -// a model only credits a prefix it has already been sent. -#[tokio::test] -async fn theoretical_cache_hit_rate_flows_through_the_chain() -> Result<()> { - let accumulator = StatsAccumulator::new(); - let request_processor = StatsRequestProcessor::new(true); - let response_processor = StatsResponseProcessor::new(accumulator.clone()); - // FakeBackend serves one response, so build a fresh one per turn. - let make_backend = || -> Result { - Ok(StatsLlmBackend::new( - Arc::new( - FakeBackend::success(ChatResponse::openai_completion(json!({ - "usage": {"prompt_tokens": 100, "completion_tokens": 4} - }))) - .with_selected_model(ModelId::new("served-model")?) - .with_tier("strong"), - ), - accumulator.clone(), - )) - }; - - // Turn 1: first sight of the conversation, nothing cached yet. - let mut ctx = ProxyContext::new(); - let request = request_processor - .process( - &mut ctx, - ChatRequest::openai_chat(json!({ - "model": "m", - "messages": [{"role": "user", "content": "aaaa"}], - })), - ) - .await?; - let response = make_backend()?.call(&mut ctx, &request).await?; - response_processor.process(&mut ctx, response).await?; - - // Turn 2: same model, the prior turn is re-presented, so half is eligible. - let mut ctx = ProxyContext::new(); - let request = request_processor - .process( - &mut ctx, - ChatRequest::openai_chat(json!({ - "model": "m", - "messages": [ - {"role": "user", "content": "aaaa"}, - {"role": "user", "content": "bbbb"}, - ], - })), - ) - .await?; - let response = make_backend()?.call(&mut ctx, &request).await?; - response_processor.process(&mut ctx, response).await?; - - let snapshot = accumulator.snapshot()?; - let model = model_stats(&snapshot, "served-model")?; - assert_eq!(model.cache_hit_rate, 0.0); - // Turn 1 cold (0/100) + turn 2 half (50/100) = 50 over 200 prompt tokens. - assert_eq!(model.theoretical_cache_hit_rate, 0.25); - Ok(()) -} - -// Backend wrapper should record served model latency and delegate lifecycle. -#[tokio::test] -async fn backend_wrapper_records_success_using_served_model_and_delegates_lifecycle() -> Result<()> -{ - let accumulator = StatsAccumulator::new(); - let inner = Arc::new( - FakeBackend::success(ChatResponse::openai_completion(json!({"id": "ok"}))) - .with_selected_model(ModelId::new("served-model")?) - .with_tier("strong"), - ); - let backend = StatsLlmBackend::new(inner.clone(), accumulator.clone()); - let request = ChatRequest::openai_chat(json!({"model": "client-model", "messages": []})); - let mut ctx = ProxyContext::new(); - - assert_eq!( - backend.supported_request_types(), - &[ChatRequestType::OpenAiChat] - ); - backend.startup().await?; - let response = backend.call(&mut ctx, &request).await?; - backend.shutdown().await?; - - assert!(matches!(response, ChatResponse::OpenAiCompletion(_))); - assert_eq!(inner.startup_count.load(Ordering::SeqCst), 1); - assert_eq!(inner.shutdown_count.load(Ordering::SeqCst), 1); - assert_eq!(inner.calls()?, vec![Some("client-model".to_string())]); - if ctx.get::().is_none() { - return Err(SwitchyardError::Other( - "backend latency should be stamped".to_string(), - )); - } - - let snapshot = accumulator.snapshot()?; - assert_eq!(snapshot.total_requests, 1); - let model = model_stats(&snapshot, "served-model")?; - assert_eq!(model.calls, 1); - assert_eq!(model.errors, 0); - assert_eq!(model.tier.as_deref(), Some("strong")); - assert_eq!(model.model_call_latency.count, 1); - Ok(()) -} - -// If a backend does not stamp a served model, request model is the fallback. -#[tokio::test] -async fn backend_wrapper_falls_back_to_request_model_when_backend_does_not_stamp_model() --> Result<()> { - let accumulator = StatsAccumulator::new(); - let backend = StatsLlmBackend::new( - Arc::new(FakeBackend::success(ChatResponse::openai_completion( - json!({"id": "ok"}), - ))), - accumulator.clone(), - ); - let request = ChatRequest::openai_chat(json!({"model": "client-fallback", "messages": []})); - let mut ctx = ProxyContext::new(); - - backend.call(&mut ctx, &request).await?; - - let snapshot = accumulator.snapshot()?; - let model = model_stats(&snapshot, "client-fallback")?; - assert_eq!(model.calls, 1); - Ok(()) -} - -// Backend wrapper should preserve the original error while counting it. -#[tokio::test] -async fn backend_wrapper_records_errors_and_preserves_original_error() -> Result<()> { - let accumulator = StatsAccumulator::new(); - let inner = Arc::new( - FakeBackend::error(SwitchyardError::Upstream("boom".to_string())) - .with_selected_model(ModelId::new("served-error-model")?) - .with_tier("weak"), - ); - let backend = StatsLlmBackend::new(inner, accumulator.clone()); - let request = ChatRequest::openai_chat(json!({"model": "client-model", "messages": []})); - let mut ctx = ProxyContext::new(); - - let Err(error) = backend.call(&mut ctx, &request).await else { - return Err(SwitchyardError::Other( - "backend wrapper should return inner error".to_string(), - )); - }; - - assert!(matches!(error, SwitchyardError::Upstream(_))); - assert!(error.to_string().contains("boom")); - let snapshot = accumulator.snapshot()?; - assert_eq!(snapshot.total_requests, 1); - assert_eq!(snapshot.total_errors, 1); - let model = model_stats(&snapshot, "served-error-model")?; - assert_eq!(model.calls, 0); - assert_eq!(model.errors, 1); - assert_eq!(model.tier.as_deref(), Some("weak")); - Ok(()) -} - -// Response stats should clamp impossible negative overhead to zero. -#[tokio::test] -async fn response_processor_records_openai_usage_latency_and_clamps_negative_overhead() -> Result<()> -{ - let accumulator = StatsAccumulator::new(); - let processor = StatsResponseProcessor::new(accumulator.clone()); - let mut ctx = ProxyContext::new(); - record_backend_selection(&mut ctx, ModelId::new("openai-model")?); - ctx.insert(StatsRequestStart::now()); - ctx.insert(StatsBackendLatency(Duration::from_secs(60))); - - let response = ChatResponse::openai_completion(json!({ - "usage": { - "prompt_tokens": 11, - "completion_tokens": 5, - "prompt_tokens_details": { - "cached_tokens": 3, - "cache_creation_tokens": 2 - }, - "completion_tokens_details": { - "reasoning_tokens": 4 - } - } - })); - let processed = processor.process(&mut ctx, response).await?; - - assert!(matches!(processed, ChatResponse::OpenAiCompletion(_))); - let snapshot = accumulator.snapshot()?; - assert_eq!(snapshot.total_requests, 0); - assert_eq!(snapshot.total_tokens.prompt, 11); - assert_eq!(snapshot.total_tokens.completion, 5); - assert_eq!(snapshot.total_tokens.cached, 3); - assert_eq!(snapshot.total_tokens.cache_creation, 2); - assert_eq!(snapshot.total_tokens.reasoning, 4); - assert_eq!(snapshot.routing_overhead.count, 1); - assert_eq!(snapshot.routing_overhead.max_ms, 0.0); - let model = model_stats(&snapshot, "openai-model")?; - assert_eq!(model.total_latency.count, 1); - Ok(()) -} - -// Anthropic cache counters should contribute to prompt token accounting. -#[tokio::test] -async fn response_processor_sums_anthropic_cache_buckets_into_prompt_tokens() -> Result<()> { - let accumulator = StatsAccumulator::new(); - let processor = StatsResponseProcessor::new(accumulator.clone()); - let mut ctx = ProxyContext::new(); - record_backend_selection(&mut ctx, ModelId::new("claude-model")?); - - let response = ChatResponse::anthropic_completion(json!({ - "usage": { - "input_tokens": 10, - "output_tokens": 6, - "cache_read_input_tokens": 3, - "cache_creation_input_tokens": 2, - "output_tokens_details": {"reasoning_tokens": 1} - } - })); - processor.process(&mut ctx, response).await?; - - let snapshot = accumulator.snapshot()?; - let model = model_stats(&snapshot, "claude-model")?; - assert_eq!(model.prompt_tokens, 15); - assert_eq!(model.completion_tokens, 6); - assert_eq!(model.cached_tokens, 3); - assert_eq!(model.cache_creation_tokens, 2); - assert_eq!(model.reasoning_tokens, 1); - Ok(()) -} - -// Responses streams should pass through unchanged while capturing final usage. -#[tokio::test] -async fn streaming_response_is_forwarded_and_records_nested_responses_usage() -> Result<()> { - let accumulator = StatsAccumulator::new(); - let processor = StatsResponseProcessor::new(accumulator.clone()); - let mut ctx = ProxyContext::new(); - record_backend_selection(&mut ctx, ModelId::new("responses-model")?); - ctx.insert(StatsRequestStart::now()); - - let events = vec![ - StreamEvent::Json(json!({"type": "response.output_text.delta", "delta": "hi"})), - StreamEvent::Json(json!({ - "type": "response.in_progress", - "response": {"usage": null} - })), - StreamEvent::Json(json!({ - "type": "response.completed", - "response": { - "usage": { - "input_tokens": 8, - "output_tokens": 13, - "input_tokens_details": {"cached_tokens": 5} - } - } - })), - ]; - let stream_events = events.clone(); - let stream = futures_util::stream::iter(stream_events.into_iter().map(Ok)); - let response = ChatResponse::OpenAiResponsesStream(Box::pin(stream)); - - let processed = processor.process(&mut ctx, response).await?; - let drained = drain_responses_stream(processed).await?; - - assert_eq!(drained, events); - let snapshot = accumulator.snapshot()?; - let model = model_stats(&snapshot, "responses-model")?; - assert_eq!(model.prompt_tokens, 8); - assert_eq!(model.completion_tokens, 13); - assert_eq!(model.cached_tokens, 5); - assert_eq!(model.total_latency.count, 1); - Ok(()) -} - -// Raw SSE frame strings (verbatim Responses passthrough) still record usage. -#[tokio::test] -async fn raw_sse_frame_responses_stream_passes_through_and_records_usage() -> Result<()> { - let accumulator = StatsAccumulator::new(); - let processor = StatsResponseProcessor::new(accumulator.clone()); - let mut ctx = ProxyContext::new(); - record_backend_selection(&mut ctx, ModelId::new("raw-frame-model")?); - ctx.insert(StatsRequestStart::now()); - - let events = vec![ - StreamEvent::Text( - "event: response.output_text.delta\n\ - data: {\"type\":\"response.output_text.delta\",\"delta\":\"hi\"}\n\n" - .to_string(), - ), - StreamEvent::Text(": keep-alive\n\n".to_string()), - StreamEvent::Text( - "event: response.completed\n\ - data: {\"type\":\"response.completed\",\"response\":{\"usage\":\ - {\"input_tokens\":8,\"output_tokens\":13,\ - \"input_tokens_details\":{\"cached_tokens\":5}}}}\n\n" - .to_string(), - ), - ]; - let stream_events = events.clone(); - let stream = futures_util::stream::iter(stream_events.into_iter().map(Ok)); - let response = ChatResponse::OpenAiResponsesStream(Box::pin(stream)); - - let processed = processor.process(&mut ctx, response).await?; - let drained = drain_responses_stream(processed).await?; - - // Frames pass through byte-identical; usage still lands in the snapshot. - assert_eq!(drained, events); - let snapshot = accumulator.snapshot()?; - let model = model_stats(&snapshot, "raw-frame-model")?; - assert_eq!(model.prompt_tokens, 8); - assert_eq!(model.completion_tokens, 13); - assert_eq!(model.cached_tokens, 5); - Ok(()) -} - -// OpenAI streams should record the first real usage block only. -#[tokio::test] -async fn openai_chat_stream_records_first_usage_chunk_only_with_details() -> Result<()> { - let accumulator = StatsAccumulator::new(); - let processor = StatsResponseProcessor::new(accumulator.clone()); - let mut ctx = ProxyContext::new(); - record_backend_selection(&mut ctx, ModelId::new("chat-stream-model")?); - - let events = vec![ - StreamEvent::Json(json!({"usage": null})), - StreamEvent::Json(json!({"choices": [{"delta": {"content": "hi"}}]})), - StreamEvent::Json(json!({ - "usage": { - "prompt_tokens": 20, - "completion_tokens": 7, - "prompt_tokens_details": {"cached_tokens": 4}, - "completion_tokens_details": {"reasoning_tokens": 2} - } - })), - StreamEvent::Json(json!({ - "usage": {"prompt_tokens": 200, "completion_tokens": 70} - })), - ]; - let stream = futures_util::stream::iter(events.clone().into_iter().map(Ok)); - let processed = processor - .process(&mut ctx, ChatResponse::OpenAiStream(Box::pin(stream))) - .await?; - let drained = drain_openai_stream(processed).await?; - - assert_eq!(drained, events); - let snapshot = accumulator.snapshot()?; - let model = model_stats(&snapshot, "chat-stream-model")?; - assert_eq!(model.prompt_tokens, 20); - assert_eq!(model.completion_tokens, 7); - assert_eq!(model.cached_tokens, 4); - assert_eq!(model.reasoning_tokens, 2); - assert_eq!(model.total_latency.count, 0); - Ok(()) -} - -// Anthropic streaming usage can arrive in start and delta events. -#[tokio::test] -async fn anthropic_stream_merges_start_and_delta_usage_and_commits_once_at_stop() -> Result<()> { - let accumulator = StatsAccumulator::new(); - let processor = StatsResponseProcessor::new(accumulator.clone()); - let mut ctx = ProxyContext::new(); - record_backend_selection(&mut ctx, ModelId::new("claude-stream-model")?); - ctx.insert(StatsRequestStart::now()); - - let events = vec![ - StreamEvent::Json(json!({ - "type": "message_start", - "message": { - "usage": { - "input_tokens": 50, - "cache_read_input_tokens": 10, - "cache_creation_input_tokens": 5 - } - } - })), - StreamEvent::Json(json!({ - "type": "message_delta", - "usage": {"output_tokens": 20} - })), - StreamEvent::Json(json!({"type": "message_stop"})), - StreamEvent::Json(json!({"type": "message_stop"})), - ]; - let stream = futures_util::stream::iter(events.clone().into_iter().map(Ok)); - let processed = processor - .process(&mut ctx, ChatResponse::AnthropicStream(Box::pin(stream))) - .await?; - let drained = drain_anthropic_stream(processed).await?; - - assert_eq!(drained, events); - let snapshot = accumulator.snapshot()?; - let model = model_stats(&snapshot, "claude-stream-model")?; - assert_eq!(model.prompt_tokens, 65); - assert_eq!(model.completion_tokens, 20); - assert_eq!(model.cached_tokens, 10); - assert_eq!(model.cache_creation_tokens, 5); - assert_eq!(model.total_latency.count, 1); - Ok(()) -} - -// Later Anthropic input-token deltas should override a zero start value. -#[tokio::test] -async fn anthropic_stream_delta_input_tokens_override_zero_start_tokens() -> Result<()> { - let accumulator = StatsAccumulator::new(); - let processor = StatsResponseProcessor::new(accumulator.clone()); - let mut ctx = ProxyContext::new(); - record_backend_selection(&mut ctx, ModelId::new("claude-delta-input")?); - - let events = vec![ - StreamEvent::Json(json!({ - "type": "message_start", - "message": {"usage": {"input_tokens": 0}} - })), - StreamEvent::Json(json!({ - "type": "message_delta", - "usage": {"input_tokens": 75, "output_tokens": 30} - })), - StreamEvent::Json(json!({"type": "message_stop"})), - ]; - let stream = futures_util::stream::iter(events.into_iter().map(Ok)); - let processed = processor - .process(&mut ctx, ChatResponse::AnthropicStream(Box::pin(stream))) - .await?; - let _ = drain_anthropic_stream(processed).await?; - - let snapshot = accumulator.snapshot()?; - let model = model_stats(&snapshot, "claude-delta-input")?; - assert_eq!(model.prompt_tokens, 75); - assert_eq!(model.completion_tokens, 30); - Ok(()) -} - -// Streams with no usage should remain transparent and avoid fake stats entries. -#[tokio::test] -async fn stream_without_usage_passes_through_without_creating_stats_entry() -> Result<()> { - let accumulator = StatsAccumulator::new(); - let processor = StatsResponseProcessor::new(accumulator.clone()); - let mut ctx = ProxyContext::new(); - record_backend_selection(&mut ctx, ModelId::new("no-usage-stream")?); - - let events = vec![ - StreamEvent::Json(json!({"choices": [{"delta": {"content": "a"}}]})), - StreamEvent::Text("plain text ignored".to_string()), - ]; - let stream = futures_util::stream::iter(events.clone().into_iter().map(Ok)); - let processed = processor - .process(&mut ctx, ChatResponse::OpenAiStream(Box::pin(stream))) - .await?; - - assert_eq!(drain_openai_stream(processed).await?, events); - assert!(accumulator.snapshot()?.models.is_empty()); - Ok(()) -} - -// Buffered responses without usage still create a zero-token model entry. -#[tokio::test] -async fn buffered_response_without_usage_records_zero_token_model_entry() -> Result<()> { - let accumulator = StatsAccumulator::new(); - let processor = StatsResponseProcessor::new(accumulator.clone()); - let mut ctx = ProxyContext::new(); - record_backend_selection(&mut ctx, ModelId::new("missing-usage-model")?); - - processor - .process( - &mut ctx, - ChatResponse::openai_completion(json!({"id": "no-usage"})), - ) - .await?; - - let snapshot = accumulator.snapshot()?; - let model = model_stats(&snapshot, "missing-usage-model")?; - assert_eq!(model.prompt_tokens, 0); - assert_eq!(model.completion_tokens, 0); - assert_eq!(snapshot.total_tokens.total, 0); - Ok(()) -} - -// Malformed usage fields should be ignored without losing valid alternatives. -#[tokio::test] -async fn malformed_usage_values_are_ignored_without_wrapping_or_panicking() -> Result<()> { - let accumulator = StatsAccumulator::new(); - let processor = StatsResponseProcessor::new(accumulator.clone()); - let mut ctx = ProxyContext::new(); - record_backend_selection(&mut ctx, ModelId::new("malformed-usage-model")?); - - processor - .process( - &mut ctx, - ChatResponse::openai_completion(json!({ - "usage": { - "prompt_tokens": -1, - "completion_tokens": "bad", - "input_tokens": 9, - "output_tokens": true, - "prompt_tokens_details": {"cached_tokens": null}, - "completion_tokens_details": {"reasoning_tokens": 3.5} - } - })), - ) - .await?; - - let snapshot = accumulator.snapshot()?; - let model = model_stats(&snapshot, "malformed-usage-model")?; - assert_eq!(model.prompt_tokens, 9); - assert_eq!(model.completion_tokens, 0); - assert_eq!(model.cached_tokens, 0); - assert_eq!(model.reasoning_tokens, 0); - Ok(()) -} - -// Random-routing context should populate tier rollups in response stats. -#[tokio::test] -async fn response_processor_uses_random_routing_decision_for_tier_rollups() -> Result<()> { - let accumulator = StatsAccumulator::new(); - let processor = StatsResponseProcessor::new(accumulator.clone()); - let mut ctx = ProxyContext::new(); - record_backend_selection(&mut ctx, ModelId::new("weak-model")?); - ctx.insert(RandomRoutingDecision { - tier: RandomRoutingTier::Weak, - selected_target: LlmTargetId::from_static("weak"), - selected_model: ModelId::new("weak-model")?, - original_model: Some("client-model".to_string()), - strong_probability: 0.25, - draw: 0.75, - }); - - processor - .process( - &mut ctx, - ChatResponse::openai_completion(json!({ - "usage": {"prompt_tokens": 4, "completion_tokens": 9} - })), - ) - .await?; - - let snapshot = accumulator.snapshot()?; - let tier = snapshot - .tiers - .get("weak") - .ok_or_else(|| SwitchyardError::Other("weak tier should be present".to_string()))?; - assert_eq!(tier.model, "weak-model"); - assert_eq!(tier.prompt_tokens, 4); - assert_eq!(tier.completion_tokens, 9); - assert_eq!(tier.total_tokens, 13); - Ok(()) -} - -// Context-selected targets should be a generic tier fallback when no richer -// router-specific label exists. -#[tokio::test] -async fn response_processor_uses_selected_target_as_tier_fallback() -> Result<()> { - let accumulator = StatsAccumulator::new(); - let processor = StatsResponseProcessor::new(accumulator.clone()); - let mut ctx = ProxyContext::new(); - record_backend_selection(&mut ctx, ModelId::new("primary-model")?); - ctx.set_selected_target(LlmTargetId::from_static("primary")); - - processor - .process( - &mut ctx, - ChatResponse::openai_completion(json!({ - "usage": {"prompt_tokens": 3, "completion_tokens": 5} - })), - ) - .await?; - - let snapshot = accumulator.snapshot()?; - let tier = snapshot - .tiers - .get("primary") - .ok_or_else(|| SwitchyardError::Other("primary tier should be present".to_string()))?; - assert_eq!(tier.model, "primary-model"); - assert_eq!(tier.prompt_tokens, 3); - assert_eq!(tier.completion_tokens, 5); - Ok(()) -} - -// Explicit stats route labels intentionally override random-routing labels. -#[tokio::test] -async fn explicit_stats_route_label_takes_precedence_over_random_routing_decision() -> Result<()> { - let accumulator = StatsAccumulator::new(); - let processor = StatsResponseProcessor::new(accumulator.clone()); - let mut ctx = ProxyContext::new(); - record_backend_selection(&mut ctx, ModelId::new("labeled-model")?); - ctx.insert(StatsRouteLabel::new("plugin")); - ctx.insert(RandomRoutingDecision { - tier: RandomRoutingTier::Weak, - selected_target: LlmTargetId::from_static("weak"), - selected_model: ModelId::new("labeled-model")?, - original_model: None, - strong_probability: 0.25, - draw: 0.75, - }); - - processor - .process( - &mut ctx, - ChatResponse::openai_completion(json!({ - "usage": {"prompt_tokens": 1, "completion_tokens": 1} - })), - ) - .await?; - - let snapshot = accumulator.snapshot()?; - let model = model_stats(&snapshot, "labeled-model")?; - assert_eq!(model.tier.as_deref(), Some("plugin")); - let plugin = snapshot - .tiers - .get("plugin") - .ok_or_else(|| SwitchyardError::Other("plugin tier should be present".to_string()))?; - assert_eq!(plugin.model, "labeled-model"); - assert_eq!(plugin.calls, 0); - assert_eq!(plugin.prompt_tokens, 1); - assert_eq!(plugin.completion_tokens, 1); - assert!(!snapshot.tiers.contains_key("weak")); - Ok(()) -} - -// Drains an OpenAI Responses stream after the stats wrapper has observed it. -async fn drain_responses_stream(response: ChatResponse) -> Result> { - let ChatResponse::OpenAiResponsesStream(mut stream) = response else { - return Err(SwitchyardError::Other( - "response should remain an OpenAI Responses stream".to_string(), - )); - }; - let mut events = Vec::new(); - while let Some(event) = stream.next().await { - events.push(event?); - } - Ok(events) -} - -// Drains an OpenAI Chat stream after the stats wrapper has observed it. -async fn drain_openai_stream(response: ChatResponse) -> Result> { - let ChatResponse::OpenAiStream(mut stream) = response else { - return Err(SwitchyardError::Other( - "response should remain an OpenAI stream".to_string(), - )); - }; - let mut events = Vec::new(); - while let Some(event) = stream.next().await { - events.push(event?); - } - Ok(events) -} - -// Drains an Anthropic stream after the stats wrapper has observed it. -async fn drain_anthropic_stream(response: ChatResponse) -> Result> { - let ChatResponse::AnthropicStream(mut stream) = response else { - return Err(SwitchyardError::Other( - "response should remain an Anthropic stream".to_string(), - )); - }; - let mut events = Vec::new(); - while let Some(event) = stream.next().await { - events.push(event?); - } - Ok(events) -} - -// Fetches one model stats block with an explicit test error on absence. -fn model_stats<'a>( - snapshot: &'a switchyard_components::StatsSnapshot, - model: &str, -) -> Result<&'a switchyard_components::ModelStatsSnapshot> { - snapshot - .models - .get(model) - .ok_or_else(|| SwitchyardError::Other(format!("model stats missing for {model}"))) -} diff --git a/crates/switchyard-components/tests/stats_usage_shapes.rs b/crates/switchyard-components/tests/stats_usage_shapes.rs deleted file mode 100644 index d25f983b1..000000000 --- a/crates/switchyard-components/tests/stats_usage_shapes.rs +++ /dev/null @@ -1,241 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Parity tests for the OpenAI vs Anthropic usage shapes. -//! -//! OpenAI exposes `prompt_tokens` as the *inclusive* total with -//! `prompt_tokens_details.{cached_tokens, cache_creation_tokens}` as nested -//! subsets. Anthropic exposes `input_tokens` (non-cached base), -//! `cache_read_input_tokens`, and `cache_creation_input_tokens` as -//! *sibling* fields — each disjoint from the others. The extractor in -//! `switchyard_components::stats::usage_from_body` must normalise both -//! into the OpenAI-style inclusive convention so the downstream cost -//! estimator (`base = prompt - cached - cache_creation`) never goes -//! negative and the cache buckets sum into the prompt total. The tests -//! below pin that contract. -//! -//! Replaces the Python `tests/test_cost_estimator_cache_wiring.py` Layer 1 -//! cases, which targeted a `_record()` helper that the Rust migration -//! eliminated. - -use serde_json::json; -use switchyard_components::TokenUsage; -use switchyard_components::stats::usage_from_body; - -#[test] -fn openai_inclusive_shape_extracts_prompt_as_inclusive_total() { - // OpenAI: `prompt_tokens` already counts the cached + cache_creation tokens. - let body = json!({ - "usage": { - "prompt_tokens": 550, - "completion_tokens": 100, - "prompt_tokens_details": { - "cached_tokens": 100, - "cache_creation_tokens": 50, - }, - } - }); - assert_eq!( - usage_from_body(&body), - TokenUsage { - prompt_tokens: 550, - completion_tokens: 100, - cached_tokens: 100, - cache_creation_tokens: 50, - reasoning_tokens: 0, - cacheable_prompt_tokens: 0, - } - ); -} - -#[test] -fn anthropic_sibling_shape_sums_into_inclusive_prompt_total() { - // Anthropic: `input_tokens` is the BASE (non-cached, non-creation) — - // `cache_read_input_tokens` and `cache_creation_input_tokens` are siblings. - // The extractor must sum the three to produce the inclusive total. - let body = json!({ - "usage": { - "input_tokens": 400, - "output_tokens": 100, - "cache_read_input_tokens": 100, - "cache_creation_input_tokens": 50, - } - }); - assert_eq!( - usage_from_body(&body), - TokenUsage { - prompt_tokens: 550, - completion_tokens: 100, - cached_tokens: 100, - cache_creation_tokens: 50, - reasoning_tokens: 0, - cacheable_prompt_tokens: 0, - } - ); -} - -#[test] -fn openai_and_anthropic_shapes_produce_identical_canonical_output() { - // Same logical request expressed in both shapes — extractor output - // must be byte-for-byte identical so cost math doesn't drift between - // providers. - let openai_body = json!({ - "usage": { - "prompt_tokens": 550, - "completion_tokens": 100, - "prompt_tokens_details": { - "cached_tokens": 100, - "cache_creation_tokens": 50, - }, - } - }); - let anthropic_body = json!({ - "usage": { - "input_tokens": 400, - "output_tokens": 100, - "cache_read_input_tokens": 100, - "cache_creation_input_tokens": 50, - } - }); - assert_eq!( - usage_from_body(&openai_body), - usage_from_body(&anthropic_body) - ); -} - -#[test] -fn anthropic_without_cache_fields_keeps_prompt_equal_to_input() { - // No cache fields: prompt_tokens degenerates to input_tokens. - let body = json!({ - "usage": { - "input_tokens": 200, - "output_tokens": 50, - } - }); - assert_eq!( - usage_from_body(&body), - TokenUsage { - prompt_tokens: 200, - completion_tokens: 50, - cached_tokens: 0, - cache_creation_tokens: 0, - reasoning_tokens: 0, - cacheable_prompt_tokens: 0, - } - ); -} - -#[test] -fn openai_without_prompt_tokens_details_keeps_cache_counts_zero() { - let body = json!({ - "usage": { - "prompt_tokens": 200, - "completion_tokens": 50, - } - }); - assert_eq!( - usage_from_body(&body), - TokenUsage { - prompt_tokens: 200, - completion_tokens: 50, - cached_tokens: 0, - cache_creation_tokens: 0, - reasoning_tokens: 0, - cacheable_prompt_tokens: 0, - } - ); -} - -#[test] -fn openai_reasoning_tokens_extracted_from_completion_tokens_details() { - let body = json!({ - "usage": { - "prompt_tokens": 100, - "completion_tokens": 200, - "completion_tokens_details": { - "reasoning_tokens": 150, - }, - } - }); - let usage = usage_from_body(&body); - assert_eq!(usage.reasoning_tokens, 150); - assert_eq!(usage.completion_tokens, 200); -} - -#[test] -fn anthropic_reasoning_tokens_extracted_from_output_tokens_details() { - let body = json!({ - "usage": { - "input_tokens": 100, - "output_tokens": 200, - "output_tokens_details": { - "reasoning_tokens": 150, - }, - } - }); - let usage = usage_from_body(&body); - assert_eq!(usage.reasoning_tokens, 150); - assert_eq!(usage.completion_tokens, 200); -} - -#[test] -fn missing_usage_block_yields_zero_usage() { - let body = json!({"id": "msg-1", "content": "hi"}); - assert!(usage_from_body(&body).is_zero()); -} - -#[test] -fn non_object_usage_block_is_ignored() { - // Defensive: a malformed `usage: "garbage"` must not panic; extractor - // returns default rather than half-populating fields. - let body = json!({"usage": "garbage"}); - assert!(usage_from_body(&body).is_zero()); -} - -#[test] -fn anthropic_with_input_tokens_details_cached_takes_precedence_over_cache_read() { - // Defensive corner: when both `input_tokens_details.cached_tokens` and - // `cache_read_input_tokens` are present, the details object wins so - // providers that surface both naming conventions don't double-count. - let body = json!({ - "usage": { - "input_tokens": 100, - "output_tokens": 50, - "cache_read_input_tokens": 30, - "input_tokens_details": {"cached_tokens": 25}, - } - }); - let usage = usage_from_body(&body); - assert_eq!(usage.cached_tokens, 25); -} - -#[test] -fn cost_estimator_invariant_no_negative_base_input() { - // The cost estimator computes - // base = prompt_tokens - cached_tokens - cache_creation_tokens - // and relies on this being non-negative. Both shapes must respect that - // — otherwise the base-input cost line silently clamps to 0 and we - // under-bill the cache-write cost. - let cases = [ - json!({"usage": { - "prompt_tokens": 550, - "prompt_tokens_details": {"cached_tokens": 100, "cache_creation_tokens": 50}, - }}), - json!({"usage": { - "input_tokens": 400, - "cache_read_input_tokens": 100, - "cache_creation_input_tokens": 50, - }}), - ]; - for body in cases { - let usage = usage_from_body(&body); - let base = usage - .prompt_tokens - .saturating_sub(usage.cached_tokens) - .saturating_sub(usage.cache_creation_tokens); - assert_eq!( - base, 400, - "{body:?} should leave 400 non-cached prompt tokens" - ); - } -} diff --git a/crates/switchyard-components/tests/support/config.rs b/crates/switchyard-components/tests/support/config.rs deleted file mode 100644 index 404873d3b..000000000 --- a/crates/switchyard-components/tests/support/config.rs +++ /dev/null @@ -1,67 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Shared test helpers that do not depend on the removed config graph. - -use std::collections::BTreeMap; - -use switchyard_components::{ - BackendFormat, EndpointConfig, LlmTarget, LlmTargetId, ModelId, Result, SwitchyardError, -}; - -/// Test result alias that works across async and blocking integration tests. -pub type TestResult = std::result::Result<(), Box>; - -/// Asserts Switchyard validation fails with a stable, useful error fragment. -pub fn assert_invalid(result: Result, message: &'static str, expected: &str) -> TestResult { - let error = error_from(result, message); - assert!( - error.to_string().contains(expected), - "expected {error:?} to contain {expected:?}" - ); - Ok(()) -} - -/// Builds an OpenAI-compatible runtime target for component tests. -pub fn openai_target(id: &'static str, model: &'static str, base_url: &str) -> Result { - Ok(LlmTarget { - id: LlmTargetId::from_static(id), - model: ModelId::from_static(model), - format: BackendFormat::OpenAi, - endpoint: EndpointConfig { - base_url: Some(base_url.to_string()), - api_key: Some("test-key".to_string()), - timeout_secs: Some(5.0), - }, - extra_body: None, - extra_headers: BTreeMap::new(), - }) -} - -/// Builds an Anthropic-compatible runtime target for component tests. -pub fn anthropic_target( - id: &'static str, - model: &'static str, - base_url: &str, -) -> Result { - Ok(LlmTarget { - id: LlmTargetId::from_static(id), - model: ModelId::from_static(model), - format: BackendFormat::Anthropic, - endpoint: EndpointConfig { - base_url: Some(base_url.to_string()), - api_key: Some("test-key".to_string()), - timeout_secs: Some(5.0), - }, - extra_body: None, - extra_headers: BTreeMap::new(), - }) -} - -// Extracts the error from a result that is expected to fail. -fn error_from(result: Result, message: &'static str) -> SwitchyardError { - match result { - Ok(_) => panic!("{message}"), - Err(error) => error, - } -} diff --git a/crates/switchyard-components/tests/support/mod.rs b/crates/switchyard-components/tests/support/mod.rs deleted file mode 100644 index 6a0744c89..000000000 --- a/crates/switchyard-components/tests/support/mod.rs +++ /dev/null @@ -1,289 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -#![allow(dead_code)] - -//! Lightweight HTTP test servers shared by backend and config tests. - -use std::collections::BTreeMap; -use std::io::{Read, Write}; -use std::net::TcpListener; -use std::sync::mpsc::{self, Receiver, RecvTimeoutError}; -use std::thread::{self, JoinHandle}; -use std::time::Duration; - -use serde_json::Value; -use switchyard_components::{Result, SwitchyardError}; - -pub mod config; - -/// One HTTP request captured by a local mock server. -#[derive(Debug)] -pub struct CapturedRequest { - /// Request method. - pub method: String, - /// Request path including query. - pub path: String, - /// Lowercased request headers. - pub headers: BTreeMap, - /// JSON request body. - pub body: Value, -} - -impl CapturedRequest { - /// Returns a header value using case-insensitive matching. - pub fn header(&self, name: &str) -> Option<&str> { - self.headers - .get(&name.to_ascii_lowercase()) - .map(String::as_str) - } -} - -/// Mock server that accepts exactly one request. -pub struct OneShotServer { - /// Base URL clients can call. - base_url: String, - /// Captured request result from the server thread. - receiver: Receiver>, - /// Server thread handle joined when captured output is read. - handle: Option>, -} - -/// Mock server that accepts a fixed sequence of requests. -pub struct SequenceServer { - /// Base URL clients can call. - base_url: String, - /// Captured request sequence from the server thread. - receiver: Receiver>>, - /// Server thread handle joined when captured output is read. - handle: Option>, -} - -impl OneShotServer { - /// Creates a one-shot server returning JSON. - pub fn json(status: u16, body: Value) -> Result { - Self::raw(status, "application/json", body.to_string()) - } - - /// Creates a one-shot server returning raw SSE. - #[allow(dead_code)] - pub fn sse(body: impl Into) -> Result { - Self::raw(200, "text/event-stream", body.into()) - } - - /// Returns the base URL for this server. - pub fn base_url(&self) -> &str { - &self.base_url - } - - /// Waits for and returns the captured request. - pub fn captured(mut self) -> Result { - let request = self.receiver.recv_timeout(Duration::from_secs(5)); - match &request { - Ok(_) | Err(RecvTimeoutError::Disconnected) => { - if let Some(handle) = self.handle.take() - && handle.join().is_err() - { - return Err(SwitchyardError::Other("server thread panicked".to_string())); - } - } - Err(RecvTimeoutError::Timeout) => {} - } - request.map_err(|error| { - SwitchyardError::Other(format!("server should capture one request: {error}")) - })? - } - - /// Creates a one-shot server returning an arbitrary content type and body. - fn raw(status: u16, content_type: &'static str, body: String) -> Result { - let listener = TcpListener::bind("127.0.0.1:0") - .map_err(|error| SwitchyardError::Other(format!("bind test server: {error}")))?; - let address = listener.local_addr().map_err(|error| { - SwitchyardError::Other(format!("read test server address: {error}")) - })?; - let base_url = format!("http://{address}"); - let (sender, receiver) = mpsc::channel(); - let handle = thread::spawn(move || { - let result = handle_one_request(listener, status, content_type, body); - let _ignored = sender.send(result); - }); - - Ok(Self { - base_url, - receiver, - handle: Some(handle), - }) - } -} - -impl SequenceServer { - /// Creates a sequence server returning one JSON response per request. - pub fn json(responses: Vec<(u16, Value)>) -> Result { - let listener = TcpListener::bind("127.0.0.1:0") - .map_err(|error| SwitchyardError::Other(format!("bind test server: {error}")))?; - let address = listener.local_addr().map_err(|error| { - SwitchyardError::Other(format!("read test server address: {error}")) - })?; - let base_url = format!("http://{address}"); - let (sender, receiver) = mpsc::channel(); - let handle = thread::spawn(move || { - let result = handle_request_sequence(listener, responses); - let _ignored = sender.send(result); - }); - - Ok(Self { - base_url, - receiver, - handle: Some(handle), - }) - } - - /// Returns the base URL for this server. - pub fn base_url(&self) -> &str { - &self.base_url - } - - /// Waits for and returns every captured request. - pub fn captured(mut self) -> Result> { - let requests = self.receiver.recv_timeout(Duration::from_secs(5)); - match &requests { - Ok(_) | Err(RecvTimeoutError::Disconnected) => { - if let Some(handle) = self.handle.take() - && handle.join().is_err() - { - return Err(SwitchyardError::Other("server thread panicked".to_string())); - } - } - Err(RecvTimeoutError::Timeout) => {} - } - requests.map_err(|error| { - SwitchyardError::Other(format!("server should capture requests: {error}")) - })? - } -} - -/// Handles one HTTP request and writes a fixed response. -fn handle_one_request( - listener: TcpListener, - status: u16, - content_type: &'static str, - body: String, -) -> Result { - let (mut stream, _) = listener - .accept() - .map_err(|error| SwitchyardError::Other(format!("accept one request: {error}")))?; - let request = read_request(&mut stream)?; - - let response = format!( - "HTTP/1.1 {status} OK\r\ncontent-type: {content_type}\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", - body.len() - ); - stream - .write_all(response.as_bytes()) - .map_err(|error| SwitchyardError::Other(format!("write test response: {error}")))?; - Ok(request) -} - -/// Handles a fixed sequence of HTTP requests with matching JSON responses. -fn handle_request_sequence( - listener: TcpListener, - responses: Vec<(u16, Value)>, -) -> Result> { - let mut requests = Vec::with_capacity(responses.len()); - for (status, body) in responses { - requests.push(handle_one_request( - listener - .try_clone() - .map_err(|error| SwitchyardError::Other(format!("clone test listener: {error}")))?, - status, - "application/json", - body.to_string(), - )?); - } - Ok(requests) -} - -/// Reads one HTTP/1.1 request from a blocking test socket. -fn read_request(stream: &mut std::net::TcpStream) -> Result { - let mut bytes = Vec::new(); - let mut chunk = [0_u8; 4096]; - loop { - let read = stream - .read(&mut chunk) - .map_err(|error| SwitchyardError::Other(format!("read request: {error}")))?; - if read == 0 { - break; - } - bytes.extend_from_slice(&chunk[..read]); - if let Some((header_end, content_length)) = request_shape(&bytes) { - let body_start = header_end + 4; - if bytes.len() >= body_start + content_length { - break; - } - } - } - - let (header_end, content_length) = request_shape(&bytes) - .ok_or_else(|| SwitchyardError::Other("request should contain HTTP headers".to_string()))?; - let header_text = std::str::from_utf8(&bytes[..header_end]).map_err(|error| { - SwitchyardError::Other(format!("headers should be valid UTF-8: {error}")) - })?; - let body_start = header_end + 4; - let body_end = body_start + content_length; - let raw_body = std::str::from_utf8(&bytes[body_start..body_end]) - .map_err(|error| SwitchyardError::Other(format!("body should be valid UTF-8: {error}")))? - .to_string(); - - let mut lines = header_text.lines(); - let start_line = lines - .next() - .ok_or_else(|| SwitchyardError::Other("request line".to_string()))?; - let mut start_parts = start_line.split_whitespace(); - let method = start_parts.next().unwrap_or_default().to_string(); - let path = start_parts.next().unwrap_or_default().to_string(); - - let mut headers = BTreeMap::new(); - for line in lines { - let Some((name, value)) = line.split_once(':') else { - continue; - }; - headers.insert(name.to_ascii_lowercase(), value.trim().to_string()); - } - - let body = if raw_body.is_empty() { - Value::Null - } else { - serde_json::from_str(&raw_body).map_err(|error| { - SwitchyardError::Other(format!("request body should be JSON: {error}")) - })? - }; - - Ok(CapturedRequest { - method, - path, - headers, - body, - }) -} - -/// Returns the header/body split and content length once headers are complete. -fn request_shape(bytes: &[u8]) -> Option<(usize, usize)> { - let header_end = find_bytes(bytes, b"\r\n\r\n")?; - let headers = std::str::from_utf8(&bytes[..header_end]).ok()?; - let content_length = headers - .lines() - .filter_map(|line| line.split_once(':')) - .find_map(|(name, value)| { - name.eq_ignore_ascii_case("content-length") - .then(|| value.trim().parse::().ok()) - .flatten() - }) - .unwrap_or(0); - Some((header_end, content_length)) -} - -/// Finds a byte sequence without pulling in memchr for tests. -fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option { - haystack - .windows(needle.len()) - .position(|window| window == needle) -} diff --git a/crates/switchyard-py/Cargo.toml b/crates/switchyard-py/Cargo.toml index ceaa7cf7f..b2b0b0c22 100644 --- a/crates/switchyard-py/Cargo.toml +++ b/crates/switchyard-py/Cargo.toml @@ -17,9 +17,7 @@ crate-type = ["cdylib", "rlib"] [dependencies] async-trait.workspace = true -futures-util.workspace = true http.workspace = true -parking_lot.workspace = true switchyard-libsy.workspace = true switchyard-llm-client.workspace = true pyo3 = { version = "0.28.3", features = ["abi3-py312", "extension-module"] } @@ -29,7 +27,4 @@ serde.workspace = true serde_json.workspace = true switchyard-protocol.workspace = true switchyard-server.workspace = true -switchyard-components.workspace = true -switchyard-translation.workspace = true tokio.workspace = true -tracing.workspace = true diff --git a/crates/switchyard-py/src/component_bindings.rs b/crates/switchyard-py/src/component_bindings.rs deleted file mode 100644 index 43a672805..000000000 --- a/crates/switchyard-py/src/component_bindings.rs +++ /dev/null @@ -1,25 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! PyO3 bindings for concrete `switchyard-components` implementations. - -use pyo3::prelude::*; - -mod backends; -pub(crate) mod config; -mod dimension_collector; -mod request_processors; -mod response_processors; -mod stage_router; -pub(crate) mod stats; - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - config::register(module)?; - stats::register(module)?; - request_processors::register(module)?; - response_processors::register(module)?; - backends::register(module)?; - dimension_collector::register(module)?; - stage_router::register(module)?; - Ok(()) -} diff --git a/crates/switchyard-py/src/component_bindings/backends.rs b/crates/switchyard-py/src/component_bindings/backends.rs deleted file mode 100644 index bf3285ed9..000000000 --- a/crates/switchyard-py/src/component_bindings/backends.rs +++ /dev/null @@ -1,350 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Python bindings for concrete LLM backends. - -use std::sync::Arc; - -use pyo3::exceptions::{PyTypeError, PyValueError}; -use pyo3::prelude::*; -use pyo3::types::{PyIterator, PyTuple}; -use switchyard_components::{ - AnthropicNativeBackend, LlmTargetBackend, MultiLlmBackend, OpenAiNativeBackend, - OpenAiPassthroughBackend, StatsLlmBackend, -}; -use switchyard_components::{ChatRequestType, EndpointConfig, LlmBackend, LlmTargetId}; - -use super::config::{PyEndpointConfig, PyLlmTarget, endpoint_config_from_python}; -use super::stats::PyStatsAccumulator; -use crate::errors::py_core_error; -use crate::interop::request::request_type_from_python; -use crate::interop::roles::PyLlmBackend; - -#[pyclass(name = "LlmTargetBackend", frozen, skip_from_py_object)] -#[derive(Clone, Debug)] -pub(crate) struct PyLlmTargetBackend { - inner: LlmTargetBackend, -} - -impl PyLlmTargetBackend { - fn clone_core(&self) -> LlmTargetBackend { - self.inner.clone() - } -} - -#[pymethods] -impl PyLlmTargetBackend { - #[new] - fn py_new(target: PyRef<'_, PyLlmTarget>, backend: PyRef<'_, PyLlmBackend>) -> PyResult { - Ok(Self { - inner: LlmTargetBackend::new( - target.clone_core(), - native_backend(&backend, "LlmTargetBackend")?, - ), - }) - } - - #[getter] - fn target(&self) -> PyLlmTarget { - PyLlmTarget::from_core(self.inner.target().clone()) - } - - fn __repr__(&self) -> String { - format!( - "LlmTargetBackend(target_id={:?}, model={:?})", - self.inner.target().id.as_str(), - self.inner.target().model.as_str(), - ) - } -} - -#[pyclass( - name = "OpenAiNativeBackend", - extends = PyLlmBackend, - skip_from_py_object -)] -#[derive(Clone, Debug)] -pub(crate) struct PyOpenAiNativeBackend { - inner: Arc, -} - -#[pymethods] -impl PyOpenAiNativeBackend { - #[new] - fn py_new(target: PyRef<'_, PyLlmTarget>) -> PyResult> { - let backend = - Arc::new(OpenAiNativeBackend::new(target.clone_core()).map_err(py_core_error)?); - let base: Arc = backend.clone(); - Ok(PyClassInitializer::from(PyLlmBackend::from_native(base)) - .add_subclass(Self { inner: backend })) - } - - #[getter] - fn target(&self) -> PyLlmTarget { - PyLlmTarget::from_core(self.inner.target().clone()) - } - - fn __repr__(&self) -> String { - format!( - "OpenAiNativeBackend(target_id={:?}, model={:?})", - self.inner.target().id.as_str(), - self.inner.target().model.as_str(), - ) - } -} - -#[pyclass( - name = "OpenAiPassthroughBackend", - extends = PyLlmBackend, - skip_from_py_object -)] -#[derive(Clone, Debug)] -pub(crate) struct PyOpenAiPassthroughBackend { - inner: Arc, -} - -#[pymethods] -impl PyOpenAiPassthroughBackend { - #[new] - #[pyo3(signature = (endpoint=None, api_key=None, base_url=None, timeout_secs=None, timeout=None))] - fn py_new( - endpoint: Option<&Bound<'_, PyAny>>, - api_key: Option, - base_url: Option, - timeout_secs: Option, - timeout: Option, - ) -> PyResult> { - let mut endpoint_config = endpoint_config_from_python(endpoint)?; - if api_key.is_some() { - endpoint_config.api_key = api_key; - } - if base_url.is_some() { - endpoint_config.base_url = base_url; - } - if timeout_secs.is_some() { - endpoint_config.timeout_secs = timeout_secs; - } - if timeout.is_some() { - endpoint_config.timeout_secs = timeout; - } - - let backend = - Arc::new(OpenAiPassthroughBackend::new(endpoint_config).map_err(py_core_error)?); - let base: Arc = backend.clone(); - Ok(PyClassInitializer::from(PyLlmBackend::from_native(base)) - .add_subclass(Self { inner: backend })) - } - - #[getter] - fn endpoint(&self) -> PyEndpointConfig { - PyEndpointConfig::from_core(self.inner.endpoint().clone()) - } - - fn __repr__(&self) -> String { - let endpoint: &EndpointConfig = self.inner.endpoint(); - format!( - "OpenAiPassthroughBackend(base_url={:?}, timeout_secs={:?})", - endpoint.base_url, endpoint.timeout_secs, - ) - } -} - -#[pyclass( - name = "AnthropicNativeBackend", - extends = PyLlmBackend, - skip_from_py_object -)] -#[derive(Clone, Debug)] -pub(crate) struct PyAnthropicNativeBackend { - inner: Arc, -} - -#[pymethods] -impl PyAnthropicNativeBackend { - #[new] - fn py_new(target: PyRef<'_, PyLlmTarget>) -> PyResult> { - let backend = AnthropicNativeBackend::new(target.clone_core()).map_err(py_core_error)?; - let backend = Arc::new(backend); - let base: Arc = backend.clone(); - Ok(PyClassInitializer::from(PyLlmBackend::from_native(base)) - .add_subclass(Self { inner: backend })) - } - - #[getter] - fn target(&self) -> PyLlmTarget { - PyLlmTarget::from_core(self.inner.target().clone()) - } - - fn __repr__(&self) -> String { - format!( - "AnthropicNativeBackend(target_id={:?}, model={:?})", - self.inner.target().id.as_str(), - self.inner.target().model.as_str(), - ) - } -} - -#[pyclass( - name = "MultiLlmBackend", - extends = PyLlmBackend, - skip_from_py_object -)] -#[derive(Clone, Debug)] -pub(crate) struct PyMultiLlmBackend { - inner: Arc, -} - -#[pymethods] -impl PyMultiLlmBackend { - #[new] - #[pyo3(signature = (targets, supported_request_types=None, default_target_id=None))] - fn py_new( - targets: &Bound<'_, PyAny>, - supported_request_types: Option<&Bound<'_, PyAny>>, - default_target_id: Option, - ) -> PyResult> { - let targets = target_backends_from_python(targets)?; - let mut backend = MultiLlmBackend::new(targets).map_err(py_core_error)?; - if let Some(supported_request_types) = request_types_from_python(supported_request_types)? { - backend = backend - .with_supported_request_types(supported_request_types) - .map_err(py_core_error)?; - } - if let Some(default_target_id) = default_target_id { - let default_target_id = LlmTargetId::new(default_target_id).map_err(|error| { - PyValueError::new_err(format!("invalid default target id: {error}")) - })?; - backend = backend - .with_default_target(default_target_id) - .map_err(py_core_error)?; - } - let backend = Arc::new(backend); - let base: Arc = backend.clone(); - Ok(PyClassInitializer::from(PyLlmBackend::from_native(base)) - .add_subclass(Self { inner: backend })) - } - - fn target_ids(&self) -> Vec { - self.inner - .targets() - .iter() - .map(|target| target.target().id.as_str().to_string()) - .collect() - } - - fn default_target_id(&self) -> Option { - self.inner - .default_target_id() - .map(|target_id| target_id.as_str().to_string()) - } - - fn __repr__(&self) -> String { - format!( - "MultiLlmBackend(target_ids={:?}, default_target_id={:?})", - self.target_ids(), - self.default_target_id(), - ) - } -} - -#[pyclass( - name = "StatsLlmBackend", - extends = PyLlmBackend, - skip_from_py_object -)] -#[derive(Clone, Debug)] -pub(crate) struct PyStatsLlmBackend { - accumulator: PyStatsAccumulator, -} - -#[pymethods] -impl PyStatsLlmBackend { - #[new] - fn py_new( - inner: PyRef<'_, PyLlmBackend>, - accumulator: PyRef<'_, PyStatsAccumulator>, - ) -> PyResult> { - let accumulator = PyStatsAccumulator::from_core(accumulator.clone_core()); - let backend: Arc = Arc::new(StatsLlmBackend::new( - native_backend(&inner, "StatsLlmBackend")?, - accumulator.clone_core(), - )); - Ok(PyClassInitializer::from(PyLlmBackend::from_native(backend)) - .add_subclass(Self { accumulator })) - } - - #[getter] - fn accumulator(&self) -> PyStatsAccumulator { - self.accumulator.clone() - } - - fn __repr__(&self) -> &'static str { - "StatsLlmBackend()" - } -} - -fn native_backend(backend: &PyRef<'_, PyLlmBackend>, owner: &str) -> PyResult> { - backend.native().ok_or_else(|| { - PyTypeError::new_err(format!( - "{owner} requires a Rust-native LLMBackend binding, not a Python-only subclass" - )) - }) -} - -fn target_backends_from_python(value: &Bound<'_, PyAny>) -> PyResult> { - let iterator = PyIterator::from_object(value)?; - let mut targets = Vec::new(); - for item in iterator { - let item = item?; - targets.push(target_backend_from_python(&item)?); - } - Ok(targets) -} - -fn target_backend_from_python(value: &Bound<'_, PyAny>) -> PyResult { - if let Ok(target_backend) = value.extract::>() { - return Ok(target_backend.clone_core()); - } - - let tuple = value.cast::().map_err(|_| { - PyTypeError::new_err( - "MultiLlmBackend targets must be LlmTargetBackend objects or (target, backend) tuples", - ) - })?; - if tuple.len() != 2 { - return Err(PyValueError::new_err( - "MultiLlmBackend target tuples must contain exactly (target, backend)", - )); - } - - let target = tuple.get_item(0)?.extract::>()?; - let backend = tuple.get_item(1)?.extract::>()?; - Ok(LlmTargetBackend::new( - target.clone_core(), - native_backend(&backend, "MultiLlmBackend")?, - )) -} - -fn request_types_from_python( - value: Option<&Bound<'_, PyAny>>, -) -> PyResult>> { - let Some(value) = value.filter(|value| !value.is_none()) else { - return Ok(None); - }; - let iterator = PyIterator::from_object(value)?; - let mut request_types = Vec::new(); - for item in iterator { - request_types.push(request_type_from_python(&item?)?); - } - Ok(Some(request_types)) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - Ok(()) -} diff --git a/crates/switchyard-py/src/component_bindings/config.rs b/crates/switchyard-py/src/component_bindings/config.rs deleted file mode 100644 index 7ce3644e5..000000000 --- a/crates/switchyard-py/src/component_bindings/config.rs +++ /dev/null @@ -1,533 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Python bindings for component configuration values. - -use pyo3::class::basic::CompareOp; -use pyo3::exceptions::PyValueError; -use pyo3::prelude::*; -use pyo3::types::PyBool; -use serde::Serialize; -use switchyard_components::{ - BackendFormat, EndpointConfig, LlmTarget, LlmTargetId, ModelId, RandomRoutingProcessorConfig, -}; - -use crate::errors::py_core_error; -use crate::py_serde::{value_from_python, value_to_python}; - -#[pyclass(name = "BackendFormat", frozen, skip_from_py_object)] -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) struct PyBackendFormat { - inner: BackendFormat, -} - -impl PyBackendFormat { - const fn new_inner(inner: BackendFormat) -> Self { - Self { inner } - } -} - -#[pymethods] -impl PyBackendFormat { - #[new] - #[pyo3(signature = (value="auto"))] - fn py_new(value: &str) -> PyResult { - Ok(Self { - inner: backend_format_from_str(value)?, - }) - } - - #[classattr] - const AUTO: Self = Self::new_inner(BackendFormat::Auto); - - #[classattr] - const OPENAI: Self = Self::new_inner(BackendFormat::OpenAi); - - #[classattr] - const RESPONSES: Self = Self::new_inner(BackendFormat::Responses); - - #[classattr] - const ANTHROPIC: Self = Self::new_inner(BackendFormat::Anthropic); - - #[getter] - fn value(&self) -> &'static str { - backend_format_name(self.inner) - } - - fn __repr__(&self) -> String { - format!("BackendFormat.{}", backend_format_variant_name(self.inner)) - } - - fn __str__(&self) -> &'static str { - backend_format_name(self.inner) - } - - fn __hash__(&self) -> isize { - match self.inner { - BackendFormat::Auto => 1, - BackendFormat::OpenAi => 2, - BackendFormat::Responses => 3, - BackendFormat::Anthropic => 4, - } - } - - fn __richcmp__( - &self, - py: Python<'_>, - other: &Bound<'_, PyAny>, - op: CompareOp, - ) -> PyResult> { - match op { - CompareOp::Eq | CompareOp::Ne => { - let equals = match backend_format_from_python(Some(other)) { - Ok(other) => self.inner == other, - Err(_) => false, - }; - let result = if matches!(op, CompareOp::Eq) { - equals - } else { - !equals - }; - Ok(PyBool::new(py, result).to_owned().unbind().into_any()) - } - _ => Ok(py.NotImplemented()), - } - } -} - -#[pyclass(name = "EndpointConfig", frozen, skip_from_py_object)] -#[derive(Clone, Debug)] -pub(crate) struct PyEndpointConfig { - inner: EndpointConfig, -} - -impl PyEndpointConfig { - pub(crate) fn from_core(inner: EndpointConfig) -> Self { - Self { inner } - } - - pub(crate) fn clone_core(&self) -> EndpointConfig { - self.inner.clone() - } -} - -#[pymethods] -impl PyEndpointConfig { - #[new] - #[pyo3(signature = (base_url=None, api_key=None, timeout_secs=None))] - fn py_new( - base_url: Option, - api_key: Option, - timeout_secs: Option, - ) -> Self { - Self { - inner: EndpointConfig { - base_url, - api_key, - timeout_secs, - }, - } - } - - #[getter] - fn base_url(&self) -> Option { - self.inner.base_url.clone() - } - - #[getter] - fn api_key(&self) -> Option { - self.inner.api_key.clone() - } - - #[getter] - fn timeout_secs(&self) -> Option { - self.inner.timeout_secs - } - - fn to_dict(&self, py: Python<'_>) -> PyResult> { - to_python(py, &self.inner) - } - - fn __repr__(&self) -> String { - format!( - "EndpointConfig(base_url={:?}, api_key={}, timeout_secs={:?})", - self.inner.base_url, - if self.inner.api_key.is_some() { - "''" - } else { - "None" - }, - self.inner.timeout_secs, - ) - } -} - -#[pyclass(name = "LlmTarget", frozen, skip_from_py_object)] -#[derive(Clone, Debug)] -pub(crate) struct PyLlmTarget { - inner: LlmTarget, -} - -impl PyLlmTarget { - pub(crate) fn from_core(inner: LlmTarget) -> Self { - Self { inner } - } - - pub(crate) fn clone_core(&self) -> LlmTarget { - self.inner.clone() - } -} - -#[pymethods] -impl PyLlmTarget { - #[new] - #[allow(clippy::too_many_arguments)] - #[pyo3(signature = ( - id=None, - model=None, - format=None, - backend_format=None, - endpoint=None, - base_url=None, - api_key=None, - timeout_secs=None, - timeout=None, - extra_body=None, - extra_headers=None, - ))] - fn py_new( - id: Option, - model: Option, - format: Option<&Bound<'_, PyAny>>, - backend_format: Option<&Bound<'_, PyAny>>, - endpoint: Option<&Bound<'_, PyAny>>, - base_url: Option, - api_key: Option, - timeout_secs: Option, - timeout: Option, - extra_body: Option<&Bound<'_, PyAny>>, - extra_headers: Option<&Bound<'_, PyAny>>, - ) -> PyResult { - let mut endpoint = endpoint_config_from_python(endpoint)?; - if base_url.is_some() { - endpoint.base_url = base_url; - } - if api_key.is_some() { - endpoint.api_key = api_key; - } - if timeout_secs.is_some() { - endpoint.timeout_secs = timeout_secs; - } - if timeout.is_some() { - endpoint.timeout_secs = timeout; - } - - let (id, model) = match (id, model) { - (Some(id), Some(model)) => (id, model), - (None, Some(model)) => ("default".to_string(), model), - (Some(_), None) => { - return Err(PyValueError::new_err( - "LlmTarget requires a model string when id is provided", - )); - } - (None, None) => return Err(PyValueError::new_err("LlmTarget requires a model string")), - }; - - let extra_body = match extra_body { - None => None, - Some(value) if value.is_none() => None, - Some(value) => Some(value_from_python(value).map_err(|error| { - PyValueError::new_err(format!( - "LlmTarget.extra_body must be JSON-serialisable: {error}" - )) - })?), - }; - - let extra_headers = match extra_headers { - None => std::collections::BTreeMap::new(), - Some(value) if value.is_none() => std::collections::BTreeMap::new(), - Some(value) => { - let raw = value_from_python(value).map_err(|error| { - PyValueError::new_err(format!( - "LlmTarget.extra_headers must be a JSON-serialisable mapping: {error}" - )) - })?; - let serde_json::Value::Object(map) = raw else { - return Err(PyValueError::new_err( - "LlmTarget.extra_headers must be a dict of str -> str", - )); - }; - let mut out = std::collections::BTreeMap::new(); - for (k, v) in map { - let serde_json::Value::String(s) = v else { - return Err(PyValueError::new_err(format!( - "LlmTarget.extra_headers[{:?}] must be a string, got {}", - k, v - ))); - }; - out.insert(k, s); - } - out - } - }; - - Ok(Self { - inner: LlmTarget { - id: LlmTargetId::new(id).map_err(|error| { - PyValueError::new_err(format!("invalid LLM target id: {error}")) - })?, - model: ModelId::new(model) - .map_err(|error| PyValueError::new_err(format!("invalid model id: {error}")))?, - format: backend_format_from_python(format.or(backend_format))?, - endpoint, - extra_body, - extra_headers, - }, - }) - } - - #[getter] - fn id(&self) -> String { - self.inner.id.as_str().to_string() - } - - #[getter] - fn model(&self) -> String { - self.inner.model.as_str().to_string() - } - - #[getter] - fn format(&self, py: Python<'_>) -> PyResult> { - backend_format_object(py, self.inner.format) - } - - #[getter] - fn backend_format(&self, py: Python<'_>) -> PyResult> { - backend_format_object(py, self.inner.format) - } - - #[getter] - fn endpoint(&self) -> PyEndpointConfig { - PyEndpointConfig { - inner: self.inner.endpoint.clone(), - } - } - - #[getter] - fn base_url(&self) -> Option { - self.inner.endpoint.base_url.clone() - } - - #[getter] - fn api_key(&self) -> Option { - self.inner.endpoint.api_key.clone() - } - - #[getter] - fn timeout(&self) -> Option { - self.inner.endpoint.timeout_secs - } - - #[getter] - fn extra_body(&self, py: Python<'_>) -> PyResult> { - match &self.inner.extra_body { - None => Ok(py.None()), - Some(value) => value_to_python(py, value), - } - } - - #[getter] - fn extra_headers(&self, py: Python<'_>) -> PyResult> { - let map: serde_json::Map = self - .inner - .extra_headers - .iter() - .map(|(k, v)| (k.clone(), serde_json::Value::String(v.clone()))) - .collect(); - value_to_python(py, &serde_json::Value::Object(map)) - } - - fn to_dict(&self, py: Python<'_>) -> PyResult> { - to_python(py, &self.inner) - } - - fn model_dump(&self, py: Python<'_>) -> PyResult> { - self.to_dict(py) - } - - fn __richcmp__( - &self, - py: Python<'_>, - other: &Bound<'_, PyAny>, - op: CompareOp, - ) -> PyResult> { - match op { - CompareOp::Eq | CompareOp::Ne => { - let equals = if let Ok(other) = other.extract::>() { - self.inner == other.inner - } else if let Ok(other) = value_from_python(other).and_then(|value| { - serde_json::from_value::(value) - .map_err(|error| PyValueError::new_err(error.to_string())) - }) { - self.inner == other - } else { - false - }; - let result = if matches!(op, CompareOp::Eq) { - equals - } else { - !equals - }; - Ok(PyBool::new(py, result).to_owned().unbind().into_any()) - } - _ => Ok(py.NotImplemented()), - } - } - - fn __repr__(&self) -> String { - format!( - "LlmTarget(id={:?}, model={:?}, format='{}')", - self.inner.id.as_str(), - self.inner.model.as_str(), - backend_format_name(self.inner.format), - ) - } -} - -#[pyclass(name = "RandomRoutingProcessorConfig", frozen, skip_from_py_object)] -#[derive(Clone, Debug)] -pub(crate) struct PyRandomRoutingProcessorConfig { - inner: RandomRoutingProcessorConfig, -} - -#[pymethods] -impl PyRandomRoutingProcessorConfig { - #[new] - #[pyo3(signature = (strong, weak, strong_probability=0.5, rng_seed=None))] - fn py_new( - strong: PyRef<'_, PyLlmTarget>, - weak: PyRef<'_, PyLlmTarget>, - strong_probability: f64, - rng_seed: Option, - ) -> PyResult { - let config = RandomRoutingProcessorConfig::new(strong.clone_core(), weak.clone_core()) - .with_strong_probability(strong_probability) - .map_err(py_core_error)? - .with_rng_seed(rng_seed); - Ok(Self { inner: config }) - } - - #[getter] - fn strong(&self) -> PyLlmTarget { - PyLlmTarget::from_core(self.inner.strong.clone()) - } - - #[getter] - fn weak(&self) -> PyLlmTarget { - PyLlmTarget::from_core(self.inner.weak.clone()) - } - - #[getter] - fn strong_probability(&self) -> f64 { - self.inner.strong_probability - } - - #[getter] - fn rng_seed(&self) -> Option { - self.inner.rng_seed - } - - fn to_dict(&self, py: Python<'_>) -> PyResult> { - to_python(py, &self.inner) - } - - fn __repr__(&self) -> String { - format!( - "RandomRoutingProcessorConfig(strong={}, weak={}, strong_probability={}, rng_seed={:?})", - self.inner.strong.model, - self.inner.weak.model, - self.inner.strong_probability, - self.inner.rng_seed, - ) - } -} - -pub(crate) fn backend_format_from_python( - value: Option<&Bound<'_, PyAny>>, -) -> PyResult { - let Some(value) = value.filter(|value| !value.is_none()) else { - return Ok(BackendFormat::Auto); - }; - if let Ok(format) = value.extract::>() { - return Ok(format.inner); - } - let raw = if let Ok(value_attr) = value.getattr("value") { - value_attr.extract::()? - } else { - value.extract::()? - }; - backend_format_from_str(&raw) -} - -fn backend_format_from_str(value: &str) -> PyResult { - match value { - "auto" => Ok(BackendFormat::Auto), - "openai" => Ok(BackendFormat::OpenAi), - "responses" => Ok(BackendFormat::Responses), - "anthropic" => Ok(BackendFormat::Anthropic), - _ => Err(PyValueError::new_err(format!( - "Unknown backend format: {value:?}" - ))), - } -} - -fn backend_format_name(format: BackendFormat) -> &'static str { - match format { - BackendFormat::Auto => "auto", - BackendFormat::OpenAi => "openai", - BackendFormat::Responses => "responses", - BackendFormat::Anthropic => "anthropic", - } -} - -fn backend_format_variant_name(format: BackendFormat) -> &'static str { - match format { - BackendFormat::Auto => "AUTO", - BackendFormat::OpenAi => "OPENAI", - BackendFormat::Responses => "RESPONSES", - BackendFormat::Anthropic => "ANTHROPIC", - } -} - -fn backend_format_object(py: Python<'_>, format: BackendFormat) -> PyResult> { - py.get_type::() - .getattr(backend_format_variant_name(format)) - .map(Bound::unbind) -} - -pub(crate) fn endpoint_config_from_python( - value: Option<&Bound<'_, PyAny>>, -) -> PyResult { - let Some(value) = value.filter(|value| !value.is_none()) else { - return Ok(EndpointConfig::default()); - }; - if let Ok(endpoint) = value.extract::>() { - return Ok(endpoint.clone_core()); - } - serde_json::from_value(value_from_python(value)?) - .map_err(|error| PyValueError::new_err(error.to_string())) -} - -fn to_python(py: Python<'_>, value: &impl Serialize) -> PyResult> { - let value = - serde_json::to_value(value).map_err(|error| PyValueError::new_err(error.to_string()))?; - value_to_python(py, &value) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - Ok(()) -} diff --git a/crates/switchyard-py/src/component_bindings/dimension_collector.rs b/crates/switchyard-py/src/component_bindings/dimension_collector.rs deleted file mode 100644 index 1cc6c22b0..000000000 --- a/crates/switchyard-py/src/component_bindings/dimension_collector.rs +++ /dev/null @@ -1,398 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Python bindings for the dimension-collector context-signal layer. -//! -//! Exposes: -//! -//! * `DimensionScore` — one scorer's output (name + score + signal). -//! * `ContextSignals` — the aggregate stamped into `ProxyContext` -//! (dimensions tuple + token-count estimate + agentic scalar). -//! * `ScoringConfig` — keyword lists + token-count thresholds. -//! * `DimensionCollector` — request-side component that runs the 15 scorers -//! and stamps `ContextSignals` into the context. -//! * `get_context_signals(ctx)` — Python-facing reader so estimators -//! built on top (LLM classifier, future rules estimator) can pick up -//! the stamped signals. - -use pyo3::prelude::*; -use pyo3::types::PyList; -use switchyard_components::ChatResponse; -use switchyard_components::{ - DimensionCollector, ResponseSignalCollector, - dimension_collector::{ - DEFAULT_RECENT_WINDOW, ResponseFlag, ResponseSignals, ToolResultSignal, - extract_response_signals as core_extract_response_signals, - }, -}; - -use crate::py_serde::value_from_python; - -use crate::interop::context::{get_cloned_from_python, lease_from_python}; -use crate::interop::request::{request_from_python, request_to_python}; -use crate::interop::response::{response_from_python, response_to_python}; - -/// Request-side component that runs the dimension collector. -#[pyclass(name = "DimensionCollector", skip_from_py_object)] -#[derive(Clone, Debug)] -pub(crate) struct PyDimensionCollector { - inner: DimensionCollector, -} - -#[pymethods] -impl PyDimensionCollector { - #[new] - #[pyo3(signature = (*, recent_window = None))] - fn py_new(recent_window: Option) -> Self { - let window = recent_window.unwrap_or(DEFAULT_RECENT_WINDOW); - Self { - inner: DimensionCollector::with_recent_window(window), - } - } - - fn startup<'py>(&self, py: Python<'py>) -> PyResult> { - pyo3_async_runtimes::tokio::future_into_py(py, async { Ok(()) }) - } - - fn shutdown<'py>(&self, py: Python<'py>) -> PyResult> { - pyo3_async_runtimes::tokio::future_into_py(py, async { Ok(()) }) - } - - fn process<'py>( - &self, - py: Python<'py>, - ctx: &Bound<'_, PyAny>, - request: &Bound<'_, PyAny>, - ) -> PyResult> { - let processor = self.inner.clone(); - let mut lease = lease_from_python(ctx)?; - let request = request_from_python(request)?; - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let result = processor.process(lease.context_mut()?, request).await; - let restore_result = lease.restore(); - let request = result.map_err(crate::errors::py_core_error)?; - restore_result?; - Python::attach(|py| request_to_python(py, request)) - }) - } - - fn __repr__(&self) -> &'static str { - "DimensionCollector()" - } -} - -/// Closed set of response-side quality flags emitted by the response -/// signal collector. Mirrors Rust's -/// [`switchyard_components::dimension_collector::ResponseFlag`] enum. -/// -/// Python sees these as a class with attribute-style variants -/// (`ResponseFlag.MALFORMED_TOOL_CALL_JSON`, etc.) plus an `__eq__` -/// implementation so set / list membership checks work naturally. -#[pyclass(name = "ResponseFlag", eq, frozen, from_py_object)] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum PyResponseFlag { - MalformedToolCallJson, - EmptyResponse, - TruncatedCompletion, - MissingRequiredArgs, -} - -impl PyResponseFlag { - fn from_core(flag: ResponseFlag) -> Self { - match flag { - ResponseFlag::MalformedToolCallJson => Self::MalformedToolCallJson, - ResponseFlag::EmptyResponse => Self::EmptyResponse, - ResponseFlag::TruncatedCompletion => Self::TruncatedCompletion, - ResponseFlag::MissingRequiredArgs => Self::MissingRequiredArgs, - } - } -} - -#[pymethods] -impl PyResponseFlag { - fn __repr__(&self) -> &'static str { - match self { - Self::MalformedToolCallJson => "ResponseFlag.MALFORMED_TOOL_CALL_JSON", - Self::EmptyResponse => "ResponseFlag.EMPTY_RESPONSE", - Self::TruncatedCompletion => "ResponseFlag.TRUNCATED_COMPLETION", - Self::MissingRequiredArgs => "ResponseFlag.MISSING_REQUIRED_ARGS", - } - } - - fn __hash__(&self) -> u64 { - *self as u64 - } -} - -/// Aggregate response-side signals stamped by [`PyResponseSignalCollector`]. -#[pyclass(name = "ResponseSignals", frozen, skip_from_py_object)] -#[derive(Clone, Debug)] -pub(crate) struct PyResponseSignals { - inner: ResponseSignals, -} - -#[pymethods] -impl PyResponseSignals { - /// Failing flags, in the order the checker ran them. - #[getter] - fn flags(&self, py: Python<'_>) -> PyResult> { - let items: Vec> = self - .inner - .flags - .iter() - .map(|flag| Py::new(py, PyResponseFlag::from_core(*flag))) - .collect::>>()?; - Ok(PyList::new(py, items)?.unbind()) - } - - /// True when at least one check failed; stage_router routers use this as - /// the per-attempt acceptability gate. - fn has_failures(&self) -> bool { - self.inner.has_failures() - } - - /// Python-friendly membership check; equivalent to - /// `flag in signals.flags`. - fn contains(&self, flag: PyResponseFlag) -> bool { - self.inner - .flags - .iter() - .any(|inner| PyResponseFlag::from_core(*inner) == flag) - } - - fn __repr__(&self) -> String { - format!("ResponseSignals(flags=<{} items>)", self.inner.flags.len()) - } -} - -impl PyResponseSignals { - fn from_core(inner: ResponseSignals) -> Self { - Self { inner } - } -} - -/// Response-side component that runs response-side signal extraction. -#[pyclass(name = "ResponseSignalCollector", skip_from_py_object)] -#[derive(Clone, Debug)] -pub(crate) struct PyResponseSignalCollector; - -#[pymethods] -impl PyResponseSignalCollector { - #[new] - fn py_new() -> Self { - Self - } - - fn startup<'py>(&self, py: Python<'py>) -> PyResult> { - pyo3_async_runtimes::tokio::future_into_py(py, async { Ok(()) }) - } - - fn shutdown<'py>(&self, py: Python<'py>) -> PyResult> { - pyo3_async_runtimes::tokio::future_into_py(py, async { Ok(()) }) - } - - fn process<'py>( - &self, - py: Python<'py>, - ctx: &Bound<'_, PyAny>, - response: &Bound<'_, PyAny>, - ) -> PyResult> { - let mut lease = lease_from_python(ctx)?; - let response = response_from_python(response)?; - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let result = ResponseSignalCollector - .process(lease.context_mut()?, response) - .await; - let restore_result = lease.restore(); - let response = result.map_err(crate::errors::py_core_error)?; - restore_result?; - Python::attach(|py| response_to_python(py, response)) - }) - } - - fn __repr__(&self) -> &'static str { - "ResponseSignalCollector()" - } -} - -/// Returns the `ResponseSignals` stamped by a `ResponseSignalCollector` run. -/// -/// `None` either means the collector hasn't run yet on this `ctx` or the -/// response was a streaming response (which the buffered-body checks -/// can't introspect). -#[pyfunction] -fn get_response_signals(ctx: &Bound<'_, PyAny>) -> PyResult> { - Ok(get_cloned_from_python::(ctx)?.map(PyResponseSignals::from_core)) -} - -/// Runs the response-side checks against an inline response body dict. -/// -/// Intended for the stage-router, which needs to evaluate -/// `ResponseSignals` between attempts without going through a full -/// response-side pass. Accepts the response's `.body` Python -/// dict directly; works for any wire shape because the checks are -/// structure-based, not variant-based (`choices[...]` vs `content[...]` -/// dispatch happens inside the Rust checks themselves). -/// -/// Returns an empty `ResponseSignals` (no failures) if the body is -/// `None` or not a dict — same fail-safe posture as the -/// `ResponseSignalCollector` adapter. -#[pyfunction] -fn extract_response_signals(body: Option<&Bound<'_, PyAny>>) -> PyResult { - let Some(body) = body else { - return Ok(PyResponseSignals::from_core(ResponseSignals::default())); - }; - if body.is_none() { - return Ok(PyResponseSignals::from_core(ResponseSignals::default())); - } - let value = value_from_python(body)?; - // The four checks dispatch on body structure, not on the - // `ChatResponse` variant. Wrap in OpenAI-completion just to satisfy - // the type; Anthropic-shaped bodies still get their dedicated - // checks via the `content[]` walk. - let response = ChatResponse::openai_completion(value); - Ok(PyResponseSignals::from_core(core_extract_response_signals( - &response, - ))) -} - -/// Tool-result context signals stamped by [`PyDimensionCollector`]. -/// -/// Read via :func:`get_tool_result_signal`. All fields default to neutral -/// values (0.0 / False) when no tool results are present in the request. -#[pyclass(name = "ToolResultSignal", frozen, skip_from_py_object)] -#[derive(Clone, Debug)] -pub(crate) struct PyToolResultSignal { - inner: ToolResultSignal, -} - -#[pymethods] -impl PyToolResultSignal { - /// Max error severity across all matched patterns in the last tool result. - /// ``0.0`` = clean; ``0.3`` = soft; ``0.7`` = hard; ``1.0`` = critical. - #[getter] - fn severity(&self) -> f32 { - self.inner.severity - } - - /// Consecutive clean tool results at the end of history (``0`` if last failed). - #[getter] - fn no_error_streak(&self) -> u32 { - self.inner.no_error_streak - } - - /// Edit-type tool calls in the conversation (refinement work). - #[getter] - fn edit_count(&self) -> u32 { - self.inner.edit_count - } - - /// Write/create-type tool calls in the conversation (scaffolding work). - #[getter] - fn write_count(&self) -> u32 { - self.inner.write_count - } - - /// Read-type tool calls (Read tool + read-like Bash inspections). - #[getter] - fn read_count(&self) -> u32 { - self.inner.read_count - } - - /// TodoWrite calls — Opus struggle signal used by the strong-default drop gate. - #[getter] - fn todowrite_count(&self) -> u32 { - self.inner.todowrite_count - } - - /// Edit-type tool calls in the most recent 3 tool calls (sliding window). - #[getter] - fn recent_edit_count(&self) -> u32 { - self.inner.recent_edit_count - } - - /// Write/create-type tool calls in the most recent 3 tool calls (sliding window). - #[getter] - fn recent_write_count(&self) -> u32 { - self.inner.recent_write_count - } - - /// Read-type tool calls in the most recent 3 tool calls (sliding window). - #[getter] - fn recent_read_count(&self) -> u32 { - self.inner.recent_read_count - } - - /// TodoWrite calls in the most recent 3 tool calls (sliding window). - #[getter] - fn recent_todowrite_count(&self) -> u32 { - self.inner.recent_todowrite_count - } - - /// Consecutive trailing `Other`-category tool calls (build-pit proxy). - #[getter] - fn pure_bash_streak(&self) -> u32 { - self.inner.pure_bash_streak - } - - /// ``True`` when a recent tool result contained passing test output. - #[getter] - fn tests_passed(&self) -> bool { - self.inner.tests_passed - } - - /// Total messages in the conversation (turn-depth proxy). - #[getter] - fn turn_depth(&self) -> u32 { - self.inner.turn_depth - } - - /// The request carries a context-compaction summary — the picker forces + holds - /// the strong tier, since compaction otherwise de-escalates the router to weak. - #[getter] - fn compacted(&self) -> bool { - self.inner.compacted - } - - fn __repr__(&self) -> String { - format!( - "ToolResultSignal(severity={:.2}, streak={}, edit={}, write={}, tests_passed={})", - self.inner.severity, - self.inner.no_error_streak, - self.inner.edit_count, - self.inner.write_count, - self.inner.tests_passed, - ) - } -} - -impl PyToolResultSignal { - fn from_core(inner: ToolResultSignal) -> Self { - Self { inner } - } - - /// The underlying Rust signal — used by the stage_router picker binding. - pub(crate) fn core(&self) -> &ToolResultSignal { - &self.inner - } -} - -/// Returns the :class:`ToolResultSignal` stamped by a :class:`DimensionCollector` run. -/// -/// Returns ``None`` when the collector has not run on this context yet. -#[pyfunction] -fn get_tool_result_signal(ctx: &Bound<'_, PyAny>) -> PyResult> { - Ok(get_cloned_from_python::(ctx)?.map(PyToolResultSignal::from_core)) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_function(wrap_pyfunction!(get_response_signals, module)?)?; - module.add_function(wrap_pyfunction!(extract_response_signals, module)?)?; - module.add_function(wrap_pyfunction!(get_tool_result_signal, module)?)?; - Ok(()) -} diff --git a/crates/switchyard-py/src/component_bindings/request_processors.rs b/crates/switchyard-py/src/component_bindings/request_processors.rs deleted file mode 100644 index 19a133c3d..000000000 --- a/crates/switchyard-py/src/component_bindings/request_processors.rs +++ /dev/null @@ -1,62 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Python bindings for concrete request-side compatibility components. - -use pyo3::prelude::*; -use switchyard_components::{StatsRequestProcessor, tracking_enabled_from_env}; - -use crate::errors::py_core_error; -use crate::interop::context::lease_from_python; -use crate::interop::request::{request_from_python, request_to_python}; - -#[pyclass(name = "StatsRequestProcessor", skip_from_py_object)] -#[derive(Clone, Copy, Debug)] -pub(crate) struct PyStatsRequestProcessor { - inner: StatsRequestProcessor, -} - -#[pymethods] -impl PyStatsRequestProcessor { - #[new] - fn py_new() -> Self { - Self { - inner: StatsRequestProcessor::new(tracking_enabled_from_env()), - } - } - - fn startup<'py>(&self, py: Python<'py>) -> PyResult> { - pyo3_async_runtimes::tokio::future_into_py(py, async { Ok(()) }) - } - - fn shutdown<'py>(&self, py: Python<'py>) -> PyResult> { - pyo3_async_runtimes::tokio::future_into_py(py, async { Ok(()) }) - } - - fn process<'py>( - &self, - py: Python<'py>, - ctx: &Bound<'_, PyAny>, - request: &Bound<'_, PyAny>, - ) -> PyResult> { - let processor = self.inner; - let mut lease = lease_from_python(ctx)?; - let request = request_from_python(request)?; - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let result = processor.process(lease.context_mut()?, request).await; - let restore_result = lease.restore(); - let request = result.map_err(py_core_error)?; - restore_result?; - Python::attach(|py| request_to_python(py, request)) - }) - } - - fn __repr__(&self) -> &'static str { - "StatsRequestProcessor()" - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - Ok(()) -} diff --git a/crates/switchyard-py/src/component_bindings/response_processors.rs b/crates/switchyard-py/src/component_bindings/response_processors.rs deleted file mode 100644 index eccaf5444..000000000 --- a/crates/switchyard-py/src/component_bindings/response_processors.rs +++ /dev/null @@ -1,79 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Python bindings for concrete response-side components. - -use pyo3::prelude::*; -use switchyard_components::StatsResponseProcessor; - -use super::stats::PyStatsAccumulator; -use crate::errors::py_core_error; -use crate::interop::context::lease_from_python; -use crate::interop::response::{response_from_python, response_to_python}; - -#[pyclass(name = "StatsResponseProcessor", skip_from_py_object)] -#[derive(Clone, Debug)] -pub(crate) struct PyStatsResponseProcessor { - inner: StatsResponseProcessor, - accumulator: PyStatsAccumulator, -} - -#[pymethods] -impl PyStatsResponseProcessor { - #[new] - fn py_new(accumulator: PyRef<'_, PyStatsAccumulator>) -> Self { - let accumulator = PyStatsAccumulator::from_core(accumulator.clone_core()); - Self { - inner: StatsResponseProcessor::new(accumulator.clone_core()), - accumulator, - } - } - - #[getter] - fn accumulator(&self) -> PyStatsAccumulator { - self.accumulator.clone() - } - - fn startup<'py>(&self, py: Python<'py>) -> PyResult> { - pyo3_async_runtimes::tokio::future_into_py(py, async { Ok(()) }) - } - - fn shutdown<'py>(&self, py: Python<'py>) -> PyResult> { - pyo3_async_runtimes::tokio::future_into_py(py, async { Ok(()) }) - } - - fn process<'py>( - &self, - py: Python<'py>, - ctx: &Bound<'_, PyAny>, - response: &Bound<'_, PyAny>, - ) -> PyResult> { - let processor = self.inner.clone(); - let mut lease = lease_from_python(ctx)?; - let response = response_from_python(response)?; - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let result = processor.process(lease.context_mut()?, response).await; - let restore_result = lease.restore(); - let response = result.map_err(py_core_error)?; - restore_result?; - Python::attach(|py| response_to_python(py, response)) - }) - } - - fn get_endpoint(&self, py: Python<'_>) -> PyResult> { - let endpoint = py - .import("switchyard.lib.endpoints.stats_endpoint")? - .getattr("StatsEndpoint")? - .call1((self.accumulator.clone(),))?; - Ok(endpoint.unbind()) - } - - fn __repr__(&self) -> &'static str { - "StatsResponseProcessor()" - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - Ok(()) -} diff --git a/crates/switchyard-py/src/component_bindings/stage_router.rs b/crates/switchyard-py/src/component_bindings/stage_router.rs deleted file mode 100644 index 33a33d3a0..000000000 --- a/crates/switchyard-py/src/component_bindings/stage_router.rs +++ /dev/null @@ -1,154 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Python binding for the shared stage_router picker. -//! -//! Exposes [`switchyard_components::stage_router::pick_tier`] as -//! `stage_pick_tier(signal, picker_mode, confidence_threshold) -> PickOutcome` -//! for Python analysis tools. This returns a resolved decision or a request to -//! consult a classifier. - -use pyo3::exceptions::PyValueError; -use pyo3::prelude::*; - -use switchyard_components::stage_router::{PickOutcome, PickerMode, Tier, pick_tier, score_signal}; - -use super::dimension_collector::PyToolResultSignal; - -fn parse_picker_mode(mode: &str) -> PyResult { - match mode { - "capable_first" => Ok(PickerMode::CapableFirst), - "efficient_first" => Ok(PickerMode::EfficientFirst), - other => Err(PyValueError::new_err(format!( - "unknown picker mode {other:?} (expected capable_first or efficient_first)" - ))), - } -} - -fn tier_str(tier: Tier) -> &'static str { - match tier { - Tier::Capable => "capable", - Tier::Efficient => "efficient", - } -} - -/// Result of [`stage_pick_tier`]. -/// -/// `resolved` is `True` when the picker decided without the classifier — then -/// `tier` and `source` are set. `resolved` is `False` when the scorer was not -/// confident: the caller runs its classifier, and falls back to `default_tier` -/// if it has none or it fails. `score` / `confidence` are always the scorer's. -#[pyclass(name = "PickOutcome", frozen)] -pub(crate) struct PyPickOutcome { - resolved: bool, - tier: Option<&'static str>, - source: Option<&'static str>, - default_tier: &'static str, - score: f64, - confidence: Option, -} - -#[pymethods] -impl PyPickOutcome { - #[getter] - fn resolved(&self) -> bool { - self.resolved - } - - /// Chosen tier (`"capable"` / `"efficient"`) — only when `resolved`. - #[getter] - fn tier(&self) -> Option<&'static str> { - self.tier - } - - /// Decision source (`"override"` / `"tests_passed"` / `"dimensions"`) — only - /// when `resolved`. - #[getter] - fn source(&self) -> Option<&'static str> { - self.source - } - - /// Tier to fall open to when the classifier does not resolve the turn. - #[getter] - fn default_tier(&self) -> &'static str { - self.default_tier - } - - #[getter] - fn score(&self) -> f64 { - self.score - } - - #[getter] - fn confidence(&self) -> Option { - self.confidence - } - - fn __repr__(&self) -> String { - if self.resolved { - format!( - "PickOutcome(resolved, tier={:?}, source={:?}, score={:.3})", - self.tier, self.source, self.score - ) - } else { - format!( - "PickOutcome(consult_classifier, default_tier={:?}, score={:.3})", - self.default_tier, self.score - ) - } - } -} - -/// Decide a turn's tier from its [`ToolResultSignal`], up to (but not including) -/// the classifier. `picker_mode` is `"capable_first"` or `"efficient_first"`. -#[pyfunction] -fn stage_pick_tier( - signal: PyRef<'_, PyToolResultSignal>, - picker_mode: &str, - confidence_threshold: f64, -) -> PyResult { - let mode = parse_picker_mode(picker_mode)?; - let outcome = match pick_tier(signal.core(), mode, confidence_threshold) { - PickOutcome::Resolved { - tier, - source, - score, - confidence, - } => PyPickOutcome { - resolved: true, - tier: Some(tier_str(tier)), - source: Some(source.as_str()), - default_tier: tier_str(tier), - score, - confidence, - }, - PickOutcome::ConsultClassifier { - score, - confidence, - default_tier, - } => PyPickOutcome { - resolved: false, - tier: None, - source: None, - default_tier: tier_str(default_tier), - score, - confidence: Some(confidence), - }, - }; - Ok(outcome) -} - -/// The pure two-axis scorer for a signal, as `(score, confidence)`. Used by -/// offline analysis to replay the raw score independent of the picker mode. -#[pyfunction] -fn stage_score_signal(signal: PyRef<'_, PyToolResultSignal>) -> (f64, f64) { - let result = score_signal(signal.core()); - (result.score, result.confidence) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(stage_pick_tier, module)?)?; - module.add_function(wrap_pyfunction!(stage_score_signal, module)?)?; - Ok(()) -} diff --git a/crates/switchyard-py/src/component_bindings/stats.rs b/crates/switchyard-py/src/component_bindings/stats.rs deleted file mode 100644 index 08a7023f8..000000000 --- a/crates/switchyard-py/src/component_bindings/stats.rs +++ /dev/null @@ -1,241 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Python bindings for shared stats state. - -use pyo3::exceptions::PyValueError; -use pyo3::prelude::*; -use serde::Serialize; -use switchyard_components::{StatsAccumulator, StatsRouteLabel, TokenUsage}; - -use crate::errors::py_core_error; -use crate::interop::context::insert_into_python; -use crate::py_serde::value_to_python; - -#[pyclass(name = "StatsAccumulator", skip_from_py_object)] -#[derive(Clone, Debug)] -pub(crate) struct PyStatsAccumulator { - inner: StatsAccumulator, -} - -impl PyStatsAccumulator { - pub(crate) fn from_core(inner: StatsAccumulator) -> Self { - Self { inner } - } - - pub(crate) fn clone_core(&self) -> StatsAccumulator { - self.inner.clone() - } -} - -#[pymethods] -impl PyStatsAccumulator { - #[new] - fn py_new() -> Self { - Self { - inner: StatsAccumulator::new(), - } - } - - #[pyo3(signature = (model, backend_latency_ms=None, tier=None))] - fn record_success<'py>( - &self, - py: Python<'py>, - model: String, - backend_latency_ms: Option, - tier: Option, - ) -> PyResult> { - let accumulator = self.inner.clone(); - pyo3_async_runtimes::tokio::future_into_py(py, async move { - accumulator - .record_success(model, backend_latency_ms, tier.as_deref()) - .map_err(py_core_error) - }) - } - - #[pyo3(signature = (model, tier=None))] - fn record_error<'py>( - &self, - py: Python<'py>, - model: String, - tier: Option, - ) -> PyResult> { - let accumulator = self.inner.clone(); - pyo3_async_runtimes::tokio::future_into_py(py, async move { - accumulator - .record_error(model, tier.as_deref()) - .map_err(py_core_error) - }) - } - - #[pyo3(signature = ( - model, - prompt_tokens=0, - completion_tokens=0, - cached_tokens=0, - cache_creation_tokens=0, - reasoning_tokens=0, - total_latency_ms=None, - routing_overhead_ms=None, - tier=None, - success_was_untiered=false - ))] - #[allow(clippy::too_many_arguments)] - fn record_usage<'py>( - &self, - py: Python<'py>, - model: String, - prompt_tokens: u64, - completion_tokens: u64, - cached_tokens: u64, - cache_creation_tokens: u64, - reasoning_tokens: u64, - total_latency_ms: Option, - routing_overhead_ms: Option, - tier: Option, - success_was_untiered: bool, - ) -> PyResult> { - let accumulator = self.inner.clone(); - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let usage = TokenUsage { - prompt_tokens, - completion_tokens, - cached_tokens, - cache_creation_tokens, - reasoning_tokens, - cacheable_prompt_tokens: 0, - }; - let result = if success_was_untiered { - accumulator.record_usage_with_success_was_untiered( - model, - usage, - total_latency_ms, - routing_overhead_ms, - tier.as_deref(), - ) - } else { - accumulator.record_usage( - model, - usage, - total_latency_ms, - routing_overhead_ms, - tier.as_deref(), - ) - }; - result.map_err(py_core_error) - }) - } - - #[pyo3(signature = ( - model, - prompt_tokens=0, - completion_tokens=0, - cached_tokens=0, - cache_creation_tokens=0, - reasoning_tokens=0, - latency_ms=None, - ))] - #[allow(clippy::too_many_arguments)] - fn record_classifier_usage<'py>( - &self, - py: Python<'py>, - model: String, - prompt_tokens: u64, - completion_tokens: u64, - cached_tokens: u64, - cache_creation_tokens: u64, - reasoning_tokens: u64, - latency_ms: Option, - ) -> PyResult> { - let accumulator = self.inner.clone(); - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let usage = TokenUsage { - prompt_tokens, - completion_tokens, - cached_tokens, - cache_creation_tokens, - reasoning_tokens, - cacheable_prompt_tokens: 0, - }; - accumulator - .record_classifier_usage(model, usage, latency_ms) - .map_err(py_core_error) - }) - } - - fn record_classifier_error<'py>( - &self, - py: Python<'py>, - model: String, - ) -> PyResult> { - let accumulator = self.inner.clone(); - pyo3_async_runtimes::tokio::future_into_py(py, async move { - accumulator - .record_classifier_error(model) - .map_err(py_core_error) - }) - } - - fn record_routing_decision<'py>( - &self, - py: Python<'py>, - profile_type: String, - source: String, - ) -> PyResult> { - let accumulator = self.inner.clone(); - pyo3_async_runtimes::tokio::future_into_py(py, async move { - accumulator - .record_routing_decision(profile_type, source) - .map_err(py_core_error) - }) - } - - fn snapshot<'py>(&self, py: Python<'py>) -> PyResult> { - let accumulator = self.inner.clone(); - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let snapshot = accumulator.snapshot().map_err(py_core_error)?; - Python::attach(|py| to_python(py, &snapshot)) - }) - } - - fn snapshot_sync(&self, py: Python<'_>) -> PyResult> { - let snapshot = self.inner.snapshot().map_err(py_core_error)?; - to_python(py, &snapshot) - } - - fn reset<'py>(&self, py: Python<'py>) -> PyResult> { - let accumulator = self.inner.clone(); - pyo3_async_runtimes::tokio::future_into_py(py, async move { - accumulator.reset().map_err(py_core_error) - }) - } - - fn reset_sync(&self) -> PyResult<()> { - self.inner.reset().map_err(py_core_error) - } - - fn __repr__(&self) -> &'static str { - "StatsAccumulator()" - } -} - -fn to_python(py: Python<'_>, value: &impl Serialize) -> PyResult> { - let value = - serde_json::to_value(value).map_err(|error| PyValueError::new_err(error.to_string()))?; - value_to_python(py, &value) -} - -#[pyfunction] -fn set_stats_route_label(ctx: &Bound<'_, PyAny>, label: &str) -> PyResult<()> { - let label = label.trim(); - if label.is_empty() { - return Err(PyValueError::new_err("stats route label must not be empty")); - } - insert_into_python(ctx, StatsRouteLabel::new(label)) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(set_stats_route_label, module)?)?; - Ok(()) -} diff --git a/crates/switchyard-py/src/errors.rs b/crates/switchyard-py/src/errors.rs index 99ab5ed88..9e0beb775 100644 --- a/crates/switchyard-py/src/errors.rs +++ b/crates/switchyard-py/src/errors.rs @@ -1,216 +1,19 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Error mapping helpers for PyO3 bindings. +//! Error mapping for the libsy Python binding. use pyo3::create_exception; -use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::*; -use switchyard_components::SwitchyardError; -use switchyard_translation::TranslationError; -create_exception!(_switchyard_rust, SwitchyardRuntimeError, PyRuntimeError); -create_exception!(_switchyard_rust, LibsyError, SwitchyardRuntimeError); -create_exception!( - _switchyard_rust, - SwitchyardConfigError, - SwitchyardRuntimeError -); -create_exception!( - _switchyard_rust, - SwitchyardInvalidIdError, - SwitchyardRuntimeError -); -create_exception!( - _switchyard_rust, - SwitchyardDuplicateRegistrationError, - SwitchyardRuntimeError -); -create_exception!( - _switchyard_rust, - SwitchyardModelNotFoundError, - SwitchyardRuntimeError -); -create_exception!( - _switchyard_rust, - SwitchyardUnsupportedRequestTypeError, - SwitchyardRuntimeError -); -// Raised by `ChatRequest.validate()` when a structurally valid body fails -// semantic validation (e.g. an empty `messages` array). Endpoints map it to -// a 4xx so agents can distinguish a client bug from a server failure. -create_exception!( - _switchyard_rust, - SwitchyardInvalidRequestError, - SwitchyardRuntimeError -); -create_exception!( - _switchyard_rust, - SwitchyardProcessorError, - SwitchyardRuntimeError -); -create_exception!( - _switchyard_rust, - SwitchyardBackendError, - SwitchyardRuntimeError -); -create_exception!( - _switchyard_rust, - SwitchyardUpstreamError, - SwitchyardRuntimeError -); -// Raised by an LLMBackend when the upstream rejects a request because the -// prompt exceeds the model's context window. Subclasses SwitchyardBackendError -// so existing broad catches still match. -create_exception!( - _switchyard_rust, - SwitchyardContextWindowExceededError, - SwitchyardBackendError -); -// Raised by compatibility/runtime code when every attempted target returned a -// context-window overflow and no fallback remains. -create_exception!( - _switchyard_rust, - SwitchyardContextPoolExhaustedError, - SwitchyardBackendError -); - -/// Converts translation crate errors into Python `ValueError`s with stable context. -pub(crate) fn py_translation_error(error: TranslationError) -> PyErr { - PyValueError::new_err(format!("{}: {}", error.kind(), error)) -} +create_exception!(_switchyard_rust, LibsyError, PyRuntimeError); /// Converts libsy execution failures into one stable Python exception. pub(crate) fn py_libsy_error(error: impl std::fmt::Display) -> PyErr { LibsyError::new_err(error.to_string()) } -/// Converts core Switchyard errors into typed Python runtime errors. -/// -/// `ContextWindowExceeded` and `ContextPoolExhausted` carry typed fields -/// (`target_id`, `model`, `last_target_id`, `reason`) — we attach those as -/// Python attributes on the raised exception so callers can inspect them -/// programmatically and `backend_error_with_ctx` can recover the typed -/// variant when a Python `LLMBackend` re-raises through the Rust chain. -/// Without these attrs the variant collapses to the string message and the -/// compatibility retry code stamps `"unknown"` for `target_id`/`model`. -pub(crate) fn py_core_error(error: SwitchyardError) -> PyErr { - let message = error.to_string(); - match error { - SwitchyardError::InvalidConfig(_) => SwitchyardConfigError::new_err(message), - SwitchyardError::InvalidId(_) => SwitchyardInvalidIdError::new_err(message), - SwitchyardError::DuplicateRegistration { .. } => { - SwitchyardDuplicateRegistrationError::new_err(message) - } - SwitchyardError::ModelNotFound { .. } => SwitchyardModelNotFoundError::new_err(message), - SwitchyardError::UnsupportedRequestType { .. } => { - SwitchyardUnsupportedRequestTypeError::new_err(message) - } - SwitchyardError::InvalidRequest(_) => SwitchyardInvalidRequestError::new_err(message), - SwitchyardError::Processor(_) => SwitchyardProcessorError::new_err(message), - SwitchyardError::Backend(_) => SwitchyardBackendError::new_err(message), - SwitchyardError::Upstream(_) => SwitchyardUpstreamError::new_err(message), - SwitchyardError::UpstreamHttp { - status_code, body, .. - } => { - let err = SwitchyardUpstreamError::new_err(message); - attach_upstream_http_attrs(&err, status_code, &body); - err - } - SwitchyardError::ContextWindowExceeded { - target_id, model, .. - } => { - let err = SwitchyardContextWindowExceededError::new_err(message); - attach_attrs(&err, &[("target_id", &target_id), ("model", &model)]); - err - } - SwitchyardError::ContextPoolExhausted { - last_target_id, - reason, - } => { - let err = SwitchyardContextPoolExhaustedError::new_err(message); - attach_attrs( - &err, - &[("last_target_id", &last_target_id), ("reason", &reason)], - ); - err - } - SwitchyardError::Other(_) => SwitchyardRuntimeError::new_err(message), - } -} - -/// Set string attributes on a freshly-constructed `PyErr`'s exception value. -/// Errors are intentionally ignored — attribute attachment is best-effort -/// diagnostic metadata, never the path-of-correctness for raising the error. -fn attach_attrs(err: &PyErr, attrs: &[(&str, &str)]) { - Python::attach(|py| { - let bound = err.value(py); - for (name, value) in attrs { - let _ = bound.setattr(*name, *value); - } - }); -} - -/// Attach typed upstream HTTP details for endpoint error handling. -fn attach_upstream_http_attrs(err: &PyErr, status_code: u16, body: &str) { - Python::attach(|py| { - let bound = err.value(py); - let _ = bound.setattr("status_code", status_code); - let _ = bound.setattr("body", body); - }); -} - -/// Registers typed exception classes in the native extension module. pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - let py = module.py(); - module.add( - "SwitchyardRuntimeError", - py.get_type::(), - )?; - module.add("LibsyError", py.get_type::())?; - module.add( - "SwitchyardConfigError", - py.get_type::(), - )?; - module.add( - "SwitchyardInvalidIdError", - py.get_type::(), - )?; - module.add( - "SwitchyardDuplicateRegistrationError", - py.get_type::(), - )?; - module.add( - "SwitchyardModelNotFoundError", - py.get_type::(), - )?; - module.add( - "SwitchyardUnsupportedRequestTypeError", - py.get_type::(), - )?; - module.add( - "SwitchyardInvalidRequestError", - py.get_type::(), - )?; - module.add( - "SwitchyardProcessorError", - py.get_type::(), - )?; - module.add( - "SwitchyardBackendError", - py.get_type::(), - )?; - module.add( - "SwitchyardUpstreamError", - py.get_type::(), - )?; - module.add( - "SwitchyardContextWindowExceededError", - py.get_type::(), - )?; - module.add( - "SwitchyardContextPoolExhaustedError", - py.get_type::(), - )?; - Ok(()) + module.add("LibsyError", module.py().get_type::()) } diff --git a/crates/switchyard-py/src/interop.rs b/crates/switchyard-py/src/interop.rs deleted file mode 100644 index 2088ed48a..000000000 --- a/crates/switchyard-py/src/interop.rs +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Private adapters between Python compatibility objects and native components. - -use pyo3::prelude::*; - -pub(crate) mod context; -pub(crate) mod request; -pub(crate) mod response; -pub(crate) mod roles; -pub(crate) mod subagent; - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - context::register(module)?; - response::register(module)?; - roles::register(module)?; - subagent::register(module)?; - Ok(()) -} diff --git a/crates/switchyard-py/src/interop/context.rs b/crates/switchyard-py/src/interop/context.rs deleted file mode 100644 index cd84bde3e..000000000 --- a/crates/switchyard-py/src/interop/context.rs +++ /dev/null @@ -1,361 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Private adapter for Rust-owned request context values. - -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; - -use parking_lot::{Mutex, MutexGuard}; -use pyo3::exceptions::{PyRuntimeError, PyValueError}; -use pyo3::prelude::*; -use std::time::Duration; -use switchyard_components::{BackendSelection, BackendSelectionReason, StatsBackendLatency}; -use switchyard_components::{EvictedTargets, LlmTargetId, ModelId, ProxyContext, RequestId}; - -use super::request::{request_type_from_python, request_type_object}; - -/// Python-facing proxy context backed by the Rust `ProxyContext`. -#[pyclass(name = "_NativeProxyContext", skip_from_py_object)] -pub(crate) struct PyProxyContext { - /// Shared Rust context guarded across Python and async Rust calls. - inner: Arc>, - /// Borrow flag that prevents Python mutation while Rust owns the context. - in_use: Arc, -} - -impl PyProxyContext { - /// Locks the context unless an async Rust component currently owns it. - fn lock(&self) -> PyResult> { - if self.in_use.load(Ordering::Acquire) { - return Err(PyRuntimeError::new_err( - "ProxyContext is already borrowed by an async Rust component", - )); - } - Ok(self.inner.lock()) - } - - /// Temporarily moves the Rust context out for async Rust component execution. - pub(crate) fn lease(&self) -> PyResult { - if self.in_use.swap(true, Ordering::AcqRel) { - return Err(PyRuntimeError::new_err( - "ProxyContext is already borrowed by an async Rust component", - )); - } - let context = { - let mut guard = self.inner.lock(); - std::mem::take(&mut *guard) - }; - Ok(PyProxyContextLease { - inner: Arc::clone(&self.inner), - in_use: Arc::clone(&self.in_use), - context: Some(context), - }) - } - - /// Inserts a typed Rust extension into the wrapped context. - pub(crate) fn insert_value(&self, value: T) -> PyResult<()> - where - T: Send + Sync + 'static, - { - self.lock()?.insert(value); - Ok(()) - } - - /// Returns a clone of a typed Rust extension if one is present. - /// - /// Companion to [`insert_value`] for Python-facing readers of typed - /// extensions stamped by Rust processors (e.g. `ContextSignals`). - pub(crate) fn get_cloned(&self) -> PyResult> - where - T: Clone + Send + Sync + 'static, - { - Ok(self.lock()?.get::().cloned()) - } -} - -/// Leases the native context carried by a Python `ProxyContext`. -pub(crate) fn lease_from_python(value: &Bound<'_, PyAny>) -> PyResult { - value - .getattr("_native")? - .extract::>()? - .lease() -} - -/// Inserts a typed value into a Python context's native state. -pub(crate) fn insert_into_python(value: &Bound<'_, PyAny>, item: T) -> PyResult<()> -where - T: Send + Sync + 'static, -{ - value - .getattr("_native")? - .extract::>()? - .insert_value(item) -} - -/// Reads a typed value from a Python context's native state. -pub(crate) fn get_cloned_from_python(value: &Bound<'_, PyAny>) -> PyResult> -where - T: Clone + Send + Sync + 'static, -{ - value - .getattr("_native")? - .extract::>()? - .get_cloned::() -} - -/// Temporary ownership lease for passing `ProxyContext` into async Rust roles. -pub(crate) struct PyProxyContextLease { - /// Shared storage that receives the context when the lease is restored. - inner: Arc>, - /// Borrow flag cleared on restore or drop. - in_use: Arc, - /// Moved-out Rust context. - context: Option, -} - -impl PyProxyContextLease { - /// Returns mutable access to the leased Rust context. - pub(crate) fn context_mut(&mut self) -> PyResult<&mut ProxyContext> { - self.context - .as_mut() - .ok_or_else(|| PyRuntimeError::new_err("ProxyContext lease has already been restored")) - } - - /// Restores the leased context into the Python wrapper. - pub(crate) fn restore(mut self) -> PyResult<()> { - if let Some(context) = self.context.take() { - let mut guard = self.inner.lock(); - *guard = context; - } - self.in_use.store(false, Ordering::Release); - Ok(()) - } -} - -impl Drop for PyProxyContextLease { - /// Restores the context during unwinding and always clears the borrow flag. - fn drop(&mut self) { - if let Some(context) = self.context.take() { - *self.inner.lock() = context; - } - self.in_use.store(false, Ordering::Release); - } -} - -#[pymethods] -impl PyProxyContext { - /// Creates a context with a validated native request ID. - /// - /// Metadata is accepted for Python compatibility but remains in the - /// Python-owned `ProxyMetadata` rather than the native context. - #[new] - #[pyo3(signature = (metadata=None, request_id=None))] - fn new(metadata: Option<&Bound<'_, PyAny>>, request_id: Option) -> PyResult { - let _ = metadata; - let request_id = request_id - .map(RequestId::new) - .transpose() - .map_err(|error| { - PyValueError::new_err(format!("invalid request_id for ProxyContext: {error}")) - })?; - - let mut inner = ProxyContext::default(); - inner.request_id = request_id; - - Ok(Self { - inner: Arc::new(Mutex::new(inner)), - in_use: Arc::new(AtomicBool::new(false)), - }) - } - - /// Returns the optional request ID. - #[getter] - fn request_id(&self) -> PyResult> { - Ok(self - .lock()? - .request_id - .as_ref() - .map(|request_id| request_id.as_str().to_string())) - } - - /// Updates the optional request ID. - #[setter] - fn set_request_id(&self, value: Option) -> PyResult<()> { - self.lock()?.request_id = value.map(RequestId::new).transpose().map_err(|error| { - PyValueError::new_err(format!("invalid request_id for ProxyContext: {error}")) - })?; - Ok(()) - } - - /// Returns the inbound request format as a Python enum object. - #[getter] - fn inbound_format(&self, py: Python<'_>) -> PyResult>> { - self.lock()? - .inbound_format - .map(|request_type| request_type_object(py, request_type)) - .transpose() - } - - /// Updates the inbound request format from a Python enum object. - #[setter] - fn set_inbound_format(&self, value: Option<&Bound<'_, PyAny>>) -> PyResult<()> { - self.lock()?.inbound_format = value - .filter(|value| !value.is_none()) - .map(request_type_from_python) - .transpose()?; - Ok(()) - } - - /// Returns the selected served model, if backend selection exists. - #[getter] - fn selected_model(&self) -> PyResult> { - Ok(self - .lock()? - .get::() - .map(|selection| selection.model.as_str().to_string())) - } - - /// Updates the selected served model using backend-selection metadata. - #[setter] - fn set_selected_model(&self, value: Option) -> PyResult<()> { - let mut inner = self.lock()?; - match value { - Some(value) => { - let model = ModelId::new(value).map_err(|error| { - PyValueError::new_err(format!( - "invalid selected_model for ProxyContext: {error}" - )) - })?; - inner.insert(BackendSelection::for_model( - model, - None, - BackendSelectionReason::PassthroughModel, - )); - } - None => { - inner.remove::(); - } - } - Ok(()) - } - - /// Returns the selected target ID, if any. - #[getter] - fn selected_target(&self) -> PyResult> { - Ok(self - .lock()? - .selected_target() - .map(|target| target.as_str().to_string())) - } - - /// Returns the set of target IDs evicted from the routing pool after a - /// context-window overflow on the current request. - /// `None` when no evictions have happened yet. - #[getter] - fn evicted_targets(&self) -> PyResult>> { - Ok(self.lock()?.get::().map(|evicted| { - let mut ids: Vec = evicted.iter().map(|id| id.as_str().to_string()).collect(); - ids.sort(); - ids - })) - } - - /// Replaces the evicted target set used by the Python compatibility chain. - #[setter] - fn set_evicted_targets(&self, value: Option>) -> PyResult<()> { - let mut inner = self.lock()?; - match value { - Some(values) => { - let mut evicted = EvictedTargets::default(); - for target in values { - evicted.insert(LlmTargetId::new(target).map_err(|error| { - PyValueError::new_err(format!( - "invalid evicted target for ProxyContext: {error}" - )) - })?); - } - inner.insert(evicted); - } - None => { - inner.remove::(); - } - } - Ok(()) - } - - /// Returns the measured backend-call latency in ms, if any backend recorded one. - #[getter] - fn backend_call_latency_ms(&self) -> PyResult> { - Ok(self - .lock()? - .get::() - .map(|latency| latency.as_millis_f64())) - } - - /// Records the measured backend-call latency in ms. - /// - /// The Rust ``StatsLlmBackend`` wraps native backends and writes this - /// slot automatically. Python-only backends that can't be wrapped record - /// their measurement here so the downstream ``StatsResponseProcessor`` can - /// compute ``routing_overhead_ms = total_latency - backend_latency`` and - /// emit it on ``/metrics``. Setting ``None`` clears the slot. - #[setter] - fn set_backend_call_latency_ms(&self, value: Option) -> PyResult<()> { - let mut inner = self.lock()?; - match value { - Some(ms) => { - if !ms.is_finite() || ms < 0.0 { - return Err(PyValueError::new_err( - "backend_call_latency_ms must be a finite, non-negative number", - )); - } - inner.insert(StatsBackendLatency(Duration::from_secs_f64(ms / 1000.0))); - } - None => { - inner.remove::(); - } - } - Ok(()) - } - - /// Updates the selected target ID. - #[setter] - fn set_selected_target(&self, value: Option) -> PyResult<()> { - let mut inner = self.lock()?; - match value { - Some(value) => { - inner.set_selected_target(LlmTargetId::new(value).map_err(|error| { - PyValueError::new_err(format!( - "invalid selected_target for ProxyContext: {error}" - )) - })?); - } - None => { - inner.clear_selected_target(); - } - } - Ok(()) - } - - /// Returns a compact debug representation for Python users. - fn __repr__(&self) -> PyResult { - let inner = self.lock()?; - let selected_model = inner - .get::() - .map(|selection| selection.model.as_str().to_string()); - let selected_target = inner - .selected_target() - .map(|target| target.as_str().to_string()); - Ok(format!( - "ProxyContext(request_id={:?}, inbound_format={:?}, selected_model={:?}, selected_target={:?})", - inner.request_id, inner.inbound_format, selected_model, selected_target, - )) - } -} - -/// Registers proxy context bindings into the Python module. -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - Ok(()) -} diff --git a/crates/switchyard-py/src/interop/request.rs b/crates/switchyard-py/src/interop/request.rs deleted file mode 100644 index 376d32722..000000000 --- a/crates/switchyard-py/src/interop/request.rs +++ /dev/null @@ -1,70 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Private adapter for Rust-owned chat request values. - -use pyo3::exceptions::PyValueError; -use pyo3::prelude::*; -use switchyard_components::{ChatRequest, ChatRequestType}; - -use crate::py_serde::{value_from_python, value_to_python}; - -/// Converts a public Python request into the native request value. -pub(crate) fn request_from_python(value: &Bound<'_, PyAny>) -> PyResult { - let request_type = request_type_from_python(&value.getattr("request_type")?)?; - let body = value_from_python(&value.getattr("_body")?)?; - Ok(match request_type { - ChatRequestType::OpenAiChat => ChatRequest::openai_chat(body), - ChatRequestType::OpenAiResponses => ChatRequest::openai_responses(body), - ChatRequestType::Anthropic => ChatRequest::anthropic(body), - }) -} - -/// Converts a native request into the public Python request value. -pub(crate) fn request_to_python(py: Python<'_>, request: ChatRequest) -> PyResult> { - let request_type = request.request_type(); - let body = value_to_python(py, request.body())?; - let factory = match request_type { - ChatRequestType::OpenAiChat => "openai_chat", - ChatRequestType::OpenAiResponses => "openai_responses", - ChatRequestType::Anthropic => "anthropic", - }; - py.import("switchyard_rust.core")? - .getattr("ChatRequest")? - .call_method1(factory, (body,)) - .map(Bound::unbind) -} - -pub(crate) fn request_type_from_python(value: &Bound<'_, PyAny>) -> PyResult { - let raw = if let Ok(value_attr) = value.getattr("value") { - value_attr.extract::()? - } else { - value.extract::()? - }; - match raw.as_str() { - "openai_chat" => Ok(ChatRequestType::OpenAiChat), - "openai_responses" => Ok(ChatRequestType::OpenAiResponses), - "anthropic" | "anthropic_messages" => Ok(ChatRequestType::Anthropic), - _ => Err(PyValueError::new_err(format!( - "Unknown request type: {raw:?}" - ))), - } -} - -pub(crate) fn request_type_variant_name(request_type: ChatRequestType) -> &'static str { - match request_type { - ChatRequestType::OpenAiChat => "OPENAI_CHAT", - ChatRequestType::OpenAiResponses => "OPENAI_RESPONSES", - ChatRequestType::Anthropic => "ANTHROPIC", - } -} - -pub(crate) fn request_type_object( - py: Python<'_>, - request_type: ChatRequestType, -) -> PyResult> { - py.import("switchyard_rust.core")? - .getattr("ChatRequestType")? - .getattr(request_type_variant_name(request_type)) - .map(Bound::unbind) -} diff --git a/crates/switchyard-py/src/interop/response.rs b/crates/switchyard-py/src/interop/response.rs deleted file mode 100644 index 1e07c0706..000000000 --- a/crates/switchyard-py/src/interop/response.rs +++ /dev/null @@ -1,613 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Private adapter for Rust-owned chat response values. - -use std::pin::Pin; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::task::{Context, Poll}; - -use futures_util::{Stream, StreamExt, stream}; -use parking_lot::Mutex; -use pyo3::exceptions::{PyRuntimeError, PyStopAsyncIteration, PyValueError}; -use pyo3::prelude::*; -use switchyard_components::{BoxResponseStream, ChatResponse, ChatResponseType, StreamEvent}; - -use crate::errors::py_core_error; -use crate::py_serde::{value_from_python, value_to_python}; - -/// Converts a public Python response into the native response value. -pub(crate) fn response_from_python(value: &Bound<'_, PyAny>) -> PyResult { - let response_type = response_type_from_python(&value.getattr("response_type")?)?; - Ok(match response_type { - ChatResponseType::OpenAiCompletion => { - ChatResponse::openai_completion(value_from_python(&value.getattr("_body")?)?) - } - ChatResponseType::OpenAiResponsesCompletion => { - ChatResponse::openai_responses_completion(value_from_python(&value.getattr("_body")?)?) - } - ChatResponseType::AnthropicCompletion => { - ChatResponse::anthropic_completion(value_from_python(&value.getattr("_body")?)?) - } - ChatResponseType::OpenAiStream => ChatResponse::OpenAiStream(stream_from_python(value)?), - ChatResponseType::OpenAiResponsesStream => { - ChatResponse::OpenAiResponsesStream(stream_from_python(value)?) - } - ChatResponseType::AnthropicStream => { - ChatResponse::AnthropicStream(stream_from_python(value)?) - } - }) -} - -/// Converts a native response into the public Python response value. -pub(crate) fn response_to_python(py: Python<'_>, response: ChatResponse) -> PyResult> { - let core = py.import("switchyard_rust.core")?; - let (factory, payload) = match response { - ChatResponse::OpenAiCompletion(response) => ( - "openai_completion", - value_to_python(py, &response.into_body())?, - ), - ChatResponse::OpenAiResponsesCompletion(response) => ( - "openai_responses_completion", - value_to_python(py, &response.into_body())?, - ), - ChatResponse::AnthropicCompletion(response) => ( - "anthropic_completion", - value_to_python(py, &response.into_body())?, - ), - ChatResponse::OpenAiStream(stream) => { - ("openai_stream", stream_to_python(py, &core, stream)?) - } - ChatResponse::OpenAiResponsesStream(stream) => ( - "openai_responses_stream", - stream_to_python(py, &core, stream)?, - ), - ChatResponse::AnthropicStream(stream) => { - ("anthropic_stream", stream_to_python(py, &core, stream)?) - } - }; - core.getattr("ChatResponse")? - .call_method1(factory, (payload,)) - .map(Bound::unbind) -} - -fn response_type_from_python(value: &Bound<'_, PyAny>) -> PyResult { - let raw = if let Ok(value_attr) = value.getattr("value") { - value_attr.extract::()? - } else { - value.extract::()? - }; - match raw.as_str() { - "openai_completion" => Ok(ChatResponseType::OpenAiCompletion), - "openai_stream" => Ok(ChatResponseType::OpenAiStream), - "openai_responses_completion" => Ok(ChatResponseType::OpenAiResponsesCompletion), - "openai_responses_stream" => Ok(ChatResponseType::OpenAiResponsesStream), - "anthropic_completion" => Ok(ChatResponseType::AnthropicCompletion), - "anthropic_stream" => Ok(ChatResponseType::AnthropicStream), - _ => Err(PyValueError::new_err(format!( - "Unknown response type: {raw:?}" - ))), - } -} - -fn stream_from_python(value: &Bound<'_, PyAny>) -> PyResult { - let native = value.getattr("stream")?.getattr("_native")?; - native - .extract::>()? - .take_core_stream() -} - -fn stream_to_python( - py: Python<'_>, - core: &Bound<'_, PyModule>, - stream: BoxResponseStream, -) -> PyResult> { - let native = Py::new(py, PyResponseStream::from_core_stream(stream))?; - core.getattr("ChatResponseStream")? - .call_method1("_from_native", (native,)) - .map(Bound::unbind) -} - -struct PyResponseStreamSource { - source: Py, - iterator: Option>, - done: bool, -} - -type BoxPyResponseStream = std::pin::Pin>> + Send>>; - -#[pyclass(name = "_NativeChatResponseStream")] -pub(crate) struct PyResponseStream { - stream: Arc>>, - taps: Arc>>>, - maps: Arc>>>, - on_complete: Arc>>>, - consumed: Arc, - completed: Arc, - // The original Python stream object (e.g. the OpenAI SDK ``AsyncStream``) - // when this stream was built from a Python source. Retained so ``aclose`` - // can release the upstream response — and the pooled connection it holds — - // on early termination. ``None`` for Rust-native streams, which own no - // closable Python resource. - source: Option>, -} - -impl PyResponseStream { - fn new(source: Py) -> Self { - let retained = Python::attach(|py| source.clone_ref(py)); - let mut stream = Self::from_stream(stream_from_python_source(source)); - stream.source = Some(retained); - stream - } - - fn from_stream(stream: BoxPyResponseStream) -> Self { - Self { - stream: Arc::new(tokio::sync::Mutex::new(Some(stream))), - taps: Arc::new(Mutex::new(Vec::new())), - maps: Arc::new(Mutex::new(Vec::new())), - on_complete: Arc::new(Mutex::new(Vec::new())), - consumed: Arc::new(AtomicBool::new(false)), - completed: Arc::new(AtomicBool::new(false)), - source: None, - } - } - - fn from_core_stream(stream: BoxResponseStream) -> Self { - Self::from_stream(Box::pin(stream.map(|event| { - Python::attach(|py| match event { - Ok(StreamEvent::Json(value)) => value_to_python(py, &value), - Ok(StreamEvent::Text(value)) => Ok(value.into_pyobject(py)?.unbind().into_any()), - Err(error) => Err(py_core_error(error)), - }) - }))) - } - - fn take_core_stream(&self) -> PyResult { - if self.consumed.swap(true, Ordering::AcqRel) { - return Err(PyRuntimeError::new_err( - "ChatResponseStream has already been consumed", - )); - } - let stream = Arc::clone(&self.stream); - let taps = Arc::clone(&self.taps); - let maps = Arc::clone(&self.maps); - let on_complete = Arc::clone(&self.on_complete); - let completed = Arc::clone(&self.completed); - let core: BoxResponseStream = Box::pin(stream::unfold( - (stream, taps, maps, on_complete, completed), - |(stream, taps, maps, on_complete, completed)| async move { - let event = next_stream_item_with_callbacks( - Arc::clone(&stream), - Arc::clone(&taps), - Arc::clone(&maps), - Arc::clone(&on_complete), - Arc::clone(&completed), - ) - .await; - event.map(|event| { - let event = match event { - Ok(event) => Python::attach(|py| stream_event_from_python(py, event)), - Err(error) => Err(error), - }; - ( - event.map_err(|error| { - switchyard_components::SwitchyardError::Processor(error.to_string()) - }), - (stream, taps, maps, on_complete, completed), - ) - }) - }, - )); - // Preserve close ownership across the Python -> core -> Python round - // trip: ``from_core_stream`` rebuilds a ``PyResponseStream`` with no - // ``source``, so its ``aclose`` could not release the upstream SDK - // stream. Carrying the source on the core stream — and closing it when - // that stream is dropped — keeps the connection releasable even after - // response processors re-wrap the stream. - match self.source.as_ref() { - Some(source) => { - let source = Python::attach(|py| source.clone_ref(py)); - Ok(Box::pin(SourceClosingStream::new(core, source))) - } - None => Ok(core), - } - } -} - -#[pymethods] -impl PyResponseStream { - #[new] - fn py_new(source: &Bound<'_, PyAny>) -> Self { - Self::new(source.clone().unbind()) - } - - fn tap(slf: PyRef<'_, Self>, callback: &Bound<'_, PyAny>) -> PyResult> { - push_callback(&slf.taps, callback)?; - Ok(slf.into_pyobject(callback.py())?.unbind().into_any()) - } - - fn map(slf: PyRef<'_, Self>, callback: &Bound<'_, PyAny>) -> PyResult> { - push_callback(&slf.maps, callback)?; - Ok(slf.into_pyobject(callback.py())?.unbind().into_any()) - } - - fn on_complete(slf: PyRef<'_, Self>, callback: &Bound<'_, PyAny>) -> PyResult> { - push_callback(&slf.on_complete, callback)?; - Ok(slf.into_pyobject(callback.py())?.unbind().into_any()) - } - - fn __aiter__(slf: PyRef<'_, Self>) -> PyResult> { - if slf.consumed.swap(true, Ordering::AcqRel) { - return Err(PyRuntimeError::new_err( - "ChatResponseStream has already been consumed", - )); - } - let py = slf.py(); - Ok(slf.into_pyobject(py)?.unbind().into_any()) - } - - fn __anext__<'py>(&self, py: Python<'py>) -> PyResult> { - let stream = Arc::clone(&self.stream); - let taps = Arc::clone(&self.taps); - let maps = Arc::clone(&self.maps); - let on_complete = Arc::clone(&self.on_complete); - let completed = Arc::clone(&self.completed); - - pyo3_async_runtimes::tokio::future_into_py(py, async move { - match next_stream_item_with_callbacks(stream, taps, maps, on_complete, completed).await - { - Some(event) => event, - None => Err(PyStopAsyncIteration::new_err(())), - } - }) - } - - fn __repr__(&self) -> &'static str { - "ChatResponseStream()" - } - - /// Release the upstream stream and its underlying connection. - /// - /// Streaming proxies must close the upstream response when iteration ends - /// early (client disconnect, mid-stream error) — otherwise the SDK - /// ``AsyncStream``'s httpx response is never closed and its pooled - /// connection leaks, exhausting the pool and pinning buffers. Marks the - /// stream consumed and drops the inner adapter; idempotent and safe to - /// call after completion. - /// - /// The upstream source is released by **two** complementary paths, because - /// the source is reachable in only one of two stream shapes: - /// * Built directly from a Python source (``ChatResponseStream(sdk_stream)``, - /// ``source`` is set): closed here, best-effort, via ``close``/``aclose``. - /// * Rebuilt from a core stream by ``from_core_stream`` after the - /// ``Switchyard.call`` round trip (``source`` is ``None``): the source was - /// moved onto the core stream by ``take_core_stream`` as a - /// ``SourceClosingStream``; dropping the inner adapter here drops that - /// wrapper, which closes the source on drop. This is the production path - /// where response processors re-wrap the core stream and ``source`` - /// cannot be recovered on this object. - fn aclose<'py>(&self, py: Python<'py>) -> PyResult> { - self.consumed.store(true, Ordering::Release); - let stream = Arc::clone(&self.stream); - let completed = Arc::clone(&self.completed); - let source = self.source.as_ref().map(|source| source.clone_ref(py)); - pyo3_async_runtimes::tokio::future_into_py(py, async move { - // Drop the inner adapter so it releases its borrow of the upstream - // iterator; subsequent ``__anext__`` then yields StopAsyncIteration. - { - let mut guard = stream.lock().await; - *guard = None; - } - completed.store(true, Ordering::Release); - if let Some(source) = source { - close_python_source(source).await; - } - Ok(()) - }) - } -} - -fn push_callback( - callbacks: &Arc>>>, - callback: &Bound<'_, PyAny>, -) -> PyResult<()> { - callbacks.lock().push(callback.clone().unbind()); - Ok(()) -} - -async fn next_stream_item( - stream: Arc>>, -) -> Option>> { - let mut guard = stream.lock().await; - match guard.as_mut() { - Some(response_stream) => { - let event = response_stream.next().await; - if event.is_none() { - *guard = None; - } - event - } - None => None, - } -} - -async fn next_stream_item_with_callbacks( - stream: Arc>>, - taps: Arc>>>, - maps: Arc>>>, - on_complete: Arc>>>, - completed: Arc, -) -> Option>> { - match next_stream_item(stream).await { - Some(Ok(event)) => { - run_taps(taps, &event).await; - Some(run_maps(maps, event).await) - } - Some(Err(error)) => Some(Err(error)), - None => { - run_completion_once(on_complete, completed).await; - None - } - } -} - -fn stream_from_python_source(source: Py) -> BoxPyResponseStream { - Box::pin(stream::unfold( - PyResponseStreamSource { - source, - iterator: None, - done: false, - }, - |mut state| async move { - if state.done { - return None; - } - let future = Python::attach(|py| { - let iterator = match &state.iterator { - Some(iterator) => iterator.clone_ref(py), - None => { - let iterator = state.source.bind(py).call_method0("__aiter__")?.unbind(); - state.iterator = Some(iterator.clone_ref(py)); - iterator - } - }; - let awaitable = iterator.bind(py).call_method0("__anext__")?; - pyo3_async_runtimes::tokio::into_future(awaitable) - }); - match future { - Ok(future) => match future.await { - Ok(event) => Some((Ok(event), state)), - Err(error) if is_stop_async_iteration(&error) => None, - Err(error) => { - state.done = true; - Some((Err(error), state)) - } - }, - Err(error) => { - state.done = true; - Some((Err(error), state)) - } - } - }, - )) -} - -/// Core stream that closes its originating Python source when dropped. -/// -/// ``take_core_stream`` converts a Python-backed ``PyResponseStream`` into a -/// Rust-core ``BoxResponseStream`` for runtime processing; ``from_core_stream`` -/// later rebuilds a Python wrapper that no longer references the source, so the -/// rebuilt wrapper's ``aclose`` can no longer release the upstream response. -/// Wrapping the core stream here preserves close ownership across that -/// conversion: dropping the stream — directly, or via a response processor that -/// re-wrapped it — schedules a best-effort close of the source so the SDK -/// stream and its pooled connection are released on early termination. -struct SourceClosingStream { - inner: BoxResponseStream, - source: Option>, - // The asyncio event loop + contextvars captured at construction. The - // source's close is a Python coroutine that must be driven on *that* loop, - // not on a bare tokio worker thread; the loop also lets the nested - // ``aclose`` re-enter ``future_into_py`` without a "no running event loop" - // error. - locals: Option, - // Runtime captured at construction. ``Drop`` is synchronous and cannot await, so the - // close future is scheduled onto this handle as fire-and-forget. - handle: Option, -} - -impl SourceClosingStream { - fn new(inner: BoxResponseStream, source: Py) -> Self { - Self { - inner, - source: Some(source), - locals: Python::attach(|py| pyo3_async_runtimes::tokio::get_current_locals(py).ok()), - handle: tokio::runtime::Handle::try_current().ok(), - } - } -} - -impl Stream for SourceClosingStream { - type Item = switchyard_components::Result; - - fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - // All fields are ``Unpin``, so projecting through ``get_mut`` is sound. - self.get_mut().inner.as_mut().poll_next(cx) - } -} - -impl Drop for SourceClosingStream { - fn drop(&mut self) { - let Some(source) = self.source.take() else { - return; - }; - let (Some(locals), Some(handle)) = (self.locals.take(), self.handle.take()) else { - // No event loop / runtime to drive the async close (e.g. interpreter - // teardown); dropping the source ref is the best we can do. - tracing::warn!("ChatResponseStream: no event loop to close stream source on drop"); - return; - }; - // Fire-and-forget on the captured runtime, scoped to the captured event - // loop so the source's async ``aclose`` runs on asyncio. Releasing the - // connection here is what stops the pool leak on client disconnect / - // mid-stream error. - handle.spawn(pyo3_async_runtimes::tokio::scope(locals, async move { - close_python_source(source).await; - })); - } -} - -/// Best-effort close of a Python stream source on teardown. -/// -/// Async generators expose ``aclose``; SDK ``AsyncStream`` objects expose -/// ``close``. Either may return a coroutine that must be awaited. Closing must -/// never mask the teardown that triggered it, so failures are logged and -/// swallowed rather than propagated. -async fn close_python_source(source: Py) { - let awaitable = match Python::attach(|py| detect_close_awaitable(source.bind(py))) { - Ok(awaitable) => awaitable, - Err(error) => { - tracing::warn!(error = %error, "ChatResponseStream: failed to close stream source"); - return; - } - }; - let Some(awaitable) = awaitable else { - return; - }; - let future = - Python::attach(|py| pyo3_async_runtimes::tokio::into_future(awaitable.bind(py).clone())); - match future { - Ok(future) => { - if let Err(error) = future.await { - tracing::warn!(error = %error, "ChatResponseStream: error closing stream source"); - } - } - Err(error) => { - tracing::warn!( - error = %error, - "ChatResponseStream: failed to schedule stream-source close" - ); - } - } -} - -/// Call ``aclose`` (preferred) or ``close`` on a stream source, returning the -/// coroutine to await when the method is asynchronous. Returns ``None`` when -/// the source exposes no closer or the closer is synchronous (already done). -fn detect_close_awaitable(source: &Bound<'_, PyAny>) -> PyResult>> { - let method = if source.hasattr("aclose")? { - "aclose" - } else if source.hasattr("close")? { - "close" - } else { - return Ok(None); - }; - let result = source.call_method0(method)?; - if result.hasattr("__await__")? { - Ok(Some(result.unbind())) - } else { - Ok(None) - } -} - -fn stream_event_from_python(py: Python<'_>, event: Py) -> PyResult { - let event = event.bind(py); - if let Ok(value) = event.extract::() { - return Ok(StreamEvent::Text(value)); - } - Ok(StreamEvent::Json(value_from_python(event)?)) -} - -async fn run_taps(callbacks: Arc>>>, event: &Py) { - let snapshot = match clone_callbacks(&callbacks) { - Ok(snapshot) => snapshot, - Err(_) => return, - }; - let mut failed = Vec::new(); - for (index, callback) in snapshot { - if let Err(error) = call_python_callback(callback, event).await { - tracing::warn!( - error = %error, - callback_index = index, - "ChatResponseStream tap failed, quarantining" - ); - failed.push(index); - } - } - if failed.is_empty() { - return; - } - let mut callbacks = callbacks.lock(); - for index in failed.into_iter().rev() { - if index < callbacks.len() { - callbacks.remove(index); - } - } -} - -async fn run_maps( - callbacks: Arc>>>, - mut event: Py, -) -> PyResult> { - let callbacks = clone_callbacks(&callbacks)?; - for (_, callback) in callbacks { - event = call_python_callback(callback, &event).await?; - } - Ok(event) -} - -async fn run_completion_once(callbacks: Arc>>>, completed: Arc) { - if completed.swap(true, Ordering::AcqRel) { - return; - } - let snapshot = match clone_callbacks(&callbacks) { - Ok(snapshot) => snapshot, - Err(_) => return, - }; - for (index, callback) in snapshot { - if let Err(error) = call_python_callback_no_args(callback).await { - tracing::warn!( - error = %error, - callback_index = index, - "ChatResponseStream completion callback failed" - ); - } - } -} - -fn clone_callbacks(callbacks: &Arc>>>) -> PyResult)>> { - Python::attach(|py| { - let callbacks = callbacks.lock(); - Ok(callbacks - .iter() - .enumerate() - .map(|(index, callback)| (index, callback.clone_ref(py))) - .collect::>()) - }) -} - -async fn call_python_callback(callback: Py, arg: &Py) -> PyResult> { - let future = Python::attach(|py| { - let result = callback.bind(py).call1((arg.clone_ref(py),))?; - pyo3_async_runtimes::tokio::into_future(result) - })?; - future.await -} - -async fn call_python_callback_no_args(callback: Py) -> PyResult> { - let future = Python::attach(|py| { - let result = callback.bind(py).call0()?; - pyo3_async_runtimes::tokio::into_future(result) - })?; - future.await -} - -fn is_stop_async_iteration(error: &PyErr) -> bool { - Python::attach(|py| error.is_instance_of::(py)) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - Ok(()) -} diff --git a/crates/switchyard-py/src/interop/roles.rs b/crates/switchyard-py/src/interop/roles.rs deleted file mode 100644 index 732ec2acc..000000000 --- a/crates/switchyard-py/src/interop/roles.rs +++ /dev/null @@ -1,119 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Private adapter for Rust-owned backend role abstractions. - -use std::sync::Arc; - -use pyo3::PyTypeInfo; -use pyo3::exceptions::{PyNotImplementedError, PyTypeError}; -use pyo3::prelude::*; -use pyo3::types::{PyDict, PyTuple, PyType}; -use switchyard_components::LlmBackend; - -use super::context::lease_from_python; -use super::request::{request_from_python, request_type_object}; -use super::response::response_to_python; -use crate::errors::py_core_error; - -#[pyclass(name = "_NativeLlmBackend", subclass)] -pub(crate) struct PyLlmBackend { - inner: Option>, -} - -impl PyLlmBackend { - pub(crate) fn from_native(inner: Arc) -> Self { - Self { inner: Some(inner) } - } - - pub(crate) fn native(&self) -> Option> { - self.inner.clone() - } -} - -#[pymethods] -impl PyLlmBackend { - #[new] - #[classmethod] - #[pyo3(signature = (*_args, **_kwargs))] - fn py_new( - cls: &Bound<'_, PyType>, - _args: &Bound<'_, PyTuple>, - _kwargs: Option<&Bound<'_, PyDict>>, - ) -> PyResult { - reject_base_instantiation::(cls, "LLMBackend")?; - Ok(Self { inner: None }) - } - - #[getter] - fn supported_request_types(&self, py: Python<'_>) -> PyResult>> { - let backend = self.inner.as_ref().ok_or_else(|| { - PyNotImplementedError::new_err("LLMBackend.supported_request_types must be implemented") - })?; - backend - .supported_request_types() - .iter() - .map(|request_type| request_type_object(py, *request_type)) - .collect() - } - - fn startup<'py>(&self, py: Python<'py>) -> PyResult> { - match self.inner.clone() { - Some(backend) => pyo3_async_runtimes::tokio::future_into_py(py, async move { - backend.startup().await.map_err(py_core_error) - }), - None => pyo3_async_runtimes::tokio::future_into_py(py, async { Ok(()) }), - } - } - - fn shutdown<'py>(&self, py: Python<'py>) -> PyResult> { - match self.inner.clone() { - Some(backend) => pyo3_async_runtimes::tokio::future_into_py(py, async move { - backend.shutdown().await.map_err(py_core_error) - }), - None => pyo3_async_runtimes::tokio::future_into_py(py, async { Ok(()) }), - } - } - - fn call<'py>( - &self, - py: Python<'py>, - ctx: &Bound<'_, PyAny>, - request: &Bound<'_, PyAny>, - ) -> PyResult> { - let backend = self - .inner - .clone() - .ok_or_else(|| PyNotImplementedError::new_err("LLMBackend.call must be implemented"))?; - let mut lease = lease_from_python(ctx)?; - let request = request_from_python(request)?; - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let result = backend.call(lease.context_mut()?, &request).await; - let restore_result = lease.restore(); - let response = result.map_err(py_core_error)?; - restore_result?; - Python::attach(|py| response_to_python(py, response)) - }) - } - - fn __repr__(&self) -> &'static str { - "LLMBackend()" - } -} - -fn reject_base_instantiation( - cls: &Bound<'_, PyType>, - name: &'static str, -) -> PyResult<()> { - if cls.is(cls.py().get_type::()) { - return Err(PyTypeError::new_err(format!( - "can't instantiate abstract role {name}" - ))); - } - Ok(()) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - Ok(()) -} diff --git a/crates/switchyard-py/src/interop/subagent.rs b/crates/switchyard-py/src/interop/subagent.rs deleted file mode 100644 index c74885cda..000000000 --- a/crates/switchyard-py/src/interop/subagent.rs +++ /dev/null @@ -1,51 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Python binding for the canonical sub-agent detection policy. -//! -//! Routing implementations must not sniff lineage headers themselves: the fact (explicit -//! `x-switchyard-is-subagent`, Claude Code agent lineage, Codex/relay markers) and the -//! work-vs-maintenance policy both live in the protocol crate, so every engine — the libsy -//! classifier and the server alike — answers "is this delegated work?" -//! identically. - -use std::collections::HashMap; - -use http::header::{HeaderName, HeaderValue}; -use pyo3::exceptions::PyValueError; -use pyo3::prelude::*; - -/// Convert Python-owned headers without allowing invalid or excessive entries. -pub(crate) fn header_map_from_python( - headers: &HashMap, -) -> PyResult { - let mut result = http::HeaderMap::new(); - - for (name, value) in headers { - let name = HeaderName::from_bytes(name.as_bytes()) - .map_err(|error| PyValueError::new_err(error.to_string()))?; - let value = HeaderValue::from_str(value) - .map_err(|error| PyValueError::new_err(error.to_string()))?; - result - .try_append(name, value) - .map_err(|error| PyValueError::new_err(error.to_string()))?; - } - - Ok(result) -} - -/// Whether `headers` mark this request as delegated sub-agent *work*. -/// -/// Wraps [`switchyard_protocol::Metadata::from_headers`] for the lineage fact and -/// `is_subagent_work` for the kind policy, so harness maintenance turns (Codex `compact`, -/// `memory_consolidation`) stay on normal routing rather than being sent to a worker target. -#[pyfunction] -fn is_subagent_request(headers: HashMap) -> PyResult { - let headers = header_map_from_python(&headers)?; - Ok(switchyard_protocol::Metadata::from_headers(&headers).is_subagent_work()) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(is_subagent_request, module)?)?; - Ok(()) -} diff --git a/crates/switchyard-py/src/lib.rs b/crates/switchyard-py/src/lib.rs index 3082697a1..1749c9ce4 100644 --- a/crates/switchyard-py/src/lib.rs +++ b/crates/switchyard-py/src/lib.rs @@ -3,20 +3,14 @@ use pyo3::prelude::*; -mod component_bindings; mod errors; -mod interop; mod libsy_bindings; mod py_serde; mod server_bindings; -mod translation; #[pymodule] fn _switchyard_rust(module: &Bound<'_, PyModule>) -> PyResult<()> { errors::register(module)?; - translation::register(module)?; - interop::register(module)?; - component_bindings::register(module)?; libsy_bindings::register(module)?; server_bindings::register(module)?; Ok(()) diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs index b4ce79d65..6ecdfa7e3 100644 --- a/crates/switchyard-py/src/libsy_bindings.rs +++ b/crates/switchyard-py/src/libsy_bindings.rs @@ -3,9 +3,11 @@ //! Minimal Python API for running Rust-owned libsy algorithms. +use std::collections::HashMap; use std::sync::Arc; use async_trait::async_trait; +use http::header::{HeaderName, HeaderValue}; use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::*; use serde_json::{Value, json}; @@ -21,9 +23,23 @@ use switchyard_protocol::{ }; use crate::errors::py_libsy_error; -use crate::interop::subagent::header_map_from_python; use crate::py_serde::{from_python, to_python}; +/// Convert Python-owned headers into the request metadata expected by libsy. +fn header_map_from_python(headers: &HashMap) -> PyResult { + let mut result = http::HeaderMap::new(); + for (name, value) in headers { + let name = HeaderName::from_bytes(name.as_bytes()) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + let value = HeaderValue::from_str(value) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + result + .try_append(name, value) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + } + Ok(result) +} + /// Adapts a Python object with `async call(request)` to libsy. struct PythonLlmClient { inner: Py, diff --git a/crates/switchyard-py/src/py_serde.rs b/crates/switchyard-py/src/py_serde.rs index f02f962d3..8a79e16bd 100644 --- a/crates/switchyard-py/src/py_serde.rs +++ b/crates/switchyard-py/src/py_serde.rs @@ -8,7 +8,6 @@ use pyo3::prelude::*; use pyo3::types::PyDict; use pythonize::{depythonize, pythonize}; use serde::{Serialize, de::DeserializeOwned}; -use serde_json::Value; /// Converts a Python mapping-like object into a Serde-owned Rust value. pub(crate) fn from_python(value: &Bound<'_, PyAny>) -> PyResult { @@ -24,16 +23,6 @@ pub(crate) fn to_python(py: Python<'_>, value: &T) -> PyResult) -> PyResult { - from_python(value) -} - -/// Converts a JSON value into a Python object. -pub(crate) fn value_to_python(py: Python<'_>, value: &Value) -> PyResult> { - to_python(py, value) -} - fn jsonable_python(value: &Bound<'_, PyAny>) -> PyResult> { if let Ok(model_dump) = value.getattr("model_dump") && model_dump.is_callable() diff --git a/crates/switchyard-py/src/translation.rs b/crates/switchyard-py/src/translation.rs deleted file mode 100644 index 5f86d7bf5..000000000 --- a/crates/switchyard-py/src/translation.rs +++ /dev/null @@ -1,123 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! PyO3 bindings for `switchyard-translation`. - -use pyo3::prelude::*; -use serde_json::Value; -use switchyard_translation::{ - StreamTranslationState, TranslationEngine, TranslationPolicy, normalize_anthropic_tool_use_ids, -}; - -use crate::errors::py_translation_error; -use crate::py_serde::{value_from_python, value_to_python}; - -#[pyclass(name = "TranslationEngine")] -struct PyTranslationEngine { - inner: TranslationEngine, - policy: TranslationPolicy, -} - -#[pymethods] -impl PyTranslationEngine { - #[new] - fn new() -> Self { - Self { - inner: TranslationEngine::default(), - policy: TranslationPolicy::default(), - } - } - - fn translate_request( - &self, - py: Python<'_>, - source: &str, - target: &str, - body: &Bound<'_, PyAny>, - ) -> PyResult> { - let body = value_from_python(body)?; - let output = self - .inner - .translate_request(source, target, &body, &self.policy) - .map_err(py_translation_error)?; - value_to_python(py, &output.body) - } - - fn translate_response( - &self, - py: Python<'_>, - source: &str, - target: &str, - body: &Bound<'_, PyAny>, - ) -> PyResult> { - let body = value_from_python(body)?; - let output = self - .inner - .translate_response(source, target, &body, &self.policy) - .map_err(py_translation_error)?; - value_to_python(py, &output.body) - } - - #[pyo3(signature = (source, target, model=None, message_id=None))] - fn stream( - &self, - source: &str, - target: &str, - model: Option, - message_id: Option, - ) -> PyStreamTranslation { - let mut state = StreamTranslationState::new(source, target); - state.target_model = model; - state.target_message_id = message_id; - PyStreamTranslation { - source: source.to_string(), - target: target.to_string(), - state, - inner: TranslationEngine::default(), - } - } - - fn normalize_anthropic_tool_use_ids( - &self, - py: Python<'_>, - messages: &Bound<'_, PyAny>, - ) -> PyResult> { - let messages = value_from_python(messages)?; - value_to_python(py, &normalize_anthropic_tool_use_ids(messages)) - } -} - -#[pyclass(name = "StreamTranslation")] -struct PyStreamTranslation { - source: String, - target: String, - state: StreamTranslationState, - inner: TranslationEngine, -} - -#[pymethods] -impl PyStreamTranslation { - fn translate_event(&mut self, py: Python<'_>, event: &Bound<'_, PyAny>) -> PyResult> { - let event = value_from_python(event)?; - let output = self - .inner - .translate_event(&mut self.state, &self.source, &self.target, &event) - .map_err(py_translation_error)?; - value_to_python(py, &Value::Array(output)) - } - - fn finish(&mut self, py: Python<'_>) -> PyResult> { - let output = self - .inner - .finish_stream(&mut self.state, &self.target) - .map_err(py_translation_error)?; - value_to_python(py, &Value::Array(output)) - } -} - -/// Registers translation bindings with the native Python module. -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - Ok(()) -} diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index b5960eb71..c06d13160 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -165,11 +165,8 @@ fixed error categories. `switchyard_total_latency_ms` observes an aggregate when it becomes available or a stream when it ends cleanly. Its clock starts in a router-wide middleware, before the request body is read and -decoded, so it covers the same span as the Python server's request-ingress-to-completion -measurement. It still excludes connection accept and TLS handshake, which hyper completes before -the server sees the request. The Rust server exports this metric as a histogram, while the Python -server exports its counterpart as a summary; this matches the existing histogram/summary difference -for model-call latency. +decoded, so it measures request ingress through response completion. It still excludes connection +accept and TLS handshake, which hyper completes before the server sees the request. `switchyard_routing_overhead_ms` is what routing cost on top of the model call: the algorithm's run time minus the call that served the request. Classifier calls are not subtracted, so an diff --git a/docs/cli_reference.md b/docs/cli_reference.md index 9f50c693e..b2374ecd7 100644 --- a/docs/cli_reference.md +++ b/docs/cli_reference.md @@ -9,7 +9,7 @@ Switchyard has two command-line paths: ## Launcher Path: `switchyard launch` -Install the launcher with `uv tool install --python 3.12 "nemo-switchyard[cli,server]"`. The +Install the launcher with `uv tool install --python 3.12 "nemo-switchyard[cli]"`. The selected coding agent must also be installed and available on `PATH`. ### Usage @@ -83,7 +83,8 @@ switchyard-server --config routes.toml \ ## Removed Setup Commands -`switchyard configure` and `switchyard verify` are not available. Export the +`switchyard configure`, `switchyard serve`, and `switchyard verify` are not +available. Export the environment variable named by `api_key_env` in the native TOML deployment, then pass that deployment on each run. For example: @@ -97,7 +98,8 @@ export PROVIDER_API_KEY="your-provider-key" # pragma: allowlist secret switchyard launch claude --model my-route --config routes.toml ``` -The CLI does not save provider credentials or deployment paths. +The CLI does not save provider credentials, deployment paths, or host a +standalone server. Use `switchyard-server --config routes.toml --dry-run` to validate a native deployment before starting the standalone server. diff --git a/docs/getting_started.md b/docs/getting_started.md index fc03dad73..74207065f 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -27,11 +27,11 @@ source "$HOME/.local/bin/env" Then install the published Switchyard tool: ```bash -uv tool install --python 3.12 "nemo-switchyard[cli,server]" +uv tool install --python 3.12 "nemo-switchyard[cli]" ``` This creates an isolated Python tool environment containing the `switchyard` -CLI, its CLI and server dependencies, and the packaged PyO3 Rust extension. +CLI, its launcher dependency, and the packaged PyO3 Rust extension. `switchyard launch` starts the native Rust server through that extension; this path does not install or run the standalone `switchyard-server` binary. @@ -122,8 +122,7 @@ Cargo builds the release binary and installs it into `~/.cargo/bin` by default. ### Configure -The Rust server reads an explicit TOML file. It does not use the Python -server's minimal YAML route bundle. +The Rust server reads an explicit TOML file. Create `routes.toml` with an LLM-classifier route: diff --git a/docs/internal/metrics_reference.md b/docs/internal/metrics_reference.md index aea807d9e..0d29bb59f 100644 --- a/docs/internal/metrics_reference.md +++ b/docs/internal/metrics_reference.md @@ -14,8 +14,7 @@ a drop-in scrape config and starter alert rules. | Auth | None | | Default scrape interval | 15s | -`GET /metrics` is served by the Python route-bundle server started with -`switchyard serve --routes PATH` and by the native Rust server. +`GET /metrics` is served by the native Rust server. A JSON summary of the same traffic lives at `GET /v1/stats`. diff --git a/docs/routing_algorithms/stage_router_routing.md b/docs/routing_algorithms/stage_router_routing.md index 207586134..1549d8e87 100644 --- a/docs/routing_algorithms/stage_router_routing.md +++ b/docs/routing_algorithms/stage_router_routing.md @@ -151,31 +151,10 @@ From the overlap tasks (those with both capable and efficient results): - `SAFE` = both pass - `HARD` = both fail -**Running the sweep** - -Replay your runs through the real Rust scorer and picker with -`benchmark/score_staged_run.py` (the `switchyard-stage-router-scorer` skill). It emits -per-turn scores and per-task routing splits at a given threshold and window — -the actual `capable_first` / `efficient_first` picker decisions, not a -counterfactual: - -```bash -# Score a probe run at a candidate threshold -uv run python benchmark/score_staged_run.py --run benchmark/tb_runs/ \ - --threshold 0.5 --window 3 -# → /tmp/-scores.jsonl (per turn: score, confidence, pick_cf, pick_ef) -# → /tmp/-per-task.csv (per task: routing split, mean score/confidence) -``` - -Sweep a few candidate thresholds and read the routing split and pass rate off -the per-task CSV; the lowest threshold that rescues the RESCUE quadrant without -over-escalating the LOSS quadrant is your calibrated value. Because the scorer -is corroborative, a `0.5` threshold takes ~1.5 signals of agreement — a policy -that escalates ~20% of tasks maps roughly to `confidence_threshold: 0.5` with -`capable_first`. - -Signals come from the actual picker replay, so even 15–20 probe tasks give a -stable result. +Sweep a few candidate thresholds in representative benchmark runs. Choose the +lowest threshold that rescues the RESCUE quadrant without over-escalating the +LOSS quadrant. Because the scorer is corroborative, a `0.5` threshold takes +roughly 1.5 signals of agreement. **Caveat on efficient outcomes in stage-router vs. pure-efficient** diff --git a/examples/minimal.py b/examples/minimal.py deleted file mode 100644 index 95b5a610b..000000000 --- a/examples/minimal.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -""" -Minimal Switchyard Example - -This example demonstrates how to use Switchyard as a Python library -to route LLM requests through a backend. - -Usage: - export OPENROUTER_API_KEY="sk-or-..." - python examples/minimal.py -""" - -import asyncio -import sys -from pathlib import Path - -# Add package to path for development (not needed when installed via pip) -sys.path.insert(0, str(Path(__file__).parent.parent)) - -from switchyard import ChatRequest -from switchyard.cli.route_bundle import load_route_bundle_table - - -async def main() -> None: - """Run a minimal Switchyard example.""" - - routes = load_route_bundle_table(Path(__file__).with_name("route.yaml")) - switchyard = routes.lookup_switchyard("fast-kimi") - - print("=" * 60) - print("Switchyard Minimal Example") - print("=" * 60) - - # Create a chat request - request = ChatRequest.openai_chat({ - "model": "fast-kimi", - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What is 2+2?"}, - ], - "max_tokens": 100, - }) - - print(f"Sending request to {request.body['model']}...") - - # Call the LLM through the switchyard - response = await switchyard.call(request) - - print("\nResponse:") - print(f" Content: {response['choices'][0]['message']['content']}") - print(f" Tokens: {response['usage']['total_tokens']}") - - print("\n" + "=" * 60) - print("Example completed!") - print("=" * 60) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/examples/route.yaml b/examples/route.yaml deleted file mode 100644 index 5b3ef858e..000000000 --- a/examples/route.yaml +++ /dev/null @@ -1,12 +0,0 @@ -defaults: - api_key: ${OPENROUTER_API_KEY} - base_url: https://openrouter.ai/api/v1 - format: openai - -routes: - fast-kimi: - type: passthrough - target: moonshotai/kimi-k2.6 - - smoke-test: - type: noop diff --git a/examples/utils.py b/examples/utils.py deleted file mode 100644 index 5d9527b1b..000000000 --- a/examples/utils.py +++ /dev/null @@ -1,97 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -""" -Utility functions for Switchyard examples. - -This module provides common configuration loading for all examples. -""" - -import json -import os -from pathlib import Path - -# Repository root (parent of examples folder) -REPO_ROOT = Path(__file__).parent.parent - -# Path to secrets file -SECRETS_FILE = REPO_ROOT / "secrets" / "secrets.json" - -# Defaults for OpenAI API -DEFAULT_BASE_URL = "https://api.openai.com/v1" -DEFAULT_MODEL = "gpt-4o-mini" - - -def load_secrets() -> dict: - """ - Load secrets from secrets/secrets.json if it exists. - - Returns: - Dictionary with secrets, or empty dict if file doesn't exist. - """ - if SECRETS_FILE.exists(): - with open(SECRETS_FILE) as f: - return json.load(f) - return {} - - -def get_config() -> dict: - """ - Get API configuration from environment variables or secrets file. - - Priority: - 1. Environment variables (OPENAI_API_KEY, OPENAI_BASE_URL) - 2. Secrets file (secrets/secrets.json) - 3. Defaults for base_url and model - - Returns: - Dictionary with api_key, base_url, and model. - """ - secrets = load_secrets() - openai_secrets = secrets.get("openai", {}) - - return { - "api_key": os.environ.get("OPENAI_API_KEY") or openai_secrets.get("api_key"), - "base_url": ( - os.environ.get("OPENAI_BASE_URL") - or openai_secrets.get("base_url") - or DEFAULT_BASE_URL - ), - "model": ( - os.environ.get("OPENAI_MODEL") - or openai_secrets.get("model") - or DEFAULT_MODEL - ), - } - - -def print_config(config: dict) -> None: - """Print configuration in a consistent format.""" - api_key = config["api_key"] - print("Configuration:") - if api_key: - print(f" API Key: {'*' * 8}...{api_key[-4:]}") - else: - print(" API Key: Not set") - print(f" Base URL: {config['base_url']}") - print(f" Model: {config['model']}") - print() - - -def check_api_key(config: dict) -> bool: - """ - Check if API key is available and print error message if not. - - Returns: - True if API key is available, False otherwise. - """ - if config["api_key"]: - return True - - print("Error: API key not found.") - print() - print("Set it via environment variable:") - print(" export OPENAI_API_KEY='sk-...'") - print() - print("Or create secrets/secrets.json (copy from secrets/secrets.template.json)") - return False diff --git a/pyproject.toml b/pyproject.toml index 30203d106..857dbb893 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,51 +23,15 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules", ] -# Core dependencies — minimal set for the foundation library (core/ + recipes/). -# No server, CLI, ML, GPU, or orchestrator packages. -# Users install extras based on their use case: -# pip install nemo-switchyard # Core + recipes only -# pip install nemo-switchyard[server] # Add FastAPI/Uvicorn for e2e -# pip install nemo-switchyard[cli] # Add prompt-toolkit for CLI -# pip install nemo-switchyard[all] # Everything -dependencies = [ - # openai: request/response schema types, plus the async client in lib/llm_client.py. - # Keep this floor low enough for downstream consumers to co-install us; NeMo Gym pins - # openai<=2.7.2. The suite passes unchanged from 2.7.0 through 2.48.0. - "openai>=2.7,<3.0", - "anthropic>=0.99.0,<1.0", - "httpx>=0.28.1,<1.0", - "pydantic>=2.13.3,<3.0", -] +dependencies = [] [project.optional-dependencies] -# Server dependencies — FastAPI + Uvicorn for e2e users who want to run -# switchyard as a proxy. Not needed for library-only usage. -server = [ - "fastapi>=0.136.1,<1.0", - "uvicorn[standard]>=0.46.0,<1.0", - "sse-starlette>=3.4.1,<4.0", -] - # CLI dependencies — prompt-toolkit for Claude Code launcher and ShellTUI. # Only needed for users running the switchyard CLI. cli = [ "prompt-toolkit>=3.0.52,<4.0", ] -# Tracing — ddtrace for Datadog APM spans on the proxy routing path -# Optional: ``switchyard.lib.tracing`` no-ops when ddtrace is not -# importable, so the default install and non-Datadog deployments are unaffected -# and pay no overhead. -tracing = [ - "ddtrace>=2.9,<4", -] - -# Everything — all optional dependencies for full-featured deployment. -all = [ - "nemo-switchyard[server,cli,tracing]", -] - # Dev tooling lives in a PEP 735 dependency group rather than an optional # extra so it is not advertised in the published wheel's METADATA. This # keeps pytest / ruff / mypy / pygments-via-pytest out of downstream @@ -82,19 +46,13 @@ dev = [ "maturin>=1.9,<2.0", "pytest>=9.0.3,<10.0", "pytest-asyncio>=1.3.0,<2.0", - "pytest-cov>=7.1.0,<8.0", - "pytest-mock>=3.15.1,<4.0", "pytest-timeout>=2.4.0,<3.0", - "respx>=0.23.1,<1.0", "ruff>=0.15.12,<1.0", "mypy>=1.20.2,<2.0", - "httpx>=0.28.1,<1.0", - "socksio>=1.0.0,<2.0", - "prometheus-client>=0.21.0,<1.0", # Harbor is needed for local evaluation runs and supports the package floor. # Package users still do not see it in published metadata. "harbor @ git+https://github.com/harbor-framework/harbor.git@v0.6.4 ; python_version >= '3.12'", - "nemo-switchyard[server]", + "nemo-switchyard[cli]", "pytest-markdown-docs>=0.9.2", ] docs = [ @@ -122,8 +80,6 @@ include = ["switchyard*"] switchyard = [ "py.typed", "cli/defaults/*.toml", - "lib/processors/prompts/*.md", - "lib/processors/stage_router/prompts/*.md", ] switchyard_rust = ["py.typed", "*.pyi"] @@ -158,23 +114,11 @@ warn_unused_configs = true warn_unreachable = true ignore_missing_imports = true -# Problem 2 scoreboard: every `data: Any` annotation is a tracked error. -# 303 violations today — each one fixed = one step toward typed chain. -# Disabled for open-source release; can be re-enabled as part of gradual typing effort. -# disallow_any_explicit = true - # Readability pretty = true show_error_codes = true show_column_numbers = true -# TODO: enable after Phase 6 migration (3,271 violations today) -# disallow_any_expr = true - -[[tool.mypy.overrides]] -module = "tests.*" -ignore_errors = true - [tool.pyright] include = ["switchyard"] exclude = [ diff --git a/switchyard/__init__.py b/switchyard/__init__.py index ae1574d2f..d978b6355 100644 --- a/switchyard/__init__.py +++ b/switchyard/__init__.py @@ -1,146 +1,14 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -""" -Switchyard - Typed LLM routing and orchestration. - -This library provides a composable, type-safe foundation for routing -requests across multiple LLM backends with intelligent tier selection, -format translation, and extensible middleware. -""" +"""Switchyard's Python launcher and libsy bindings.""" from importlib import metadata as _metadata -from typing import TYPE_CHECKING, Any - -from switchyard.lib.backends import ( - AnthropicNativeBackend, - OpenAiNativeBackend, -) -from switchyard.lib.backends.llm_target import ( - BackendFormat, - LlmTarget, -) -from switchyard.lib.chat_request import ( - AnthropicChatRequest, - OpenAIChatRequest, - ResponsesChatRequest, -) -from switchyard.lib.chat_response import ( - AnthropicChatResponse, - AnthropicResponseStream, - AnthropicStreamingChatResponse, - AnyResponseStream, - CompletionChatResponse, - ResponsesApiChatResponse, - ResponsesApiStream, - ResponsesApiStreamingChatResponse, - ResponseStream, - StreamingChatResponse, -) -from switchyard.lib.processors.rl_logging_request_processor import RlLoggingRequestProcessor -from switchyard.lib.processors.rl_logging_response_processor import RlLoggingResponseProcessor -from switchyard.lib.request_metadata import RequestMetadata -from switchyard.lib.roles import ( - LLMBackend, -) -from switchyard.lib.route_table import RouteTable -from switchyard.lib.switchyard import Switchyard -from switchyard_rust.components import RandomRoutingProcessorConfig -from switchyard_rust.core import ( - ChatRequest, - ChatRequestType, - ChatResponse, - ChatResponseType, -) -from switchyard_rust.translation import TranslationEngine - -if TYPE_CHECKING: - from switchyard.lib.endpoints.anthropic_messages_endpoint import ( - AnthropicMessagesEndpoint, - ) - from switchyard.lib.endpoints.models_endpoint import ModelsEndpoint - from switchyard.lib.endpoints.openai_chat_endpoint import ( - OpenAIChatEndpoint, - ) - from switchyard.lib.endpoints.responses_endpoint import ResponsesEndpoint - from switchyard.server.switchyard_app import build_switchyard_app - - -def __getattr__(name: str) -> Any: - """Lazy-load optional server exports that require the ``server`` extra.""" - if name == "OpenAIChatEndpoint": - from switchyard.lib.endpoints.openai_chat_endpoint import ( - OpenAIChatEndpoint, - ) - - return OpenAIChatEndpoint - if name == "AnthropicMessagesEndpoint": - from switchyard.lib.endpoints.anthropic_messages_endpoint import ( - AnthropicMessagesEndpoint, - ) - - return AnthropicMessagesEndpoint - if name == "ResponsesEndpoint": - from switchyard.lib.endpoints.responses_endpoint import ResponsesEndpoint - - return ResponsesEndpoint - if name == "ModelsEndpoint": - from switchyard.lib.endpoints.models_endpoint import ModelsEndpoint - - return ModelsEndpoint - if name == "build_switchyard_app": - from switchyard.server.switchyard_app import build_switchyard_app - - return build_switchyard_app - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - - -__all__ = [ - # ChatRequest types - "AnthropicChatRequest", - "ChatRequest", - "ChatRequestType", - "OpenAIChatRequest", - "ResponsesChatRequest", - # Chain infrastructure - "Switchyard", - "LLMBackend", - "AnthropicNativeBackend", - "OpenAiNativeBackend", - "OpenAIChatEndpoint", - "AnthropicMessagesEndpoint", - "ResponsesEndpoint", - "ModelsEndpoint", - "build_switchyard_app", - # Route dispatch table - "RouteTable", - "RequestMetadata", - "RlLoggingRequestProcessor", - "RlLoggingResponseProcessor", - # Random Routing usage case - "BackendFormat", - "RandomRoutingProcessorConfig", - "LlmTarget", - # Deterministic (LLM-classifier) routing usage case - # Translation engine - "TranslationEngine", - # ChatResponse types - "AnthropicChatResponse", - "ChatResponse", - "ChatResponseType", - "CompletionChatResponse", - "StreamingChatResponse", - "ResponsesApiChatResponse", - "ResponsesApiStreamingChatResponse", - "AnthropicStreamingChatResponse", - "ResponseStream", - "ResponsesApiStream", - "AnthropicResponseStream", - "AnyResponseStream", -] try: __version__ = _metadata.version("nemo-switchyard") except _metadata.PackageNotFoundError: # A source checkout may not have installed distribution metadata. __version__ = "0.0.0+unknown" + +__all__ = ["__version__"] diff --git a/switchyard/cli/command_utils.py b/switchyard/cli/command_utils.py index 1539a2124..1b826383c 100644 --- a/switchyard/cli/command_utils.py +++ b/switchyard/cli/command_utils.py @@ -3,15 +3,6 @@ """Small helpers shared by the remaining CLI commands.""" -import logging - - -def quiet_dependency_loggers() -> None: - """Keep third-party INFO logs out of launcher terminal UIs.""" - - for noisy in ("httpx", "httpcore", "openai", "anthropic", "uvicorn.access"): - logging.getLogger(noisy).setLevel(logging.WARNING) - def strip_forwarded_args(args: list[str] | None) -> list[str]: """Strip argparse's leading ``--`` sentinel from forwarded arguments.""" @@ -22,4 +13,4 @@ def strip_forwarded_args(args: list[str] | None) -> list[str]: return forwarded -__all__ = ["quiet_dependency_loggers", "strip_forwarded_args"] +__all__ = ["strip_forwarded_args"] diff --git a/switchyard/cli/launchers/claude_code_launcher.py b/switchyard/cli/launchers/claude_code_launcher.py index 4b69ab298..42279731d 100644 --- a/switchyard/cli/launchers/claude_code_launcher.py +++ b/switchyard/cli/launchers/claude_code_launcher.py @@ -22,7 +22,7 @@ from switchyard.cli.launchers.native_server import NativeServer from switchyard.cli.launchers.proxy_health_monitor import ProxyHealthMonitor from switchyard.cli.launchers.session_summary import print_session_summary -from switchyard.server.shell_tui import ShellTUI +from switchyard.cli.launchers.shell_tui import ShellTUI logger = logging.getLogger(__name__) diff --git a/switchyard/cli/launchers/codex_cli_launcher.py b/switchyard/cli/launchers/codex_cli_launcher.py index 973835d7d..78c85f659 100644 --- a/switchyard/cli/launchers/codex_cli_launcher.py +++ b/switchyard/cli/launchers/codex_cli_launcher.py @@ -30,7 +30,7 @@ from switchyard.cli.launchers.native_server import NativeServer from switchyard.cli.launchers.proxy_health_monitor import ProxyHealthMonitor from switchyard.cli.launchers.session_summary import print_session_summary -from switchyard.server.shell_tui import ShellTUI +from switchyard.cli.launchers.shell_tui import ShellTUI logger = logging.getLogger(__name__) diff --git a/switchyard/lib/cost_estimator.py b/switchyard/cli/launchers/cost_estimator.py similarity index 99% rename from switchyard/lib/cost_estimator.py rename to switchyard/cli/launchers/cost_estimator.py index ed86c00f2..642da5e25 100644 --- a/switchyard/lib/cost_estimator.py +++ b/switchyard/cli/launchers/cost_estimator.py @@ -9,10 +9,9 @@ Usage:: - from switchyard.lib.cost_estimator import estimate_cost, MODEL_PRICING + from switchyard.cli.launchers.cost_estimator import estimate_cost - stats = requests.get("http://localhost:4000/v1/routing/stats").json() - breakdown = estimate_cost(stats["models"]) + breakdown = estimate_cost({"model": {"prompt_tokens": 10}}) print(f"Total: ${breakdown['total_cost']:.4f}") See ``docs.anthropic.com/en/docs/about-claude/pricing`` for the diff --git a/switchyard/cli/launchers/launcher_runtime.py b/switchyard/cli/launchers/launcher_runtime.py index 4584d1235..8a5149aee 100644 --- a/switchyard/cli/launchers/launcher_runtime.py +++ b/switchyard/cli/launchers/launcher_runtime.py @@ -69,20 +69,13 @@ def configure_debug_file_logging(*, display_model: str) -> Path: handler.close() root.setLevel(logging.WARNING) - for name in ( - "switchyard", - "httpx", - "httpcore", - "openai", - "anthropic", - ): - logger = logging.getLogger(name) - for handler in logger.handlers[:]: - logger.removeHandler(handler) - handler.close() - logger.addHandler(file_handler) - logger.setLevel(logging.DEBUG) - logger.propagate = False + switchyard_logger = logging.getLogger("switchyard") + for handler in switchyard_logger.handlers[:]: + switchyard_logger.removeHandler(handler) + handler.close() + switchyard_logger.addHandler(file_handler) + switchyard_logger.setLevel(logging.DEBUG) + switchyard_logger.propagate = False logging.getLogger("switchyard").info( "=== switchyard debug log: model=%s pid=%d ===", @@ -94,14 +87,7 @@ def configure_debug_file_logging(*, display_model: str) -> Path: def silence_launch_loggers(*, local_logger: logging.Logger) -> None: """Keep dependency chatter out of a child process terminal UI.""" - for noisy in ( - "switchyard", - "httpx", - "httpcore", - "openai", - "anthropic", - ): - logging.getLogger(noisy).setLevel(logging.WARNING) + logging.getLogger("switchyard").setLevel(logging.WARNING) local_logger.setLevel(logging.INFO) @@ -113,30 +99,6 @@ def stdin_is_tty() -> bool: return False -def route_bundle_strategy_summary(route_bundle: str, default_model: str) -> str: - """Describe the default route in a Python server bundle.""" - try: - from collections.abc import Mapping as _Mapping - from importlib import import_module - yaml = import_module("yaml") - raw = yaml.safe_load(Path(route_bundle).read_text()) - routes = raw.get("routes") if isinstance(raw, dict) else None - if isinstance(routes, _Mapping) and routes: - first_key = next(iter(routes)) - route = routes[first_key] - route_type = route.get("type") if isinstance(route, _Mapping) else None - if isinstance(route_type, str): - if route_type == "noop": - return "noop" - if route_type == "passthrough": - target = route.get("target") - model = target.get("model") if isinstance(target, _Mapping) else target - return f"passthrough: model={model or first_key}" - except Exception: - pass - return f"route: {default_model}" - - # Keys that are abbreviated in the banner display. _KEY_ABBREV: dict[str, str] = { "confidence_threshold": "conf", diff --git a/switchyard/cli/launchers/live_stats_footer.py b/switchyard/cli/launchers/live_stats_footer.py index ade41d6c6..89855e7ca 100644 --- a/switchyard/cli/launchers/live_stats_footer.py +++ b/switchyard/cli/launchers/live_stats_footer.py @@ -3,8 +3,8 @@ """Shared live token-usage footer for launcher TUI sessions. -One layout for every routing strategy: an aggregate row across all chains plus -one indented row per active outbound model tier. +One layout for every routing strategy: an aggregate row across all requests +plus one indented row per active outbound model. """ from __future__ import annotations @@ -14,13 +14,12 @@ from switchyard.cli.launchers.proxy_health_monitor import ProxyHealthMonitor from switchyard.cli.launchers.stats_source import StatsSource -from switchyard.lib.route_table import RouteTable FOOTER_ROWS = 2 class LiveStatsFooter: - """Live stats footer: aggregate row + one row per active model tier.""" + """Live stats footer: aggregate row plus one row per active model.""" def __init__( self, @@ -28,13 +27,11 @@ def __init__( model: str, health: ProxyHealthMonitor, *, - table: RouteTable | None = None, strategy_label: str | None = None, ) -> None: self._stats = stats self._default_model_short = model.rsplit("/", 1)[-1] self._health = health - self._table = table self._strategy_label = strategy_label # Ordered list of models seen in traffic so far. Grows as new tiers # receive their first request; order is first-seen, which keeps the @@ -44,7 +41,7 @@ def __init__( @property def height(self) -> int: - """Current footer height: 1 aggregate row + 1 row per seen tier (min 2).""" + """Current footer height: one aggregate plus each seen model (minimum two).""" return FOOTER_ROWS - 1 + max(1, len(self._seen_models)) def as_footer_fn(self) -> Callable[[int], list[tuple[str, int]]]: @@ -87,20 +84,21 @@ def _aggregate_row(self, snapshot: Mapping[str, object]) -> tuple[str, int]: def _tier_rows( self, snapshot: Mapping[str, object], ) -> list[tuple[str, int]]: - """Return one row per model tier that has received traffic. + """Return one row per model that has received traffic. - Before any traffic lands, returns a single placeholder row using the - table's last-looked-up id or the launch default. Once traffic - arrives, the list grows to match the number of distinct models seen, - in first-seen order, and never shrinks. + Before traffic lands, a placeholder uses the launch model. Once traffic + arrives, rows grow in first-seen order and never shrink. """ models = _mapping(snapshot, "models") if not models: - fallback = ( - (self._table.last_looked_up if self._table else None) - or self._default_model_short - ) - return [_model_row(fallback, calls=0, errors=0, prompt=0, completion=0, cached=0)] + return [_model_row( + self._default_model_short, + calls=0, + errors=0, + prompt=0, + completion=0, + cached=0, + )] for m in models: if m not in self._seen_set: diff --git a/switchyard/cli/launchers/openclaw_launcher.py b/switchyard/cli/launchers/openclaw_launcher.py index da5bcbc25..a0dbcc87a 100644 --- a/switchyard/cli/launchers/openclaw_launcher.py +++ b/switchyard/cli/launchers/openclaw_launcher.py @@ -26,7 +26,7 @@ from switchyard.cli.launchers.native_server import NativeServer from switchyard.cli.launchers.proxy_health_monitor import ProxyHealthMonitor from switchyard.cli.launchers.session_summary import print_session_summary -from switchyard.server.shell_tui import ShellTUI +from switchyard.cli.launchers.shell_tui import ShellTUI logger = logging.getLogger(__name__) diff --git a/switchyard/cli/launchers/session_summary.py b/switchyard/cli/launchers/session_summary.py index e676782a1..681ae466b 100644 --- a/switchyard/cli/launchers/session_summary.py +++ b/switchyard/cli/launchers/session_summary.py @@ -10,8 +10,8 @@ from collections.abc import Mapping from typing import cast +from switchyard.cli.launchers.cost_estimator import estimate_model_cost from switchyard.cli.launchers.stats_source import StatsSource -from switchyard.lib.cost_estimator import estimate_model_cost _RULE = "─" * 51 _LOG = logging.getLogger(__name__) diff --git a/switchyard/server/shell_tui.py b/switchyard/cli/launchers/shell_tui.py similarity index 100% rename from switchyard/server/shell_tui.py rename to switchyard/cli/launchers/shell_tui.py diff --git a/switchyard/cli/model_catalog/__init__.py b/switchyard/cli/model_catalog/__init__.py deleted file mode 100644 index a6b4c862b..000000000 --- a/switchyard/cli/model_catalog/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Model catalog discovery and selection helpers.""" - diff --git a/switchyard/cli/model_catalog/model_discovery.py b/switchyard/cli/model_catalog/model_discovery.py deleted file mode 100644 index 24f29593d..000000000 --- a/switchyard/cli/model_catalog/model_discovery.py +++ /dev/null @@ -1,50 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""OpenAI-compatible model discovery for YAML route bundles.""" - -import httpx - - -class ModelDiscoveryError(RuntimeError): - """Raised when an upstream model catalog cannot be fetched.""" - - -def fetch_model_ids( - base_url: str, - api_key: str, - timeout_s: float = 10.0, -) -> list[str]: - """Return sorted model IDs from an OpenAI-compatible ``GET /models``.""" - - try: - with httpx.Client(timeout=timeout_s) as client: - response = client.get( - f"{base_url.rstrip('/')}/models", - headers={"Authorization": f"Bearer {api_key}"}, - ) - response.raise_for_status() - body = response.json() - except (httpx.HTTPError, ValueError) as exc: - raise ModelDiscoveryError(str(exc)) from exc - - raw_items = body.get("data", []) if isinstance(body, dict) else body - if not isinstance(raw_items, list): - raise ModelDiscoveryError("GET /models response did not contain a model list") - - model_ids: list[str] = [] - for item in raw_items: - if isinstance(item, str): - model_ids.append(item) - continue - if not isinstance(item, dict): - continue - for key in ("id", "model", "name"): - value = item.get(key) - if isinstance(value, str) and value: - model_ids.append(value) - break - return sorted(set(model_ids)) - - -__all__ = ["ModelDiscoveryError", "fetch_model_ids"] diff --git a/switchyard/cli/route_bundle.py b/switchyard/cli/route_bundle.py deleted file mode 100644 index 61f210fa8..000000000 --- a/switchyard/cli/route_bundle.py +++ /dev/null @@ -1,231 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Build a Python server route table from a minimal YAML bundle.""" - -from __future__ import annotations - -import os -import re -import time -from collections.abc import Mapping, Sequence -from importlib import import_module -from pathlib import Path -from typing import Any, Protocol, cast - -from switchyard.lib.backends.llm_target import LlmTarget, coerce_llm_target -from switchyard.lib.backends.multi_llm_backend import build_native_backend -from switchyard.lib.backends.stats_llm_backend import StatsLlmBackend -from switchyard.lib.processors.stats_request_processor import StatsRequestProcessor -from switchyard.lib.processors.stats_response_processor_accumulator import ( - StatsResponseProcessor, -) -from switchyard.lib.proxy_context import ProxyContext -from switchyard.lib.roles import LLMBackend -from switchyard.lib.route_table import ChainRuntime, RouteTable -from switchyard.lib.stats_accumulator import StatsAccumulator -from switchyard.lib.switchyard import Switchyard -from switchyard_rust.core import ChatRequest, ChatRequestType, ChatResponse -from switchyard_rust.translation import TranslationEngine - -_ENV_REF_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}") -_TOP_LEVEL_KEYS = frozenset({"defaults", "routes"}) -_ROUTE_METADATA_KEYS = frozenset({"display_name", "description"}) -_ROUTE_KEYS = { - "noop": frozenset({"type"}) | _ROUTE_METADATA_KEYS, - "passthrough": frozenset({"type", "target"}) | _ROUTE_METADATA_KEYS, -} - - -class RouteBundleConfigError(ValueError): - """Raised when a Python server route bundle is invalid.""" - - -class _YamlModule(Protocol): - def safe_load(self, stream: str) -> object: ... - - -class _NoopBackend(LLMBackend): - """Return a fixed response without making an upstream request.""" - - @property - def supported_request_types(self) -> list[ChatRequestType]: - return list(ChatRequestType) - - async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: - model = request.model or "switchyard/noop" - ctx.selected_model = model - ctx.selected_target = model - return ChatResponse.openai_completion({ - "id": "switchyard-noop", - "object": "chat.completion", - "created": int(time.time()), - "model": model, - "choices": [{ - "index": 0, - "message": {"role": "assistant", "content": "OK"}, - "finish_reason": "stop", - }], - "usage": { - "prompt_tokens": 0, - "completion_tokens": 0, - "total_tokens": 0, - }, - }) - - -def parse_route_bundle_file(path: str | Path) -> dict[str, object]: - """Read a YAML route bundle and return its top-level mapping.""" - resolved = Path(path) - try: - contents = resolved.read_text() - except FileNotFoundError as error: - raise RouteBundleConfigError(f"{resolved}: file not found") from error - except (OSError, UnicodeError) as error: - raise RouteBundleConfigError(f"{resolved}: cannot read: {error}") from error - - try: - yaml = cast(_YamlModule, import_module("yaml")) - raw = yaml.safe_load(contents) - except Exception as error: - message = " ".join(str(error).splitlines()) - raise RouteBundleConfigError(f"{resolved}: invalid YAML: {message}") from error - return _mapping(raw, "route bundle") - - -def load_route_bundle_table( - path: str | Path, - *, - stats_accumulator: StatsAccumulator | None = None, - pre_routing_request_processors: Sequence[Any] = (), - extra_response_processors: Sequence[Any] = (), -) -> RouteTable: - """Load a YAML route bundle into a server dispatch table.""" - return build_route_bundle_table( - parse_route_bundle_file(path), - stats_accumulator=stats_accumulator, - pre_routing_request_processors=pre_routing_request_processors, - extra_response_processors=extra_response_processors, - ) - - -def build_route_bundle_table( - raw: object, - *, - stats_accumulator: StatsAccumulator | None = None, - pre_routing_request_processors: Sequence[Any] = (), - extra_response_processors: Sequence[Any] = (), -) -> RouteTable: - """Build a table containing only noop and passthrough routes.""" - bundle = _mapping(_expand_env(raw), "route bundle") - _reject_unknown_keys(bundle, _TOP_LEVEL_KEYS, "route bundle") - defaults = _mapping(bundle.get("defaults", {}), "defaults") - routes = _mapping(bundle.get("routes"), "routes") - if not routes: - raise RouteBundleConfigError("routes must contain at least one route") - - stats = stats_accumulator or StatsAccumulator() - table = RouteTable() - for route_id, raw_route in routes.items(): - if not route_id: - raise RouteBundleConfigError("route ids must be non-empty strings") - route = _mapping(raw_route, f"route {route_id!r}") - route_type = route.get("type") - if not isinstance(route_type, str): - raise RouteBundleConfigError(f"route {route_id!r}: missing string 'type'") - if route_type not in _ROUTE_KEYS: - raise RouteBundleConfigError( - f"route {route_id!r}: unsupported route type {route_type!r}; " - "expected 'noop' or 'passthrough'" - ) - _reject_unknown_keys(route, _ROUTE_KEYS[route_type], f"route {route_id!r}") - - if route_type == "noop": - runtime = _build_runtime( - _NoopBackend(), - stats, - pre_routing_request_processors, - extra_response_processors, - ) - else: - target = _target(route_id, route.get("target"), defaults) - runtime = _build_runtime( - StatsLlmBackend(build_native_backend(target), stats), - stats, - pre_routing_request_processors, - extra_response_processors, - ) - - metadata = { - key: value - for key in _ROUTE_METADATA_KEYS - if (value := route.get(key)) is not None - } - table.register(route_id, runtime, metadata=metadata, default=table.default_model() is None) - return table - - -def _build_runtime( - backend: LLMBackend, - stats: StatsAccumulator, - request_processors: Sequence[Any], - response_processors: Sequence[Any], -) -> ChainRuntime: - return Switchyard( - request_processors=[StatsRequestProcessor(), *request_processors], - backend=backend, - response_processors=[StatsResponseProcessor(stats), *response_processors], - translator=TranslationEngine(), - ) - - -def _target(route_id: str, value: object, defaults: Mapping[str, object]) -> LlmTarget: - if isinstance(value, str): - target: dict[str, object] = {"model": value} - else: - target = _mapping(value, f"route {route_id!r} target") - try: - return coerce_llm_target({**defaults, **target}, default_id=route_id) - except (TypeError, ValueError) as error: - raise RouteBundleConfigError(f"route {route_id!r}: invalid target: {error}") from error - - -def _expand_env(value: object) -> object: - if isinstance(value, dict): - return {key: _expand_env(item) for key, item in value.items()} - if isinstance(value, list): - return [_expand_env(item) for item in value] - if not isinstance(value, str): - return value - - def replace(match: re.Match[str]) -> str: - name = match.group(1) - if name not in os.environ: - raise RouteBundleConfigError(f"environment variable {name} is not set") - return os.environ[name] - - return _ENV_REF_RE.sub(replace, value) - - -def _mapping(value: object, where: str) -> dict[str, object]: - if not isinstance(value, Mapping): - raise RouteBundleConfigError(f"{where} must be a mapping") - if not all(isinstance(key, str) for key in value): - raise RouteBundleConfigError(f"{where} keys must be strings") - return {str(key): item for key, item in value.items()} - - -def _reject_unknown_keys( - value: Mapping[str, object], allowed: frozenset[str], where: str -) -> None: - unknown = sorted(set(value) - allowed) - if unknown: - raise RouteBundleConfigError(f"unknown key(s) for {where}: {', '.join(unknown)}") - - -__all__ = [ - "RouteBundleConfigError", - "build_route_bundle_table", - "load_route_bundle_table", - "parse_route_bundle_file", -] diff --git a/switchyard/cli/switchyard_cli.py b/switchyard/cli/switchyard_cli.py index 3698e4aa9..49ed84bdd 100644 --- a/switchyard/cli/switchyard_cli.py +++ b/switchyard/cli/switchyard_cli.py @@ -5,68 +5,13 @@ """Switchyard command-line entry point.""" import argparse -import logging -import os from switchyard import __version__ -from switchyard.cli.command_utils import ( - quiet_dependency_loggers as _quiet_dependency_loggers, -) from switchyard.cli.launch_command import ( cmd_launch_claude, cmd_launch_codex, cmd_launch_openclaw, ) -from switchyard.cli.route_bundle import RouteBundleConfigError, load_route_bundle_table -from switchyard.lib.processors.rl_logging_response_processor import build_rl_logging_processors -from switchyard.server.server_util import ( - add_transport_args, - build_and_serve, - resolve_rl_log_dir, -) - -logger = logging.getLogger(__name__) - -def _cmd_serve(args: argparse.Namespace) -> None: - """Serve an explicit Python route bundle.""" - - request_processors, response_processors = build_rl_logging_processors( - resolve_rl_log_dir(args) - ) - if args.routing_log_file: - from switchyard.lib.processors.routing_log_response_processor import ( - RoutingLogResponseProcessor, - ) - - response_processors.append(RoutingLogResponseProcessor(args.routing_log_file)) - - table = load_route_bundle_table( - args.routes, - pre_routing_request_processors=request_processors, - extra_response_processors=response_processors, - ) - logger.info( - "Switchyard route bundle loaded %d route(s) from %s", - len(table.registered_models()), - args.routes, - ) - strategy_summary: str | None = None - default_model = table.default_model() - if default_model: - from switchyard.cli.launchers.launcher_runtime import ( - route_bundle_strategy_summary, - ) - - strategy_summary = route_bundle_strategy_summary( - args.routes, - default_model, - ) - build_and_serve( - args, - table, - inbound_default="both", - strategy_summary=strategy_summary, - ) def _add_launch_parser( @@ -116,26 +61,6 @@ def _build_parser() -> argparse.ArgumentParser: ) subparsers = parser.add_subparsers(dest="command") - serve = subparsers.add_parser("serve", help="Serve a Python route bundle") - serve.add_argument( - "--routes", - "-c", - required=True, - metavar="PATH", - help="YAML bundle containing noop and passthrough routes.", - ) - serve.add_argument("--enable-rl-logging", action="store_true") - serve.add_argument("--rl-log-dir", default="./rl_data", metavar="DIR") - add_transport_args(serve) - serve.add_argument("--routing-log-file", default=None, metavar="PATH") - serve.add_argument( - "--workers", - "-w", - type=int, - default=int(os.environ.get("SWITCHYARD_WORKERS", "1")), - ) - serve.set_defaults(func=_cmd_serve) - _add_launch_parser(subparsers) return parser @@ -143,20 +68,12 @@ def _build_parser() -> argparse.ArgumentParser: def main() -> None: """Run the Switchyard CLI.""" - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", - ) - _quiet_dependency_loggers() parser = _build_parser() args = parser.parse_args() if not hasattr(args, "func"): parser.print_help() raise SystemExit(1) - try: - args.func(args) - except RouteBundleConfigError as exc: - raise SystemExit(f"error: invalid route bundle: {exc}") from exc + args.func(args) if __name__ == "__main__": diff --git a/switchyard/lib/__init__.py b/switchyard/lib/__init__.py deleted file mode 100644 index b7032e225..000000000 --- a/switchyard/lib/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Core library — Rust-backed chat values, translation, and Python serving. - -This subpackage holds the protocol-agnostic building blocks used across the rest -of the library: - -- ``ChatRequest`` — Rust-backed request values (OpenAI, Responses, Anthropic) -- ``chat_response`` — Rust-backed response values plus Python stream adapters -- ``translation`` — pure format-conversion functions and typed translation engines -- ``processors`` — reusable request/response components -- ``backends`` — LLM backend implementations (OpenAI, Anthropic, multi-tier routing) -- ``roles`` — backend role definitions and translation response aliases -""" diff --git a/switchyard/lib/backends/__init__.py b/switchyard/lib/backends/__init__.py deleted file mode 100644 index 53c20ae6f..000000000 --- a/switchyard/lib/backends/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Concrete :class:`LLMBackend` implementations + colocated backend config. - -Each file defines one ``LLMBackend``. Re-exports live here for ergonomic imports like -``from switchyard.lib.backends import OpenAiNativeBackend``. -""" - -from switchyard.lib.backends.backend_format_resolver import ( - BackendFormatResolution, - BackendFormatResolver, -) -from switchyard.lib.backends.stats_llm_backend import ( - StatsLlmBackend, -) -from switchyard_rust.components import ( - AnthropicNativeBackend, - OpenAiNativeBackend, - OpenAiPassthroughBackend, -) - -__all__ = [ - "AnthropicNativeBackend", - "BackendFormatResolution", - "BackendFormatResolver", - "OpenAiPassthroughBackend", - "OpenAiNativeBackend", - "StatsLlmBackend", -] diff --git a/switchyard/lib/backends/anthropic_native_llm_backend.py b/switchyard/lib/backends/anthropic_native_llm_backend.py deleted file mode 100644 index 2d824641d..000000000 --- a/switchyard/lib/backends/anthropic_native_llm_backend.py +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Rust-owned Anthropic-native backend export.""" - -from switchyard_rust.components import AnthropicNativeBackend - -__all__ = ["AnthropicNativeBackend"] diff --git a/switchyard/lib/backends/backend_format_resolver.py b/switchyard/lib/backends/backend_format_resolver.py deleted file mode 100644 index e74ffa10c..000000000 --- a/switchyard/lib/backends/backend_format_resolver.py +++ /dev/null @@ -1,425 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Resolve generic backend tier formats into concrete backend wire formats. - -``BackendFormat.AUTO`` probes the upstream in priority order: - 1. OpenAI Chat Completions (``/v1/chat/completions``) → ``OPENAI`` - 2. Anthropic Messages (``/v1/messages``) → ``ANTHROPIC`` - 3. OpenAI Responses (``/v1/responses``) → ``RESPONSES`` - 4. Fallback → ``OPENAI`` (Chat Completions, assumed universal) - -Chat Completions is probed first because it is the most widely supported format. -Endpoints that bridge multiple API surfaces (e.g. NVIDIA Inference Hub via -LiteLLM) will satisfy all three probes; preferring Chat Completions avoids -silently upgrading NIM models to Anthropic Messages format. Anthropic-native -endpoints (api.anthropic.com) return 404 for Chat Completions, so they correctly -fall through to the Anthropic probe. - -The TranslationEngine converts any inbound format to any backend format through -a neutral IR, so all (inbound, backend) combinations are valid regardless of -which format the client uses. -""" - -import json -import logging -from dataclasses import dataclass -from typing import Any - -import httpx - -from switchyard.lib import startup_timing -from switchyard.lib.backends.llm_target import ( - BackendFormat, - LlmTarget, -) - -log = logging.getLogger(__name__) -_DEFAULT_TIMEOUT_S = 3.0 - - -@dataclass(frozen=True) -class BackendFormatResolution: - """Concrete backend format selected for a generic tier.""" - - format: BackendFormat - reason: str - - -class BackendFormatResolver: - """Resolve ``BackendFormat.AUTO`` through reusable capability probes.""" - - @staticmethod - def resolve(tier: LlmTarget) -> BackendFormatResolution: - """Return the concrete backend format for ``tier``. - - Explicit formats are already resolved. ``AUTO`` needs a real endpoint - probe, so missing probe inputs fail fast instead of silently picking - a backend that may only work by accident. - """ - if tier.format != BackendFormat.AUTO: - return BackendFormatResolution( - format=tier.format, - reason="backend format is explicitly configured", - ) - - return BackendFormatResolver._resolve_auto(tier) - - @staticmethod - def _resolve_auto(tier: LlmTarget) -> BackendFormatResolution: - if not tier.endpoint.base_url: - raise ValueError( - "format='auto' requires base_url so Switchyard can probe upstream capabilities.", - ) - if not tier.endpoint.api_key: - raise ValueError( - "format='auto' requires api_key so Switchyard can probe upstream capabilities.", - ) - - if _model_is_anthropic(tier.model): - return BackendFormatResolution( - format=BackendFormat.ANTHROPIC, - reason="model prefix indicates native Anthropic; skipping probes", - ) - - timeout_s = tier.endpoint.timeout_secs or _DEFAULT_TIMEOUT_S - - # startup_timing marks let `switchyard launch --startup-timing` show each - # probe as its own line, so a slow AUTO detection names which route stalled. - startup_timing.mark("chain init") - - # Chat Completions is probed first: it is the most widely supported - # format and the universal fallback. A transport timeout (the endpoint - # is reachable but slow, e.g. a cold NVCF/Azure deployment) is a - # different signal from a fast "not wired" 404. On a timeout, assume - # Chat Completions rather than serially probing the rarer /v1/messages - # and /v1/responses routes — each of those would stack another timeout - # and turn one slow startup into several seconds. A 404 (even a slow - # one) still falls through to the probes below, so Anthropic-native and - # Responses-only endpoints are detected as before. - try: - chat_supported = probe_openai_chat_completions_support_sync( - base_url=tier.endpoint.base_url, - api_key=tier.endpoint.api_key, - model=tier.model, - timeout_s=timeout_s, - ) - except httpx.TimeoutException: - startup_timing.mark("probe: /v1/chat/completions (timed out)") - return BackendFormatResolution( - format=BackendFormat.OPENAI, - reason="chat-completions probe timed out; assuming Chat Completions", - ) - startup_timing.mark("probe: /v1/chat/completions") - if chat_supported: - return BackendFormatResolution( - format=BackendFormat.OPENAI, - reason="upstream /v1/chat/completions probe succeeded", - ) - - anthropic_supported = probe_anthropic_messages_support_sync( - base_url=tier.endpoint.base_url, - api_key=tier.endpoint.api_key, - model=tier.model, - timeout_s=timeout_s, - ) - startup_timing.mark("probe: /v1/messages") - if anthropic_supported: - return BackendFormatResolution( - format=BackendFormat.ANTHROPIC, - reason="upstream /v1/messages probe succeeded; Chat Completions not available", - ) - - responses_supported = probe_openai_responses_support_sync( - base_url=tier.endpoint.base_url, - api_key=tier.endpoint.api_key, - model=tier.model, - timeout_s=timeout_s, - ) - startup_timing.mark("probe: /v1/responses") - if responses_supported: - return BackendFormatResolution( - format=BackendFormat.RESPONSES, - reason="upstream /v1/responses probe succeeded; Chat Completions not available", - ) - - return BackendFormatResolution( - format=BackendFormat.OPENAI, - reason="all probes failed; assuming Chat Completions", - ) - - -def _model_is_anthropic(model: str | None) -> bool: - """Return True if the model ID prefix unambiguously identifies a native Anthropic model. - - Matches ``anthropic/`` and ``claude<…>`` prefixes only — these map - exclusively to the Anthropic API or OpenRouter's direct Anthropic passthrough, - both of which require ``/v1/messages``. - - Gateway-namespaced paths like ``aws/anthropic/bedrock-…`` and - ``openrouter/anthropic/…`` are intentionally NOT matched: those gateways - also expose Chat Completions, so probing is preferred over assuming. - """ - if not model: - return False - m = model.lower() - return m.startswith("anthropic/") or m.startswith("claude") - - -def probe_openai_chat_completions_support_sync( - *, - base_url: str, - api_key: str, - model: str | None = None, - timeout_s: float = _DEFAULT_TIMEOUT_S, -) -> bool: - """Return True iff ``{base_url}/chat/completions`` is a functional route. - - Sends a minimal-body probe POST with Bearer auth. A 404 means the - Chat Completions endpoint is not wired — the caller should probe - Anthropic Messages or Responses next. - - Raises ``httpx.TimeoutException`` on a transport timeout. A timeout means - the endpoint is reachable but slow (e.g. a cold NVCF/Azure deployment), - which is a different signal from a fast "not wired" 404: the resolver - stops probing and assumes Chat Completions instead of stacking the slower - ``/v1/messages`` and ``/v1/responses`` probes. - """ - url = f"{base_url.rstrip('/')}/chat/completions" - headers = {"Authorization": f"Bearer {api_key}", "content-type": "application/json"} - req_body: dict[str, Any] = { - "messages": [{"role": "user", "content": " "}], - "max_tokens": 1, - **({"model": model} if model else {}), - } - try: - with httpx.Client(timeout=timeout_s) as client: - resp = client.post(url, headers=headers, json=req_body) - except httpx.TimeoutException: - raise - except httpx.RequestError as e: - log.warning( - "OpenAI /v1/chat/completions probe failed (%s); " - "falling back to Anthropic/Responses probes.", - type(e).__name__, - ) - return False - if resp.status_code == 404: - return False - if resp.status_code == 401: - log.warning( - "OpenAI /v1/chat/completions probe got HTTP 401 — check --api-key. " - "Falling back to Anthropic/Responses probes.", - ) - return False - if 200 <= resp.status_code < 500: - return True - log.warning( - "OpenAI /v1/chat/completions probe got HTTP %d; " - "falling back to Anthropic/Responses probes.", - resp.status_code, - ) - return False - - -def _interpret_status(status: int, body: bytes = b"") -> bool: - """Return True iff the status code indicates the route is wired. - - When ``body`` is provided, a 400/422 that names the model as not found or - unsupported is treated as a probe failure — the route exists but this model - is not valid for it. Without a body (legacy call-sites) the old behaviour - is preserved. - """ - if status == 404: - return False - if status == 401: - log.warning( - "Anthropic /v1/messages probe got HTTP 401 — check --api-key. " - "Falling back to translation mode.", - ) - return False - if 200 <= status < 300: - return True - if status in (400, 422) and body: - if _body_signals_model_error(body): - log.debug( - "Anthropic /v1/messages probe got HTTP %d with model error; " - "model is unsupported on this endpoint.", - status, - ) - return False - return True # validation error about other fields — route and model exist - if 200 <= status < 500: - return True - log.warning( - "Anthropic /v1/messages probe got HTTP %d; falling back to translation mode.", - status, - ) - return False - - -def _body_signals_model_error(body: bytes) -> bool: - """Return True if the JSON error body indicates the model is not available.""" - try: - data = json.loads(body) - except Exception: - return False - if not isinstance(data, dict): - return False - error = data.get("error") - if not isinstance(error, dict): - return False - if error.get("type") in ("not_found_error",): - return True - msg = (error.get("message") or "").lower() - return "model" in msg and any( - kw in msg for kw in ("not found", "not supported", "unsupported", "unknown", "invalid") - ) - - -def _probe_headers(api_key: str) -> dict[str, str]: - # Include both auth styles: native Anthropic uses x-api-key, but - # OpenAI-compatible gateways (e.g. NVIDIA Inference Hub) that also expose - # /v1/messages expect Authorization: Bearer. - return { - "x-api-key": api_key, - "authorization": f"Bearer {api_key}", - "anthropic-version": "2023-06-01", - "content-type": "application/json", - } - - -def strip_v1_suffix(base_url: str) -> str: - """Return *base_url* with a trailing ``/v1`` path component removed. - - Switchyard's ``--base-url`` convention follows OpenAI's (e.g. - ``https://openrouter.ai/api/v1``), but the Anthropic SDK and - raw ``/v1/messages`` probing both treat the base URL as the API - root and append ``/v1/messages`` themselves. Without this trim the - two conventions collide — ``https://host/v1`` + ``/v1/messages`` - becomes ``https://host/v1/v1/messages``. - """ - stripped = base_url.rstrip("/") - if stripped.endswith("/v1"): - return stripped[:-3] - return stripped - - -def probe_openai_responses_support_sync( - *, - base_url: str, - api_key: str, - model: str | None = None, - timeout_s: float = _DEFAULT_TIMEOUT_S, -) -> bool: - """Return True iff ``{base_url}/responses`` is a functional route. - - Sends a minimal-body probe POST with Bearer auth. A 404 means the - Responses endpoint is not wired upstream; callers should fall back to - ``BackendFormat.OPENAI`` (Chat Completions). Non-OpenAI upstreams - (e.g. NVIDIA NIM) commonly 404 here even when they support Chat - Completions. - """ - url = f"{base_url.rstrip('/')}/responses" - headers = {"Authorization": f"Bearer {api_key}", "content-type": "application/json"} - req_body: dict[str, Any] = { - "input": "", - "stream": False, - **({"model": model} if model else {}), - } - try: - with httpx.Client(timeout=timeout_s) as client: - resp = client.post(url, headers=headers, json=req_body) - except httpx.RequestError as e: - log.warning( - "OpenAI /v1/responses probe failed (%s); " - "falling back to Chat Completions format.", - type(e).__name__, - ) - return False - if resp.status_code == 404: - return False - if resp.status_code == 401: - log.warning( - "OpenAI /v1/responses probe got HTTP 401; " - "falling back to Chat Completions format.", - ) - return False - if 200 <= resp.status_code < 500: - return True - log.warning( - "OpenAI /v1/responses probe got HTTP %d; falling back to Chat Completions format.", - resp.status_code, - ) - return False - - -def probe_anthropic_messages_support_sync( - *, - base_url: str, - api_key: str, - model: str | None = None, - timeout_s: float = _DEFAULT_TIMEOUT_S, -) -> bool: - """Synchronous version of :func:`probe_anthropic_messages_support`. - - Preferred at startup (no running event loop required). Uses - ``httpx.Client`` so no asyncio event loop is created; async clients - built afterward bind their connection pools to uvicorn's event loop - on first use rather than to a now-closed startup loop. - """ - url = f"{strip_v1_suffix(base_url)}/v1/messages" - req_body: dict[str, Any] = { - "messages": [{"role": "user", "content": " "}], - "max_tokens": 1, - **({"model": model} if model else {}), - } - try: - with httpx.Client(timeout=timeout_s) as client: - resp = client.post(url, headers=_probe_headers(api_key), json=req_body) - except httpx.RequestError as e: - log.warning( - "Anthropic /v1/messages probe failed (%s); " - "falling back to translation mode.", - type(e).__name__, - ) - return False - return _interpret_status(resp.status_code, resp.content) - - -async def probe_anthropic_messages_support( - *, - base_url: str, - api_key: str, - model: str | None = None, - timeout_s: float = _DEFAULT_TIMEOUT_S, -) -> bool: - """Return True iff ``{base_url}/v1/messages`` is a functional route. - - Sends a model-scoped probe POST with real auth. Response interpretation: - - * 404 → route not wired → return False (use translation) - * 401 → return False (credential validation happens on the real request) - * 400 / 422 with model error body → model unsupported → return False - * 400 / 422 with field error body → route exists → return True - * 200 → route exists → return True - * 5xx / timeout / network error → return False - """ - url = f"{strip_v1_suffix(base_url)}/v1/messages" - req_body: dict[str, Any] = { - "messages": [{"role": "user", "content": " "}], - "max_tokens": 1, - **({"model": model} if model else {}), - } - try: - async with httpx.AsyncClient(timeout=timeout_s) as client: - resp = await client.post(url, headers=_probe_headers(api_key), json=req_body) - except httpx.RequestError as e: - log.debug( - "Anthropic /v1/messages probe unavailable (%s); " - "using OpenAI translation mode.", - type(e).__name__, - ) - return False - - return _interpret_status(resp.status_code, resp.content) diff --git a/switchyard/lib/backends/llm_target.py b/switchyard/lib/backends/llm_target.py deleted file mode 100644 index ad22d0738..000000000 --- a/switchyard/lib/backends/llm_target.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Rust-owned LLM target configuration helpers.""" - -from __future__ import annotations - -from typing import Any - -from pydantic import BaseModel - -from switchyard_rust.components import BackendFormat, EndpointConfig, LlmTarget - -_DISABLE_THINKING_MODEL_FRAGMENTS = ("nemotron-3-super",) - - -def coerce_llm_target(value: object, *, default_id: str) -> LlmTarget: - """Build a Rust ``LlmTarget`` from a target object or legacy mapping.""" - if isinstance(value, LlmTarget): - if value.id == "default" and default_id != "default": - return LlmTarget( - id=default_id, - model=value.model, - format=value.format, - endpoint=value.endpoint, - extra_body=value.extra_body, - extra_headers=value.extra_headers, - ) - return value - if isinstance(value, BaseModel): - value = value.model_dump() - if not isinstance(value, dict): - raise TypeError(f"expected LlmTarget or dict, got {type(value).__name__}") - - data: dict[str, Any] = dict(value) - target_id = str(data.pop("id", default_id)) - model = data.pop("model", None) - if not isinstance(model, str): - raise TypeError("LlmTarget.model must be a string") - - target_format = data.pop("format", data.pop("backend_format", BackendFormat.OPENAI)) - endpoint = data.pop("endpoint", None) - base_url = data.pop("base_url", None) - api_key = data.pop("api_key", None) - timeout_secs = data.pop("timeout_secs", data.pop("timeout", None)) - extra_body = data.pop("extra_body", None) - extra_headers = data.pop("extra_headers", None) - data.pop("tuning", None) - if data: - unknown = ", ".join(sorted(data)) - raise ValueError(f"unknown LlmTarget fields: {unknown}") - - return LlmTarget( - id=target_id, - model=model, - format=target_format, - endpoint=endpoint, - base_url=base_url, - api_key=api_key, - timeout_secs=timeout_secs, - extra_body=extra_body, - extra_headers=extra_headers, - ) - - -def llm_target_with_format(target: LlmTarget, target_format: BackendFormat) -> LlmTarget: - """Return ``target`` with a resolved backend format.""" - return LlmTarget( - id=target.id, - model=target.model, - format=target_format, - endpoint=target.endpoint, - extra_body=target.extra_body, - extra_headers=target.extra_headers, - ) - - -def llm_target_with_runtime_defaults(target: LlmTarget) -> LlmTarget: - """Return ``target`` with Switchyard's safe per-model runtime defaults.""" - if target.extra_body: - return target - if not _should_disable_thinking(target.model): - return target - return LlmTarget( - id=target.id, - model=target.model, - format=target.format, - endpoint=target.endpoint, - extra_body={"chat_template_kwargs": {"enable_thinking": False}}, - extra_headers=target.extra_headers, - ) - - -def _should_disable_thinking(model: str) -> bool: - normalized = model.lower() - return any(fragment in normalized for fragment in _DISABLE_THINKING_MODEL_FRAGMENTS) - - -__all__ = [ - "BackendFormat", - "EndpointConfig", - "LlmTarget", - "coerce_llm_target", - "llm_target_with_format", - "llm_target_with_runtime_defaults", -] diff --git a/switchyard/lib/backends/multi_llm_backend.py b/switchyard/lib/backends/multi_llm_backend.py deleted file mode 100644 index 48afdcb4f..000000000 --- a/switchyard/lib/backends/multi_llm_backend.py +++ /dev/null @@ -1,54 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Build a native backend for a single LLM target.""" - -from __future__ import annotations - -import logging - -from switchyard.lib.backends.backend_format_resolver import BackendFormatResolver -from switchyard.lib.backends.llm_target import ( - BackendFormat, - LlmTarget, - llm_target_with_format, - llm_target_with_runtime_defaults, -) -from switchyard.lib.roles import LLMBackend -from switchyard_rust.components import ( - AnthropicNativeBackend, - OpenAiNativeBackend, -) - -log = logging.getLogger(__name__) - - -def resolve_llm_target(target: LlmTarget) -> LlmTarget: - """Resolve ``BackendFormat.AUTO`` into the concrete native backend format.""" - if target.format != BackendFormat.AUTO: - return target - resolution = BackendFormatResolver.resolve(target) - log.debug( - "resolved LLM target id=%s model=%s format=%s: %s", - target.id, - target.model, - resolution.format.value, - resolution.reason, - ) - return llm_target_with_format(target, resolution.format) - - -def build_native_backend(target: LlmTarget) -> LLMBackend: - """Build the native Rust backend for one resolved or auto ``LlmTarget``.""" - target = llm_target_with_runtime_defaults(resolve_llm_target(target)) - if target.format in (BackendFormat.OPENAI, BackendFormat.RESPONSES): - return OpenAiNativeBackend(target) - if target.format == BackendFormat.ANTHROPIC: - return AnthropicNativeBackend(target) - raise ValueError(f"Unsupported backend format: {target.format!r}") - - -__all__ = [ - "build_native_backend", - "resolve_llm_target", -] diff --git a/switchyard/lib/backends/openai_llm_backend.py b/switchyard/lib/backends/openai_llm_backend.py deleted file mode 100644 index 8fcab95d6..000000000 --- a/switchyard/lib/backends/openai_llm_backend.py +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Compatibility import path for the Rust-owned OpenAI passthrough backend.""" - -from switchyard_rust.components import OpenAiPassthroughBackend - -__all__ = ["OpenAiPassthroughBackend"] diff --git a/switchyard/lib/backends/openai_native_backend.py b/switchyard/lib/backends/openai_native_backend.py deleted file mode 100644 index 646445880..000000000 --- a/switchyard/lib/backends/openai_native_backend.py +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Rust-owned OpenAI-native backend export.""" - -from switchyard_rust.components import OpenAiNativeBackend - -__all__ = ["OpenAiNativeBackend"] diff --git a/switchyard/lib/backends/stats_llm_backend.py b/switchyard/lib/backends/stats_llm_backend.py deleted file mode 100644 index 9ed76d617..000000000 --- a/switchyard/lib/backends/stats_llm_backend.py +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Rust-owned stats backend wrapper export.""" - -from switchyard_rust.components import StatsLlmBackend - -__all__ = ["StatsLlmBackend"] diff --git a/switchyard/lib/chat_request/__init__.py b/switchyard/lib/chat_request/__init__.py deleted file mode 100644 index 90113e115..000000000 --- a/switchyard/lib/chat_request/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Compatibility aliases for Rust-backed request values.""" - -from switchyard.lib.chat_request.anthropic import AnthropicChatRequest -from switchyard.lib.chat_request.base import ChatRequest, ChatRequestType -from switchyard.lib.chat_request.openai_chat import OpenAIChatRequest -from switchyard.lib.chat_request.openai_responses import ResponsesChatRequest - -__all__ = [ - "AnthropicChatRequest", - "ChatRequest", - "ChatRequestType", - "OpenAIChatRequest", - "ResponsesChatRequest", -] diff --git a/switchyard/lib/chat_request/anthropic.py b/switchyard/lib/chat_request/anthropic.py deleted file mode 100644 index 3fbdf162f..000000000 --- a/switchyard/lib/chat_request/anthropic.py +++ /dev/null @@ -1,12 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Anthropic Messages request compatibility alias.""" - -from typing import TypeAlias - -from switchyard_rust.core import ChatRequest as _ChatRequest - -AnthropicChatRequest: TypeAlias = _ChatRequest - -__all__ = ["AnthropicChatRequest"] diff --git a/switchyard/lib/chat_request/base.py b/switchyard/lib/chat_request/base.py deleted file mode 100644 index 5aff06bb2..000000000 --- a/switchyard/lib/chat_request/base.py +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Compatibility re-export for Rust-owned request values.""" - -from typing import TypeAlias - -from switchyard_rust.core import ChatRequest as _ChatRequest -from switchyard_rust.core import ChatRequestType as _ChatRequestType - -ChatRequest: TypeAlias = _ChatRequest -ChatRequestType: TypeAlias = _ChatRequestType - -__all__ = ["ChatRequest", "ChatRequestType"] diff --git a/switchyard/lib/chat_request/openai_chat.py b/switchyard/lib/chat_request/openai_chat.py deleted file mode 100644 index 3051a683e..000000000 --- a/switchyard/lib/chat_request/openai_chat.py +++ /dev/null @@ -1,12 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""OpenAI Chat Completions request compatibility alias.""" - -from typing import TypeAlias - -from switchyard_rust.core import ChatRequest as _ChatRequest - -OpenAIChatRequest: TypeAlias = _ChatRequest - -__all__ = ["OpenAIChatRequest"] diff --git a/switchyard/lib/chat_request/openai_responses.py b/switchyard/lib/chat_request/openai_responses.py deleted file mode 100644 index 7a35a5e57..000000000 --- a/switchyard/lib/chat_request/openai_responses.py +++ /dev/null @@ -1,12 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""OpenAI Responses API request compatibility alias.""" - -from typing import TypeAlias - -from switchyard_rust.core import ChatRequest as _ChatRequest - -ResponsesChatRequest: TypeAlias = _ChatRequest - -__all__ = ["ResponsesChatRequest"] diff --git a/switchyard/lib/chat_response/__init__.py b/switchyard/lib/chat_response/__init__.py deleted file mode 100644 index c453843ba..000000000 --- a/switchyard/lib/chat_response/__init__.py +++ /dev/null @@ -1,39 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Rust-backed response values and stream adapters.""" - -from switchyard.lib.chat_response.anthropic import ( - AnthropicChatResponse, - AnthropicResponseStream, - AnthropicStreamingChatResponse, -) -from switchyard.lib.chat_response.openai_chat import ( - CompletionChatResponse, - ResponseStream, - StreamingChatResponse, -) -from switchyard.lib.chat_response.openai_responses import ( - ResponsesApiChatResponse, - ResponsesApiStream, - ResponsesApiStreamingChatResponse, -) -from switchyard_rust.core import ChatResponse, ChatResponseStream, ChatResponseType - -AnyResponseStream = ChatResponseStream - -__all__ = [ - "AnyResponseStream", - "AnthropicChatResponse", - "AnthropicResponseStream", - "AnthropicStreamingChatResponse", - "ChatResponse", - "ChatResponseStream", - "ChatResponseType", - "CompletionChatResponse", - "ResponsesApiChatResponse", - "ResponsesApiStream", - "ResponsesApiStreamingChatResponse", - "ResponseStream", - "StreamingChatResponse", -] diff --git a/switchyard/lib/chat_response/anthropic.py b/switchyard/lib/chat_response/anthropic.py deleted file mode 100644 index fc8da3024..000000000 --- a/switchyard/lib/chat_response/anthropic.py +++ /dev/null @@ -1,19 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Anthropic Messages API stream adapter re-export.""" - -from typing import TypeAlias - -from switchyard_rust.core import ChatResponse as _ChatResponse -from switchyard_rust.core import ChatResponseStream as _ChatResponseStream - -AnthropicChatResponse: TypeAlias = _ChatResponse -AnthropicStreamingChatResponse: TypeAlias = _ChatResponse -AnthropicResponseStream: TypeAlias = _ChatResponseStream - -__all__ = [ - "AnthropicChatResponse", - "AnthropicResponseStream", - "AnthropicStreamingChatResponse", -] diff --git a/switchyard/lib/chat_response/base.py b/switchyard/lib/chat_response/base.py deleted file mode 100644 index 8fcc14f72..000000000 --- a/switchyard/lib/chat_response/base.py +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Compatibility re-export for Rust-owned response values.""" - -from typing import TypeAlias - -from switchyard_rust.core import ChatResponse as _ChatResponse -from switchyard_rust.core import ChatResponseType as _ChatResponseType - -ChatResponse: TypeAlias = _ChatResponse -ChatResponseType: TypeAlias = _ChatResponseType - -__all__ = ["ChatResponse", "ChatResponseType"] diff --git a/switchyard/lib/chat_response/openai_chat.py b/switchyard/lib/chat_response/openai_chat.py deleted file mode 100644 index a181c3a36..000000000 --- a/switchyard/lib/chat_response/openai_chat.py +++ /dev/null @@ -1,15 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""OpenAI Chat Completions stream adapter re-export.""" - -from typing import TypeAlias - -from switchyard_rust.core import ChatResponse as _ChatResponse -from switchyard_rust.core import ChatResponseStream as _ChatResponseStream - -CompletionChatResponse: TypeAlias = _ChatResponse -StreamingChatResponse: TypeAlias = _ChatResponse -ResponseStream: TypeAlias = _ChatResponseStream - -__all__ = ["CompletionChatResponse", "ResponseStream", "StreamingChatResponse"] diff --git a/switchyard/lib/chat_response/openai_responses.py b/switchyard/lib/chat_response/openai_responses.py deleted file mode 100644 index c18cca56a..000000000 --- a/switchyard/lib/chat_response/openai_responses.py +++ /dev/null @@ -1,19 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""OpenAI Responses API stream adapter re-export.""" - -from typing import TypeAlias - -from switchyard_rust.core import ChatResponse as _ChatResponse -from switchyard_rust.core import ChatResponseStream as _ChatResponseStream - -ResponsesApiChatResponse: TypeAlias = _ChatResponse -ResponsesApiStreamingChatResponse: TypeAlias = _ChatResponse -ResponsesApiStream: TypeAlias = _ChatResponseStream - -__all__ = [ - "ResponsesApiChatResponse", - "ResponsesApiStream", - "ResponsesApiStreamingChatResponse", -] diff --git a/switchyard/lib/chat_response/streaming_response_accumulator.py b/switchyard/lib/chat_response/streaming_response_accumulator.py deleted file mode 100644 index 968321457..000000000 --- a/switchyard/lib/chat_response/streaming_response_accumulator.py +++ /dev/null @@ -1,752 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Accumulate streaming responses into completed native responses.""" - -from __future__ import annotations - -import json -import time -from collections.abc import Awaitable, Callable, Mapping -from dataclasses import dataclass -from typing import Literal, Protocol, cast - -from anthropic.types import ( - ContentBlock, - InputJSONDelta, - RawContentBlockDeltaEvent, - RawContentBlockStartEvent, - RawMessageDeltaEvent, - RawMessageStartEvent, - RawMessageStreamEvent, - StopReason, - TextBlock, - TextDelta, - ThinkingDelta, - ToolUseBlock, -) -from anthropic.types import ( - Message as AnthropicMessage, -) -from anthropic.types import ( - Usage as AnthropicUsage, -) -from openai.types.chat import ChatCompletion, ChatCompletionChunk -from openai.types.chat.chat_completion import Choice as ChatCompletionChoice -from openai.types.chat.chat_completion_message import ChatCompletionMessage -from openai.types.chat.chat_completion_message_function_tool_call import ( - ChatCompletionMessageFunctionToolCall, -) -from openai.types.chat.chat_completion_message_function_tool_call import ( - Function as ChatCompletionToolCallFunction, -) -from openai.types.completion_usage import CompletionUsage -from openai.types.responses import ( - Response as OpenAIResponse, -) -from openai.types.responses import ( - ResponseCompletedEvent, - ResponseFunctionCallArgumentsDeltaEvent, - ResponseFunctionToolCall, - ResponseOutputItemAddedEvent, - ResponseOutputItemDoneEvent, - ResponseOutputMessage, - ResponseOutputText, - ResponseTextDeltaEvent, -) -from openai.types.responses.response_usage import ( - InputTokensDetails, - OutputTokensDetails, - ResponseUsage, -) -from pydantic import TypeAdapter, ValidationError - -from switchyard_rust.core import ( - ChatResponse, - ChatResponseType, - response_type_matches, -) - -_ANTHROPIC_EVENT_ADAPTER: TypeAdapter[RawMessageStreamEvent] = TypeAdapter(RawMessageStreamEvent) -_OpenAIFinishReason = Literal[ - "stop", - "length", - "tool_calls", - "content_filter", - "function_call", -] - - -class StreamingResponseAccumulator(Protocol): - """Accumulates provider-native stream events into a completed response.""" - - def consume(self, event: object) -> None: ... - - def as_response(self) -> ChatResponse: ... - - -CompletedResponseCallback = Callable[[ChatResponse], Awaitable[None]] - - -@dataclass -class _OpenAIToolCallState: - id: str | None = None - name: str | None = None - arguments: str = "" - - def to_openai(self, *, fallback_id: str) -> ChatCompletionMessageFunctionToolCall: - return ChatCompletionMessageFunctionToolCall( - id=self.id or fallback_id, - type="function", - function=ChatCompletionToolCallFunction( - name=self.name or "", - arguments=self.arguments, - ), - ) - - -@dataclass -class _AnthropicContentBlockState: - kind: Literal["text", "tool_use"] - text: str = "" - id: str | None = None - name: str | None = None - arguments: str = "" - - def to_anthropic(self, *, fallback_id: str) -> TextBlock | ToolUseBlock: - if self.kind == "text": - return TextBlock(type="text", text=self.text) - return ToolUseBlock( - type="tool_use", - id=self.id or fallback_id, - name=self.name or "", - input=_json_object_from_argument_string(self.arguments), - ) - - -@dataclass -class _ResponsesToolCallState: - id: str | None = None - call_id: str | None = None - name: str = "" - arguments: str = "" - - def to_response_output_item(self, *, fallback_id: str) -> ResponseFunctionToolCall | None: - if not self.name and not self.arguments: - return None - return ResponseFunctionToolCall( - type="function_call", - id=self.id, - call_id=self.call_id or self.id or fallback_id, - name=self.name, - arguments=self.arguments, - status="completed", - ) - - -def attach_final_response_callback( - response: ChatResponse, - *, - served_model: str, - callback: CompletedResponseCallback, -) -> bool: - """Attach a callback that receives the completed native response. - - Returns ``False`` when *response* is not a supported streaming response. - The callback only runs when the stream drains normally; stream wrappers own - that completion contract. - """ - accumulator = create_streaming_response_accumulator( - response, - served_model=served_model, - ) - if accumulator is None: - return False - - async def _tap(event: object) -> None: - accumulator.consume(event) - - async def _on_complete() -> None: - await callback(accumulator.as_response()) - - if response_type_matches(response, ChatResponseType.OPENAI_STREAM): - response.stream.tap(_tap).on_complete(_on_complete) - elif response_type_matches(response, ChatResponseType.ANTHROPIC_STREAM): - response.stream.tap(_tap).on_complete(_on_complete) - elif response_type_matches(response, ChatResponseType.OPENAI_RESPONSES_STREAM): - response.stream.tap(_tap).on_complete(_on_complete) - else: - return False - return True - - -def create_streaming_response_accumulator( - response: ChatResponse, - *, - served_model: str, -) -> StreamingResponseAccumulator | None: - """Create the native accumulator for a streaming response.""" - if response_type_matches(response, ChatResponseType.OPENAI_STREAM): - return _OpenAIChatStreamAccumulator(served_model=served_model) - if response_type_matches(response, ChatResponseType.ANTHROPIC_STREAM): - return _AnthropicStreamAccumulator(served_model=served_model) - if response_type_matches(response, ChatResponseType.OPENAI_RESPONSES_STREAM): - return _ResponsesStreamAccumulator(served_model=served_model) - return None - - -class _OpenAIChatStreamAccumulator: - """Accumulate OpenAI Chat Completion chunks into one ChatCompletion.""" - - def __init__(self, *, served_model: str) -> None: - self._served_model = served_model - self._content = "" - self._reasoning_content = "" - self._tool_calls: list[_OpenAIToolCallState] = [] - self._usage: CompletionUsage | None = None - self._finish_reason: str | None = None - self._response_id: str | None = None - self._created: int | None = None - - def consume(self, event: object) -> None: - chunk = event - if isinstance(chunk, Mapping): - chunk = ChatCompletionChunk.model_validate(chunk) - if not isinstance(chunk, ChatCompletionChunk): - return - - self._response_id = chunk.id or self._response_id - self._created = chunk.created or self._created - self._served_model = chunk.model or self._served_model - if chunk.usage is not None: - self._usage = chunk.usage - if not chunk.choices: - return - - choice = chunk.choices[0] - delta = choice.delta - if delta is not None: - if isinstance(delta.content, str): - self._content += delta.content - reasoning_content = getattr(delta, "reasoning_content", None) - if isinstance(reasoning_content, str): - self._reasoning_content += reasoning_content - for tool_call in delta.tool_calls or []: - self._merge_tool_call(tool_call) - - if choice.finish_reason is not None: - self._finish_reason = choice.finish_reason - - def as_response(self) -> ChatResponse: - message = ChatCompletionMessage( - role="assistant", - content=self._content or None, - tool_calls=[ - tool_call.to_openai(fallback_id=f"call_switchyard_{index}") - for index, tool_call in enumerate(self._tool_calls) - ] or None, - ) - response = ChatCompletion( - id=self._response_id or "chatcmpl-switchyard-stream", - object="chat.completion", - created=self._created or int(time.time()), - model=self._served_model, - choices=[ - ChatCompletionChoice( - index=0, - message=message, - finish_reason=_openai_finish_reason( - self._finish_reason, - has_tools=bool(self._tool_calls), - ), - ), - ], - usage=self._usage, - ) - if self._reasoning_content: - # The OpenAI SDK does not type vendor-specific reasoning fields - # on final messages. Preserve them at the serialization boundary. - response_dict = response.model_dump(mode="json", exclude_none=True) - response_dict["choices"][0]["message"]["reasoning_content"] = ( - self._reasoning_content - ) - response = ChatCompletion.model_validate(response_dict) - return ChatResponse.openai_completion(response) - - def _merge_tool_call(self, tool_call: object) -> None: - index = getattr(tool_call, "index", None) - if not isinstance(index, int): - index = len(self._tool_calls) - while len(self._tool_calls) <= index: - self._tool_calls.append(_OpenAIToolCallState()) - existing = self._tool_calls[index] - tool_call_id = getattr(tool_call, "id", None) - if isinstance(tool_call_id, str) and tool_call_id: - existing.id = tool_call_id - func = getattr(tool_call, "function", None) - if func is None: - return - name = getattr(func, "name", None) - if isinstance(name, str) and name: - existing.name = name - arguments = getattr(func, "arguments", None) - if isinstance(arguments, str) and arguments: - existing.arguments += arguments - - -class _AnthropicStreamAccumulator: - """Accumulate Anthropic Messages events into one Anthropic message.""" - - def __init__(self, *, served_model: str) -> None: - self._served_model = served_model - self._response_id: str | None = None - self._content_blocks: dict[int, _AnthropicContentBlockState] = {} - self._usage: dict[str, int] = {} - self._stop_reason: str | None = None - - def consume(self, event: object) -> None: - typed_event = _coerce_anthropic_event(event) - if isinstance(typed_event, Mapping): - self._consume_mapping_event(typed_event) - return - - if isinstance(typed_event, RawMessageStartEvent): - self._response_id = typed_event.message.id or self._response_id - self._served_model = typed_event.message.model or self._served_model - self._merge_usage(typed_event.message.usage) - return - - if isinstance(typed_event, RawContentBlockStartEvent): - block = typed_event.content_block - if isinstance(block, ToolUseBlock): - self._content_blocks[typed_event.index] = _AnthropicContentBlockState( - kind="tool_use", - id=block.id, - name=block.name, - arguments=_json_argument_string(block.input), - ) - return - if isinstance(block, TextBlock): - self._content_blocks[typed_event.index] = _AnthropicContentBlockState( - kind="text", - text=block.text, - ) - return - self._content_blocks[typed_event.index] = _AnthropicContentBlockState( - kind="text", - ) - return - - if isinstance(typed_event, RawContentBlockDeltaEvent): - state = self._content_blocks.setdefault( - typed_event.index, - _AnthropicContentBlockState(kind="text"), - ) - delta = typed_event.delta - if isinstance(delta, InputJSONDelta): - if state.kind != "tool_use": - state.kind = "tool_use" - state.text = "" - state.id = None - state.name = None - state.arguments = "" - state.arguments += delta.partial_json - return - if isinstance(delta, TextDelta) and state.kind == "text": - state.text += delta.text - return - if isinstance(delta, ThinkingDelta) and state.kind == "text": - state.text += delta.thinking - return - - if isinstance(typed_event, RawMessageDeltaEvent): - self._stop_reason = typed_event.delta.stop_reason or self._stop_reason - self._merge_usage(typed_event.usage) - - def _consume_mapping_event(self, event: Mapping[str, object]) -> None: - event_type = event.get("type") - if event_type == "message_start": - message = event.get("message") - if not isinstance(message, Mapping): - return - response_id = message.get("id") - if isinstance(response_id, str): - self._response_id = response_id - model = message.get("model") - if isinstance(model, str): - self._served_model = model - self._merge_usage(message.get("usage")) - return - - if event_type == "content_block_start": - index = _int_value(event.get("index"), 0) - block = event.get("content_block") - if not isinstance(block, Mapping): - return - if block.get("type") == "tool_use": - self._content_blocks[index] = _AnthropicContentBlockState( - kind="tool_use", - id=_str_value(block.get("id")), - name=_str_value(block.get("name")), - arguments=_json_argument_string(block.get("input")), - ) - return - text = block.get("text") - self._content_blocks[index] = _AnthropicContentBlockState( - kind="text", - text=text if isinstance(text, str) else "", - ) - return - - if event_type == "content_block_delta": - index = _int_value(event.get("index"), 0) - delta = event.get("delta") - if not isinstance(delta, Mapping): - return - block = self._content_blocks.setdefault( - index, - _AnthropicContentBlockState(kind="text"), - ) - if delta.get("type") == "input_json_delta": - if block.kind != "tool_use": - block.kind = "tool_use" - block.text = "" - block.id = None - block.name = None - block.arguments = "" - partial_json = delta.get("partial_json") - if isinstance(partial_json, str): - block.arguments += partial_json - return - if block.kind != "text": - return - text = delta.get("text") - if isinstance(text, str): - block.text += text - return - thinking = delta.get("thinking") - if isinstance(thinking, str): - block.text += thinking - return - - if event_type == "message_delta": - delta = event.get("delta") - if isinstance(delta, Mapping): - stop_reason = delta.get("stop_reason") - if isinstance(stop_reason, str): - self._stop_reason = stop_reason - self._merge_usage(event.get("usage")) - - def as_response(self) -> ChatResponse: - content: list[ContentBlock] = [] - has_tools = False - for _, block in sorted(self._content_blocks.items()): - if block.kind == "tool_use": - has_tools = True - content.append( - block.to_anthropic(fallback_id=f"toolu_switchyard_{len(content)}"), - ) - - response = AnthropicMessage( - id=self._response_id or "msg_switchyard_stream", - type="message", - role="assistant", - content=content, - model=self._served_model, - stop_reason=cast(StopReason, self._stop_reason or ("tool_use" if has_tools else "end_turn")), - stop_sequence=None, - usage=AnthropicUsage( - input_tokens=self._usage.get("input_tokens", 0), - output_tokens=self._usage.get("output_tokens", 0), - cache_creation_input_tokens=self._usage.get( - "cache_creation_input_tokens", - 0, - ), - cache_read_input_tokens=self._usage.get("cache_read_input_tokens", 0), - ), - ) - return ChatResponse.anthropic_completion(response) - - def _merge_usage(self, usage: object) -> None: - if usage is None: - return - for key in ( - "input_tokens", - "output_tokens", - "cache_creation_input_tokens", - "cache_read_input_tokens", - ): - value = usage.get(key) if isinstance(usage, Mapping) else getattr(usage, key, None) - if isinstance(value, int): - self._usage[key] = value - - -class _ResponsesStreamAccumulator: - """Accumulate Responses API stream events into one Response.""" - - def __init__(self, *, served_model: str) -> None: - self._served_model = served_model - self._response_id: str | None = None - self._created_at: float | None = None - self._content = "" - self._tool_calls: dict[int, _ResponsesToolCallState] = {} - self._usage: ResponseUsage | None = None - self._final_response: OpenAIResponse | Mapping[str, object] | None = None - - def consume(self, event: object) -> None: - if isinstance(event, ResponseTextDeltaEvent): - self._content += event.delta - return - - if isinstance(event, (ResponseOutputItemAddedEvent, ResponseOutputItemDoneEvent)): - self._merge_output_item(event.item, event.output_index) - return - - if isinstance(event, ResponseFunctionCallArgumentsDeltaEvent): - tool_call = self._tool_calls.setdefault( - event.output_index, - _ResponsesToolCallState(), - ) - tool_call.arguments += event.delta - return - - if isinstance(event, ResponseCompletedEvent): - self._capture_response_metadata(event.response) - self._final_response = event.response - return - - if isinstance(event, Mapping): - self._consume_mapping_event(event) - return - - response = getattr(event, "response", None) - if isinstance(response, OpenAIResponse): - self._capture_response_metadata(response) - - def as_response(self) -> ChatResponse: - if isinstance(self._final_response, OpenAIResponse): - return ChatResponse.openai_responses_completion(self._final_response) - if isinstance(self._final_response, Mapping): - return ChatResponse.openai_responses_completion( - OpenAIResponse.model_validate(dict(self._final_response)), - ) - - response = OpenAIResponse.model_validate({ - "id": self._response_id or "resp_switchyard_stream", - "object": "response", - "created_at": self._created_at or time.time(), - "status": "completed", - "model": self._served_model, - "output": self._output_items(), - "parallel_tool_calls": False, - "tool_choice": "auto", - "tools": [], - "usage": self._usage.model_dump(mode="json") if self._usage else None, - }) - return ChatResponse.openai_responses_completion(response) - - def _consume_mapping_event(self, event: Mapping[str, object]) -> None: - event_type = event.get("type") - response = event.get("response") - if isinstance(response, Mapping): - self._capture_response_mapping(response) - - if event_type == "response.output_text.delta": - delta = event.get("delta") - if isinstance(delta, str): - self._content += delta - return - if event_type in {"response.output_item.added", "response.output_item.done"}: - self._merge_output_item_mapping( - event.get("item"), - _int_value(event.get("output_index"), 0), - ) - return - if event_type == "response.function_call_arguments.delta": - tool_call = self._tool_calls.setdefault( - _int_value(event.get("output_index"), 0), - _ResponsesToolCallState(), - ) - delta = event.get("delta") - if isinstance(delta, str): - tool_call.arguments += delta - return - if event_type == "response.completed" and isinstance(response, Mapping): - self._final_response = response - - def _capture_response_metadata(self, response: OpenAIResponse) -> None: - self._response_id = response.id or self._response_id - self._served_model = response.model or self._served_model - self._created_at = response.created_at or self._created_at - if response.usage is not None: - self._usage = response.usage - - def _capture_response_mapping(self, response: Mapping[str, object]) -> None: - response_id = response.get("id") - if isinstance(response_id, str): - self._response_id = response_id - model = response.get("model") - if isinstance(model, str): - self._served_model = model - created = response.get("created_at") or response.get("created") - if isinstance(created, (int, float)): - self._created_at = float(created) - usage = response.get("usage") - if isinstance(usage, Mapping): - self._usage = _responses_usage_from_mapping(usage) - - def _merge_output_item(self, item: object, output_index: int) -> None: - if isinstance(item, ResponseOutputMessage): - content = _extract_responses_message_text(item) - if content and not self._content: - self._content = content - return - if not isinstance(item, ResponseFunctionToolCall): - return - tool_call = self._tool_calls.setdefault(output_index, _ResponsesToolCallState()) - tool_call.id = item.id - tool_call.call_id = item.call_id - tool_call.name = item.name - tool_call.arguments = item.arguments - - def _merge_output_item_mapping(self, item: object, output_index: int) -> None: - if not isinstance(item, Mapping): - return - item_type = item.get("type") - if item_type == "message": - content = _extract_responses_message_text_mapping(item) - if content and not self._content: - self._content = content - return - if item_type != "function_call": - return - tool_call = self._tool_calls.setdefault(output_index, _ResponsesToolCallState()) - call_id = item.get("call_id") or item.get("id") - if isinstance(call_id, str): - tool_call.call_id = call_id - name = item.get("name") - if isinstance(name, str): - tool_call.name = name - arguments = item.get("arguments") - if isinstance(arguments, str): - tool_call.arguments = arguments - - def _output_items(self) -> list[dict[str, object]]: - output: list[dict[str, object]] = [] - if self._content: - output.append({ - "type": "message", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": self._content}], - }) - for _, call in sorted(self._tool_calls.items()): - item = call.to_response_output_item( - fallback_id=f"fc_switchyard_{len(output)}", - ) - if item is not None: - output.append(item.model_dump(mode="json", exclude_none=True)) - return output - - -def _coerce_anthropic_event( - event: object, -) -> RawMessageStreamEvent | Mapping[str, object]: - if isinstance(event, Mapping): - try: - return _ANTHROPIC_EVENT_ADAPTER.validate_python(event) - except ValidationError: - return event - return cast(RawMessageStreamEvent, event) - - -def _json_argument_string(value: object) -> str: - if isinstance(value, str): - return value - if isinstance(value, Mapping): - return json.dumps(dict(value)) - return "" - - -def _json_object_from_argument_string(value: object) -> dict[str, object]: - if isinstance(value, Mapping): - return dict(value) - if not isinstance(value, str) or not value: - return {} - try: - parsed = json.loads(value) - except json.JSONDecodeError: - return {} - return dict(parsed) if isinstance(parsed, Mapping) else {} - - -def _openai_finish_reason( - value: str | None, - *, - has_tools: bool, -) -> _OpenAIFinishReason: - if value in {"stop", "length", "tool_calls", "content_filter", "function_call"}: - return cast(_OpenAIFinishReason, value) - return "tool_calls" if has_tools else "stop" - - -def _responses_usage_from_mapping(usage: Mapping[str, object]) -> ResponseUsage: - input_tokens = _int_value(usage.get("input_tokens"), 0) - output_tokens = _int_value(usage.get("output_tokens"), 0) - total_tokens = _int_value( - usage.get("total_tokens"), - input_tokens + output_tokens, - ) - input_details = usage.get("input_tokens_details") - output_details = usage.get("output_tokens_details") - return ResponseUsage( - input_tokens=input_tokens, - output_tokens=output_tokens, - total_tokens=total_tokens, - input_tokens_details=InputTokensDetails( - cached_tokens=( - _int_value(input_details.get("cached_tokens"), 0) - if isinstance(input_details, Mapping) - else 0 - ), - ), - output_tokens_details=OutputTokensDetails( - reasoning_tokens=( - _int_value(output_details.get("reasoning_tokens"), 0) - if isinstance(output_details, Mapping) - else 0 - ), - ), - ) - - -def _extract_responses_message_text(item: ResponseOutputMessage) -> str: - parts: list[str] = [] - for part in item.content: - if isinstance(part, ResponseOutputText): - parts.append(part.text) - return "".join(parts) - - -def _extract_responses_message_text_mapping(item: Mapping[str, object]) -> str: - content = item.get("content") - if not isinstance(content, list): - return "" - parts: list[str] = [] - for part in content: - if not isinstance(part, Mapping): - continue - if part.get("type") not in {"output_text", "text"}: - continue - text = part.get("text") - if isinstance(text, str): - parts.append(text) - return "".join(parts) - - -def _int_value(value: object, default: int) -> int: - return value if isinstance(value, int) else default - - -def _str_value(value: object) -> str | None: - return value if isinstance(value, str) and value else None diff --git a/switchyard/lib/conversation_turn.py b/switchyard/lib/conversation_turn.py deleted file mode 100644 index b075dc4cd..000000000 --- a/switchyard/lib/conversation_turn.py +++ /dev/null @@ -1,65 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Turn counting helpers for conversation-scoped routing decisions.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from switchyard_rust.core import ChatRequest - - -def conversation_turn_number(request: ChatRequest) -> int: - """Return the 1-indexed LLM invocation number for ``request``. - - OpenAI Chat and Anthropic Messages count prior assistant messages. OpenAI - Responses uses a coarse acknowledgement count because responses can emit - multiple model-side items per turn. Unknown or malformed request bodies are - treated as turn 1 so routing falls back to first-turn behavior. - """ - from switchyard_rust.core import ChatRequestType - - body = getattr(request, "body", None) - if not isinstance(body, dict): - return 1 - - match request.request_type: - case ChatRequestType.OPENAI_CHAT | ChatRequestType.ANTHROPIC: - return _count_assistant_messages(body.get("messages")) + 1 - case ChatRequestType.OPENAI_RESPONSES: - return _count_responses_turn(body) - - -def _count_assistant_messages(messages: Any) -> int: - """Count prior assistant messages in chat-style request bodies.""" - if not isinstance(messages, list): - return 0 - return sum( - 1 - for msg in messages - if isinstance(msg, dict) and msg.get("role") == "assistant" - ) - - -def _count_responses_turn(body: dict[str, Any]) -> int: - """Approximate the turn number for an OpenAI Responses request body.""" - input_val = body.get("input") - if isinstance(input_val, str): - return 1 - if not isinstance(input_val, list): - return 1 - - acks = 0 - for item in input_val: - if not isinstance(item, dict): - continue - if item.get("role") == "user": - acks += 1 - elif item.get("type") == "function_call_output": - acks += 1 - return max(acks, 1) - - -__all__ = ["conversation_turn_number"] diff --git a/switchyard/lib/endpoints/__init__.py b/switchyard/lib/endpoints/__init__.py deleted file mode 100644 index fa6c61bd1..000000000 --- a/switchyard/lib/endpoints/__init__.py +++ /dev/null @@ -1,64 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""HTTP endpoint modules (and their SSE helpers). - -Note: HTTP endpoint classes require fastapi (install with [server] extra). -They are lazily loaded to avoid hard dependency on fastapi for library-only users. -""" - -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from switchyard.lib.endpoints.anthropic_messages_endpoint import ( - AnthropicMessagesEndpoint, - ) - from switchyard.lib.endpoints.models_endpoint import ModelsEndpoint - from switchyard.lib.endpoints.openai_chat_endpoint import ( - OpenAIChatEndpoint, - ) - from switchyard.lib.endpoints.responses_endpoint import ResponsesEndpoint - from switchyard.lib.endpoints.routing_log_stats_endpoint import ( - RoutingLogStatsEndpoint, - ) - from switchyard.lib.endpoints.stats_endpoint import StatsEndpoint - -__all__ = [ - "StatsEndpoint", - "AnthropicMessagesEndpoint", - "ModelsEndpoint", - "OpenAIChatEndpoint", - "ResponsesEndpoint", - "RoutingLogStatsEndpoint", -] - - -def __getattr__(name: str) -> Any: - """Lazy load HTTP endpoint classes that require fastapi.""" - if name == "StatsEndpoint": - from switchyard.lib.endpoints.stats_endpoint import StatsEndpoint - return StatsEndpoint - elif name == "AnthropicMessagesEndpoint": - from switchyard.lib.endpoints.anthropic_messages_endpoint import ( - AnthropicMessagesEndpoint, - ) - return AnthropicMessagesEndpoint - elif name == "OpenAIChatEndpoint": - from switchyard.lib.endpoints.openai_chat_endpoint import ( - OpenAIChatEndpoint, - ) - return OpenAIChatEndpoint - elif name == "ModelsEndpoint": - from switchyard.lib.endpoints.models_endpoint import ModelsEndpoint - return ModelsEndpoint - elif name == "ResponsesEndpoint": - from switchyard.lib.endpoints.responses_endpoint import ( - ResponsesEndpoint, - ) - return ResponsesEndpoint - elif name == "RoutingLogStatsEndpoint": - from switchyard.lib.endpoints.routing_log_stats_endpoint import ( - RoutingLogStatsEndpoint, - ) - return RoutingLogStatsEndpoint - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/switchyard/lib/endpoints/anthropic_messages_endpoint.py b/switchyard/lib/endpoints/anthropic_messages_endpoint.py deleted file mode 100644 index a497a4075..000000000 --- a/switchyard/lib/endpoints/anthropic_messages_endpoint.py +++ /dev/null @@ -1,123 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""HTTP endpoint serving a ``Switchyard`` as ``POST /v1/messages`` (Anthropic Messages API). - -Paper-thin by design: wrap the raw JSON body in a Rust-backed Anthropic request, -run the chain, serialize the result. All Anthropic ↔ OpenAI format -conversion lives inside the chain (``TranslationEngine`` and -``TranslationEngine``), so the endpoint itself contains zero -translation logic. - -Streaming contract: - -- When the request body carries ``"stream": true``, the chain's - translation engine surfaces an async iterator of Anthropic event dicts; the - endpoint frames them into Anthropic-style named-event SSE - (``event: message_start\\ndata: {...}\\n\\n``, …) via - :func:`iter_anthropic_sse`. -- Non-streaming requests return the Anthropic ``Message`` body as JSON. -""" - -import logging -from typing import Annotated, Any - -from fastapi import APIRouter, Body, FastAPI, Request -from fastapi.responses import Response - -from switchyard.lib.endpoints.base import Endpoint as NemoSwitchyardEndpoint -from switchyard.lib.endpoints.dispatch import dispatch_chat_request, serialize_chain_result -from switchyard.lib.endpoints.sse_helpers import iter_anthropic_sse -from switchyard.lib.endpoints.upstream_error import ( - context_exhausted_response, - handle_chain_exception, -) -from switchyard.lib.proxy_context import ProxyContext -from switchyard.lib.request_metadata import ( - RequestMetadata, - attach_caller_api_key, - attach_request_metadata, -) -from switchyard_rust.core import ( - ChatRequest, - SwitchyardContextPoolExhaustedError, - SwitchyardContextWindowExceededError, -) - -log = logging.getLogger(__name__) - - -def _strip_unsupported_output_config(body: dict[str, Any]) -> None: - """Drop ``output_config.format`` from an inbound Anthropic body in place. - - Claude Code 2.1.1x sends ``output_config.format`` (a structured-output - schema) that upstream Anthropic model groups reject with HTTP 400. - ``output_config.effort`` is accepted, so only the ``format`` key is - removed; if that leaves ``output_config`` empty it is dropped entirely. - """ - oc = body.get("output_config") - if isinstance(oc, dict) and "format" in oc: - oc.pop("format", None) - if not oc: - body.pop("output_config", None) - - -class AnthropicMessagesEndpoint(NemoSwitchyardEndpoint): - """Composable endpoint that exposes ``POST /v1/messages``.""" - - def register(self, app: FastAPI) -> None: - """Attach ``POST /v1/messages`` onto *app*.""" - router = APIRouter() - - @router.post("/v1/messages", response_model=None) - async def anthropic_messages( - request: Request, - body: Annotated[dict[str, Any], Body(...)], - ) -> Response: - """Anthropic-compatible Messages endpoint.""" - obj = request.app.state.switchyard - _strip_unsupported_output_config(body) - model = str(body.get("model", "")) - stream = bool(body.get("stream")) - log.debug( - "POST /v1/messages model=%s stream=%s keys=%s", - model, - stream, - list(body.keys()), - ) - ctx = ProxyContext() - attach_request_metadata( - ctx, - RequestMetadata.from_headers(request.headers), - request.headers, - ) - attach_caller_api_key(ctx, request.headers) - - chat_request = ChatRequest.anthropic(body) - # Reject semantically invalid input (e.g. empty messages) at the - # inbound boundary; raises SwitchyardInvalidRequestError -> 400. - chat_request.validate() - - try: - result: Any = await dispatch_chat_request(obj, chat_request, ctx) - if not isinstance(result, Response): - log.debug( - "POST /v1/messages chain returned model=%s stream=%s result=%s", - model, - stream, - type(result).__name__, - ) - return serialize_chain_result( - result, stream=stream, sse_iter=iter_anthropic_sse, ctx=ctx - ) - except (SwitchyardContextPoolExhaustedError, SwitchyardContextWindowExceededError) as exc: - return context_exhausted_response(exc, inbound="anthropic") - except Exception as exc: - return handle_chain_exception( - exc, - ctx, - inbound="anthropic", - log_msg=f"POST /v1/messages chain raised model={model}", - ) - - app.include_router(router, tags=["Anthropic Compatible"]) diff --git a/switchyard/lib/endpoints/base.py b/switchyard/lib/endpoints/base.py deleted file mode 100644 index 8e4ac59db..000000000 --- a/switchyard/lib/endpoints/base.py +++ /dev/null @@ -1,39 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Abstract base class for HTTP endpoints.""" - -from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, ClassVar - -if TYPE_CHECKING: - from fastapi import FastAPI - - -class Endpoint(ABC): - """ - Abstract base class for composable HTTP endpoint modules. - - Each endpoint module encapsulates a set of HTTP routes (e.g., OpenAI chat, - Anthropic messages, health checks) and registers them onto a FastAPI app. - - Endpoint modules are composed into a list and registered via - :func:`build_switchyard_app`. - """ - - register_once: ClassVar[bool] = False - """Whether only the first instance of this concrete endpoint type is mounted.""" - - @abstractmethod - def register(self, app: "FastAPI") -> None: - """Register this endpoint's routes onto the FastAPI application. - - Args: - app: The FastAPI application instance to register routes on. - """ - pass - - @property - def name(self) -> str: - """Human-readable name for logging and introspection.""" - return type(self).__name__ diff --git a/switchyard/lib/endpoints/dispatch.py b/switchyard/lib/endpoints/dispatch.py deleted file mode 100644 index 12f55e1a9..000000000 --- a/switchyard/lib/endpoints/dispatch.py +++ /dev/null @@ -1,126 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Shared endpoint dispatch for Python chains and model registries.""" - -from collections.abc import AsyncIterator, Callable -from typing import Any, cast - -from fastapi.responses import JSONResponse, Response, StreamingResponse - -from switchyard.lib.endpoints.error_envelope import error_response -from switchyard.lib.endpoints.route_selection import route_selection_headers -from switchyard.lib.endpoints.upstream_error import record_upstream_attempt_success -from switchyard.lib.proxy_context import ProxyContext -from switchyard.lib.roles import TranslatedResponse -from switchyard.lib.route_table import RouteTable -from switchyard_rust.core import ChatRequest - -_MISSING_MODEL_LABEL = "" - - -def model_not_found_response(model: str) -> JSONResponse: - """Build the OpenAI-compatible error payload for unknown model IDs.""" - return error_response( - 404, - f"No route registered for model {model}", - error_type="model_not_found", - code="model_not_found", - ) - - -def _model_label(model: object | None) -> str: - """Return a stable human label for model-not-found errors.""" - return str(model) if model else _MISSING_MODEL_LABEL - - -def invalid_request_response(message: str, *, code: str = "invalid_request_error") -> JSONResponse: - """Build the OpenAI-compatible error payload for invalid requests.""" - return error_response( - 400, - message, - error_type="invalid_request_error", - code=code, - ) - - -async def dispatch_chat_request( - app_state: object, - chat_request: ChatRequest, - ctx: ProxyContext, -) -> TranslatedResponse | Response: - """Dispatch one request through the configured app state. - - Single-chain apps return already-translated Python payloads because - ``Switchyard`` still owns its terminal translator. - """ - if isinstance(app_state, RouteTable): - model = _model_label(chat_request.model) - try: - table_chain = app_state.lookup_switchyard(model) - except KeyError: - # No upstream call happened — a 404 is not an upstream attempt. - return model_not_found_response(model) - result = await table_chain.call(chat_request, ctx=ctx) - record_upstream_attempt_success(ctx) - return result - - chain: Any = app_state - result = cast(TranslatedResponse, await chain.call(chat_request, ctx=ctx)) - record_upstream_attempt_success(ctx) - return result - - -def model_entries(app_state: object) -> list[dict[str, Any]]: - """Return OpenAI-compatible model entries for table app state.""" - if isinstance(app_state, RouteTable): - return app_state.registered_model_entries() - return [] - - -def model_listing_warnings(app_state: object) -> list[str]: - """Return non-fatal model listing warnings for table-backed apps.""" - if isinstance(app_state, RouteTable): - return app_state.model_listing_warnings() - return [] - - -def model_listing_default(app_state: object) -> str | None: - """Return the default model id advertised by ``GET /v1/models``.""" - if isinstance(app_state, RouteTable): - return app_state.default_model() - return None - - -def serialize_chain_result( - result: Any, - *, - stream: bool, - sse_iter: Callable[[Any], AsyncIterator[str]], - ctx: ProxyContext, -) -> Response: - """Serialize a chain result to the appropriate HTTP response. - - Returns the result itself if it is already a ``Response``, wraps it in a - ``StreamingResponse`` when streaming is requested, or JSON-serializes it. - Any route selection recorded on *ctx* is stamped as ``x-switchyard-*`` - response headers on every branch, pre-built responses included (streaming - too — the backend call completed before the response object is built, so - the selection is final). ``ctx`` is required so a new endpoint cannot - silently opt out of spend attribution. - """ - headers = route_selection_headers(ctx) - if isinstance(result, Response): - # Merge rather than pass through untouched: no current chain path - # yields a pre-built Response after a billed upstream success, but if - # one ever does, dropping the recorded selection here would silently - # break spend attribution. - result.headers.update(headers) - return result - if stream and hasattr(result, "__aiter__"): - return StreamingResponse( - sse_iter(result), media_type="text/event-stream", headers=headers - ) - if hasattr(result, "model_dump"): - return JSONResponse(content=result.model_dump(), headers=headers) - return JSONResponse(content=result, headers=headers) diff --git a/switchyard/lib/endpoints/error_envelope.py b/switchyard/lib/endpoints/error_envelope.py deleted file mode 100644 index d4e995336..000000000 --- a/switchyard/lib/endpoints/error_envelope.py +++ /dev/null @@ -1,171 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Shared HTTP error envelopes for Switchyard LLM-serving endpoints.""" - -import json -from collections.abc import Mapping -from dataclasses import dataclass -from typing import Any - -from fastapi.responses import JSONResponse - -from switchyard.lib.proxy_context import ERROR_SOURCE_PROVIDER, ERROR_SOURCE_SWITCHYARD - -_DEFAULT_UPSTREAM_MESSAGE = "upstream returned HTTP {status}" - -#: Response header naming the layer that originated the error: ``switchyard`` -#: (this proxy rejected or failed the request itself) or ``provider`` (an -#: upstream LLM failure passed through). Layers above Switchyard (e.g. a -#: LiteLLM front proxy) are expected to tag their own failures the same way -#: and propagate this header from below — Switchyard cannot see them. The -#: values live in :mod:`switchyard.lib.proxy_context` so FastAPI-free backend -#: code can stamp them. -ERROR_SOURCE_HEADER = "x-switchyard-error-source" - -#: Response header carrying the upstream model actually attempted when the -#: surfaced failure happened, when a routing selection took place. -UPSTREAM_MODEL_HEADER = "x-switchyard-upstream-model" - - -def error_payload( - message: str, - *, - error_type: str, - code: str, - extra: Mapping[str, object] | None = None, -) -> dict[str, dict[str, object]]: - """Return the normalized JSON body used by all LLM HTTP endpoints.""" - error: dict[str, object] = { - "message": message, - "type": error_type, - "code": code, - } - if extra: - error.update(extra) - return {"error": error} - - -def error_response( - status_code: int, - message: str, - *, - error_type: str, - code: str, - extra: Mapping[str, object] | None = None, - error_source: str | None = ERROR_SOURCE_SWITCHYARD, - upstream_model: str | None = None, -) -> JSONResponse: - """Build a JSONResponse with Switchyard's normalized error envelope. - - Stamps the failure-source headers: every direct caller synthesizes a - Switchyard-originated envelope, so ``error_source`` defaults to - ``switchyard``; the upstream passthrough path overrides it with - ``provider``. Headers rather than body fields keep the passthrough - contract intact — provider error bodies flow through unmodified. - """ - headers: dict[str, str] = {} - if error_source: - headers[ERROR_SOURCE_HEADER] = error_source - if upstream_model: - headers[UPSTREAM_MODEL_HEADER] = upstream_model - return JSONResponse( - status_code=status_code, - content=error_payload(message, error_type=error_type, code=code, extra=extra), - headers=headers or None, - ) - - -def upstream_error_response( - status_code: int, - body: object, - *, - error_source: str = ERROR_SOURCE_PROVIDER, - upstream_model: str | None = None, -) -> JSONResponse: - """Normalize an upstream provider error body into Switchyard's envelope. - - ``error_source`` defaults to ``provider`` — this path renders upstream - failures — but a backend that deliberately routes its own rejection - through the upstream-status stash (e.g. the ``caller_required`` 401) - overrides it back to ``switchyard`` via ``ctx``. - """ - parsed = _upstream_error_fields(status_code, body) - return error_response( - status_code, - parsed.message, - error_type=parsed.error_type, - code=parsed.code, - extra=parsed.extra, - error_source=error_source, - upstream_model=upstream_model, - ) - - -@dataclass(frozen=True) -class _UpstreamErrorFields: - """Internal value object for provider error fields after normalization.""" - - message: str - error_type: str = "upstream_error" - code: str = "upstream_error" - extra: Mapping[str, object] | None = None - - -def _upstream_error_fields(status_code: int, body: object) -> _UpstreamErrorFields: - """Extract stable error fields from common provider error shapes.""" - default_message = _DEFAULT_UPSTREAM_MESSAGE.format(status=status_code) - if isinstance(body, str): - return _UpstreamErrorFields(message=body or default_message) - if isinstance(body, Mapping): - return _fields_from_mapping(status_code, body) - if isinstance(body, list): - return _UpstreamErrorFields(message=_compact_json(body)) - return _UpstreamErrorFields(message=default_message) - - -def _fields_from_mapping(status_code: int, body: Mapping[str, object]) -> _UpstreamErrorFields: - """Handle OpenAI-style ``{"error": {...}}`` and flat error dictionaries.""" - error = body.get("error") - source = error if isinstance(error, Mapping) else body - default_message = _DEFAULT_UPSTREAM_MESSAGE.format(status=status_code) - - message = _string_field(source, "message") or _compact_json(body) or default_message - error_type = _string_field(source, "type") or "upstream_error" - code = _string_or_number_field(source, "code") or ( - error_type if error_type != "upstream_error" else "upstream_error" - ) - extra = { - key: value - for key, value in { - "param": _string_or_number_field(source, "param"), - }.items() - if value is not None - } - return _UpstreamErrorFields( - message=message, - error_type=error_type, - code=code, - extra=extra, - ) - - -def _string_field(source: Mapping[str, object], key: str) -> str | None: - value = source.get(key) - return value if isinstance(value, str) and value else None - - -def _string_or_number_field(source: Mapping[str, object], key: str) -> str | None: - value = source.get(key) - if isinstance(value, str) and value: - return value - if isinstance(value, int | float): - return str(value) - return None - - -def _compact_json(value: Any) -> str: - try: - return json.dumps(value, separators=(",", ":"), sort_keys=True) - except TypeError: - return str(value) diff --git a/switchyard/lib/endpoints/models_endpoint.py b/switchyard/lib/endpoints/models_endpoint.py deleted file mode 100644 index 9c735fdf8..000000000 --- a/switchyard/lib/endpoints/models_endpoint.py +++ /dev/null @@ -1,42 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""HTTP endpoint serving ``GET /v1/models`` for local model discovery.""" - -import logging - -from fastapi import APIRouter, FastAPI, Request -from fastapi.responses import JSONResponse - -from switchyard.lib.endpoints.base import Endpoint as NemoSwitchyardEndpoint -from switchyard.lib.endpoints.dispatch import ( - model_entries, - model_listing_default, - model_listing_warnings, -) -from switchyard.lib.model_listing import model_list_payload - -log = logging.getLogger(__name__) - - -class ModelsEndpoint(NemoSwitchyardEndpoint): - """Expose registered model ids for clients with model discovery.""" - - def register(self, app: FastAPI) -> None: - """Attach ``GET /v1/models`` onto *app*.""" - router = APIRouter() - - @router.get("/v1/models", response_model=None) - async def models(request: Request) -> JSONResponse: - obj = request.app.state.switchyard - entries = model_entries(obj) - log.debug("GET /v1/models returned %d model(s)", len(entries)) - return JSONResponse( - content=model_list_payload( - entries, - default_model=model_listing_default(obj), - warnings=model_listing_warnings(obj), - ) - ) - - app.include_router(router, tags=["Model Discovery"]) diff --git a/switchyard/lib/endpoints/openai_chat_endpoint.py b/switchyard/lib/endpoints/openai_chat_endpoint.py deleted file mode 100644 index fb3880df5..000000000 --- a/switchyard/lib/endpoints/openai_chat_endpoint.py +++ /dev/null @@ -1,102 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""HTTP endpoint serving a ``Switchyard`` as ``POST /v1/chat/completions``. - -The class is stateless — at request time it reads the switchyard from -``request.app.state.switchyard``. Wire-up is performed by the -``build_switchyard_app()`` convenience factory. - -Streaming contract: - -- When the request body carries ``"stream": true``, the chain's - translation engine surfaces an async iterator of ``ChatCompletionChunk``; the - endpoint wraps it in a ``StreamingResponse`` emitting OpenAI-style - SSE frames (``data: {...}\\n\\n`` + ``data: [DONE]\\n\\n``). -- Upstream failures (auth, rate-limit, connection) surface before the - ``StreamingResponse`` is constructed — they propagate as exceptions - to the global handler and map to proper HTTP error responses. Only - mid-stream iteration errors land in the SSE error branch of - :func:`iter_chat_completion_sse`. -""" - -import logging -from typing import Annotated, Any - -from fastapi import APIRouter, Body, FastAPI, Request -from fastapi.responses import Response - -from switchyard.lib.endpoints.base import Endpoint as NemoSwitchyardEndpoint -from switchyard.lib.endpoints.dispatch import dispatch_chat_request, serialize_chain_result -from switchyard.lib.endpoints.sse_helpers import iter_chat_completion_sse -from switchyard.lib.endpoints.upstream_error import ( - context_exhausted_response, - handle_chain_exception, -) -from switchyard.lib.proxy_context import ProxyContext -from switchyard.lib.request_metadata import ( - RequestMetadata, - attach_caller_api_key, - attach_request_metadata, -) -from switchyard_rust.core import ( - ChatRequest, - SwitchyardContextPoolExhaustedError, - SwitchyardContextWindowExceededError, -) - -log = logging.getLogger(__name__) - - -class OpenAIChatEndpoint(NemoSwitchyardEndpoint): - """Composable endpoint that exposes ``POST /v1/chat/completions``. - - Reads the raw JSON body, wraps it in a Rust-backed OpenAI chat request (no - validation or field-stripping, so provider-specific fields pass - through transparently), runs the chain, and either JSON-serializes - the result (non-streaming) or wraps the async chunk iterator in an - SSE ``StreamingResponse`` (streaming). - - Streaming support is limited to same-format passthrough today — - i.e. OpenAI Chat Completions inbound against an OpenAI-native - backend. Cross-format streaming (Anthropic / Responses inbound) - raises ``NotImplementedError`` from ``TranslationEngine`` - until streaming translation lands for those formats. - """ - - def register(self, app: FastAPI) -> None: - """Attach ``POST /v1/chat/completions`` onto *app*.""" - router = APIRouter() - - @router.post("/v1/chat/completions", response_model=None) - async def chat_completions( - request: Request, - body: Annotated[dict[str, Any], Body(...)], - ) -> Response: - """OpenAI-compatible Chat Completions endpoint.""" - obj = request.app.state.switchyard - chat_request = ChatRequest.openai_chat(body) - # Reject semantically invalid input (e.g. empty messages) at the - # inbound boundary; raises SwitchyardInvalidRequestError -> 400. - chat_request.validate() - ctx = ProxyContext() - attach_request_metadata( - ctx, - RequestMetadata.from_headers(request.headers), - request.headers, - ) - attach_caller_api_key(ctx, request.headers) - stream = bool(body.get("stream")) - try: - result: Any = await dispatch_chat_request(obj, chat_request, ctx) - return serialize_chain_result( - result, stream=stream, sse_iter=iter_chat_completion_sse, ctx=ctx - ) - except (SwitchyardContextPoolExhaustedError, SwitchyardContextWindowExceededError) as exc: - return context_exhausted_response(exc, inbound="openai") - except Exception as exc: - return handle_chain_exception( - exc, ctx, inbound="openai", log_msg="POST /v1/chat/completions chain raised" - ) - - app.include_router(router, tags=["OpenAI Compatible"]) diff --git a/switchyard/lib/endpoints/outcome_metrics.py b/switchyard/lib/endpoints/outcome_metrics.py deleted file mode 100644 index dacf87fbf..000000000 --- a/switchyard/lib/endpoints/outcome_metrics.py +++ /dev/null @@ -1,249 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Outcome counters used to compute router-vs-direct error-rate ratios. - -Three process-wide counters published on ``/metrics``: - -* ``switchyard_client_responses_total{outcome}`` — every HTTP response - returned to a client on an LLM-serving route. The denominator for the - router-served error rate. -* ``switchyard_upstream_attempts_total{outcome, code}`` — every individual - upstream call attempt (including ones absorbed by retry). The - denominator for the direct-to-endpoint baseline error rate. The ``code`` - label carries the raw upstream HTTP status (``"429"``, ``"500"`` …) so a - dashboard can plot the error-code distribution over time; - ``code="none"`` marks a non-HTTP failure (network error, pre-status - timeout) that has no status code. Unknown codes are clamped to their - class (``"4xx"`` / ``"5xx"`` / …) to keep label cardinality bounded. -* ``switchyard_router_retry_recovered_total`` — global counter - incremented whenever a request's first upstream attempt failed and a - subsequent attempt succeeded — direct evidence the steering logic - rescued the request. - -Bucket semantics: - -* ``success`` — HTTP 2xx. -* ``retryable_error`` — HTTP 429 / 500 / 504 — the categories the - success criterion measures (router should be absorbing these). -* ``other_error`` — everything else (400 / 401 / 403 / 422 / …), - i.e. bad payload, bad credentials, high-reasoning timeout. Excluded - from the success criterion per the spec. - -Computing the ratios from these:: - - router_error_rate = client_responses{outcome="retryable_error"} - / sum(client_responses) - - direct_error_rate = sum(upstream_attempts{outcome="retryable_error"}) - / sum(upstream_attempts) - - error_rate_reduction = direct_error_rate - router_error_rate - -Because ``upstream_attempts`` now carries the ``code`` label, a bare -selector returns one series per code — always aggregate it with ``sum()`` -when you want the layer total. The error-code distribution itself is just -``sum by (code) (rate(upstream_attempts{code!="200"}[5m]))``. - -The two layers have different denominators by design: one client request -can produce N upstream attempts (retry fan-out), and that asymmetry is -exactly the reason a health-aware router reduces the rate seen by the -client. -""" - -from __future__ import annotations - -from threading import Lock -from typing import Literal - -OutcomeBucket = Literal["success", "retryable_error", "other_error"] - -#: HTTP status codes the success criterion counts as router-rescuable -#: errors. 429 (rate limit), 500 (server error), 504 (gateway timeout). -RETRYABLE_STATUSES: frozenset[int] = frozenset({429, 500, 504}) - -#: Status codes emitted verbatim as the ``code`` label. Anything else seen -#: on the wire is clamped to its class (``"4xx"`` / ``"5xx"`` / …) so a -#: misbehaving upstream returning oddball codes cannot inflate label -#: cardinality. Covers the common success / client-error / server-error -#: codes an LLM endpoint actually returns. -KNOWN_STATUS_CODES: frozenset[int] = frozenset( - {200, 400, 401, 403, 404, 408, 409, 422, 429, 500, 502, 503, 504} -) - -#: ``code`` label value for a non-HTTP failure (network error, pre-status -#: timeout) — the request never received a status line, so there is no code. -NO_STATUS_CODE: str = "none" - -_lock = Lock() -_client_responses: dict[str, int] = { - "success": 0, - "retryable_error": 0, - "other_error": 0, -} - - -def _seed_upstream() -> dict[tuple[str, str], int]: - """Fresh upstream-attempt counter with canonical ``(outcome, code)`` series at 0. - - Seeding the codes a dashboard plots means their time series exist from - process start, so a Grafana ``rate()`` renders a flat zero line rather - than "no data" before the first matching attempt. Non-seeded codes (a - one-off 403, say) are created lazily on first occurrence. - """ - return { - ("success", "200"): 0, - ("retryable_error", "429"): 0, - ("retryable_error", "500"): 0, - ("retryable_error", "504"): 0, - ("retryable_error", NO_STATUS_CODE): 0, - } - - -#: Keyed by ``(outcome, code)``: ``code`` is the upstream HTTP status as a -#: string, ``"none"`` for a non-HTTP failure, or a clamped ``"Nxx"`` class. -_upstream_attempts: dict[tuple[str, str], int] = _seed_upstream() -_retry_recovered: int = 0 - - -def classify(status_code: int) -> OutcomeBucket: - """Map an HTTP status code to its outcome bucket. - - 2xx → ``success``. The codes listed in :data:`RETRYABLE_STATUSES` - (429 / 500 / 504) → ``retryable_error``. Everything else (1xx, 3xx, - most 4xx, other 5xx) → ``other_error``. - """ - if 200 <= status_code < 300: - return "success" - if status_code in RETRYABLE_STATUSES: - return "retryable_error" - return "other_error" - - -def code_label(status_code: int | None) -> str: - """Render the ``code`` label for one upstream attempt. - - ``None`` (non-HTTP failure) → :data:`NO_STATUS_CODE`. A code in - :data:`KNOWN_STATUS_CODES` is emitted verbatim (``"429"``). Any other - HTTP code is clamped to its class (``"4xx"``, ``"5xx"``, …), and an - out-of-range value to ``"other"``, so label cardinality stays bounded - no matter what an upstream returns. - """ - if status_code is None: - return NO_STATUS_CODE - if status_code in KNOWN_STATUS_CODES: - return str(status_code) - if 100 <= status_code < 600: - return f"{status_code // 100}xx" - return "other" - - -def record_client_response(status_code: int) -> None: - """Record one HTTP response sent to a client on an LLM-serving route.""" - bucket = classify(status_code) - with _lock: - _client_responses[bucket] += 1 - - -def record_upstream_attempt(status_code: int | None) -> None: - """Record one individual upstream attempt outcome. - - ``status_code=None`` is used for non-HTTP failures (network errors, - pre-status timeouts) and is bucketed as ``retryable_error`` — those - are exactly the kind of fault a health-aware router should be able - to absorb by retrying on a different endpoint. - """ - bucket: OutcomeBucket - if status_code is None: - bucket = "retryable_error" - else: - bucket = classify(status_code) - key = (bucket, code_label(status_code)) - with _lock: - _upstream_attempts[key] = _upstream_attempts.get(key, 0) + 1 - - -def record_retry_recovered() -> None: - """Record that a retry succeeded after at least one prior attempt failed. - - Direct evidence the router's steering logic kicked in: without - retry, this request would have surfaced as a client-side error. - """ - global _retry_recovered - with _lock: - _retry_recovered += 1 - - -def render_lines() -> list[str]: - """Render the current counter state as Prometheus exposition lines. - - Returns an ordered list of lines (no trailing newline). The - ``/metrics`` endpoint concatenates this with the accumulator output. - """ - with _lock: - client = dict(_client_responses) - upstream = dict(_upstream_attempts) - recovered = _retry_recovered - - lines: list[str] = [] - lines.append( - "# HELP switchyard_client_responses_total " - "HTTP responses returned to clients on LLM-serving routes, " - "bucketed by outcome (success / retryable_error / other_error)." - ) - lines.append("# TYPE switchyard_client_responses_total counter") - for outcome in ("success", "retryable_error", "other_error"): - lines.append( - f'switchyard_client_responses_total{{outcome="{outcome}"}} ' - f"{client[outcome]}" - ) - - lines.append( - "# HELP switchyard_upstream_attempts_total " - "Individual upstream call attempts, bucketed by outcome and labelled " - "with the upstream HTTP status code (code=\"none\" for non-HTTP " - "failures). One client request can produce multiple attempts via retry." - ) - lines.append("# TYPE switchyard_upstream_attempts_total counter") - # Sorted for deterministic exposition order across scrapes. - for outcome, code in sorted(upstream): - lines.append( - f'switchyard_upstream_attempts_total{{outcome="{outcome}",code="{code}"}} ' - f"{upstream[(outcome, code)]}" - ) - - lines.append( - "# HELP switchyard_router_retry_recovered_total " - "Requests whose first upstream attempt failed but a subsequent " - "attempt succeeded — direct evidence steering logic rescued the request." - ) - lines.append("# TYPE switchyard_router_retry_recovered_total counter") - lines.append(f"switchyard_router_retry_recovered_total {recovered}") - - return lines - - -def _reset_for_tests() -> None: - """Zero every counter — tests only.""" - global _retry_recovered, _upstream_attempts - with _lock: - for key in _client_responses: - _client_responses[key] = 0 - # Rebuild rather than zero in place: drops any lazily-added codes so - # each test starts from the same canonical seed. - _upstream_attempts = _seed_upstream() - _retry_recovered = 0 - - -__all__ = [ - "KNOWN_STATUS_CODES", - "NO_STATUS_CODE", - "RETRYABLE_STATUSES", - "OutcomeBucket", - "classify", - "code_label", - "record_client_response", - "record_retry_recovered", - "record_upstream_attempt", - "render_lines", -] diff --git a/switchyard/lib/endpoints/prometheus_emitter.py b/switchyard/lib/endpoints/prometheus_emitter.py deleted file mode 100644 index aafe0274c..000000000 --- a/switchyard/lib/endpoints/prometheus_emitter.py +++ /dev/null @@ -1,70 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Process-wide table for objects that contribute extra ``/metrics`` lines. - -The accumulator-derived exposition rendered by -:func:`switchyard.lib.prometheus_exposition.render_prometheus` covers -request flow (counts, tokens, latency). Components that own state which -is not request-derived register an emitter here so their lines appear on -the same ``/metrics`` scrape rather than a sidecar URL. - -Single-process table by design: a Switchyard server is one process, -emitters are owned by component lifetimes, and the table is -write-once-read-many across startup. -""" - -from __future__ import annotations - -from collections.abc import Callable - -PrometheusEmitter = Callable[[], list[str]] -"""A no-arg callable returning Prometheus exposition lines (no trailing newline). - -Each call snapshots the emitter's current state. The table composes -output in registration order; emitters must not assume any ordering -relative to other emitters or to the accumulator-derived block. -""" - -_EMITTERS: list[PrometheusEmitter] = [] - - -def register(emitter: PrometheusEmitter) -> None: - """Register an emitter. Idempotent — re-registering is a no-op.""" - if emitter not in _EMITTERS: - _EMITTERS.append(emitter) - - -def unregister(emitter: PrometheusEmitter) -> None: - """Remove a previously-registered emitter. No-op if not present. - - Backends call this from their ``shutdown()`` hook so a re-built chain - does not leave a stale closure pointing at a torn-down backend. - """ - try: - _EMITTERS.remove(emitter) - except ValueError: - pass - - -def render() -> str: - """Compose registered emitter output as Prometheus exposition text. - - Returns an empty string when no emitter is registered, so callers can - unconditionally concatenate the result to the accumulator-derived - exposition without producing trailing whitespace artefacts. - """ - lines: list[str] = [] - for emitter in _EMITTERS: - lines.extend(emitter()) - if not lines: - return "" - return "\n".join(lines) + "\n" - - -def _clear_for_tests() -> None: - """Drop every registered emitter — test fixtures only.""" - _EMITTERS.clear() - - -__all__ = ["PrometheusEmitter", "register", "unregister", "render"] diff --git a/switchyard/lib/endpoints/responses_endpoint.py b/switchyard/lib/endpoints/responses_endpoint.py deleted file mode 100644 index d5ae97e2e..000000000 --- a/switchyard/lib/endpoints/responses_endpoint.py +++ /dev/null @@ -1,101 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""HTTP endpoint serving a ``Switchyard`` as ``POST /v1/responses`` (OpenAI Responses API). - -Paper-thin by design: wrap the raw JSON body in a Rust-backed Responses request, -run the chain, serialize the result. All Responses ↔ Chat Completions -format conversion lives inside the chain's ``TranslationEngine``, so the -endpoint itself contains zero translation logic. - -Streaming contract: - -- When the request body carries ``"stream": true``, the chain's - translation engine surfaces an async iterator of pre-formatted Responses API - SSE frames; :func:`iter_preframed_sse` forwards them verbatim through - a ``StreamingResponse`` with mid-stream error quarantine. -- Non-streaming requests return the Responses ``Response`` body as JSON. -""" - -import logging -from typing import Annotated, Any - -from fastapi import APIRouter, Body, FastAPI, Request -from fastapi.responses import Response - -from switchyard.lib.endpoints.base import Endpoint as NemoSwitchyardEndpoint -from switchyard.lib.endpoints.dispatch import dispatch_chat_request, serialize_chain_result -from switchyard.lib.endpoints.sse_helpers import iter_preframed_sse -from switchyard.lib.endpoints.upstream_error import ( - context_exhausted_response, - handle_chain_exception, -) -from switchyard.lib.proxy_context import ProxyContext -from switchyard.lib.request_metadata import ( - RequestMetadata, - attach_caller_api_key, - attach_request_metadata, -) -from switchyard_rust.core import ( - ChatRequest, - SwitchyardContextPoolExhaustedError, - SwitchyardContextWindowExceededError, -) - -log = logging.getLogger(__name__) - - -class ResponsesEndpoint(NemoSwitchyardEndpoint): - """Composable endpoint that exposes ``POST /v1/responses``.""" - - def register(self, app: FastAPI) -> None: - """Attach ``POST /v1/responses`` onto *app*.""" - router = APIRouter() - - @router.post("/v1/responses", response_model=None) - async def responses( - request: Request, - body: Annotated[dict[str, Any], Body(...)], - ) -> Response: - """OpenAI-compatible Responses endpoint.""" - obj = request.app.state.switchyard - model = str(body.get("model", "")) - stream = bool(body.get("stream")) - log.debug( - "POST /v1/responses model=%s stream=%s keys=%s", - model, - stream, - list(body.keys()), - ) - - chat_request = ChatRequest.openai_responses(body) - ctx = ProxyContext() - attach_request_metadata( - ctx, - RequestMetadata.from_headers(request.headers), - request.headers, - ) - attach_caller_api_key(ctx, request.headers) - try: - result: Any = await dispatch_chat_request(obj, chat_request, ctx) - if not isinstance(result, Response): - log.debug( - "POST /v1/responses chain returned model=%s stream=%s result=%s", - model, - stream, - type(result).__name__, - ) - return serialize_chain_result( - result, stream=stream, sse_iter=iter_preframed_sse, ctx=ctx - ) - except (SwitchyardContextPoolExhaustedError, SwitchyardContextWindowExceededError) as exc: - return context_exhausted_response(exc, inbound="openai-responses") - except Exception as exc: - return handle_chain_exception( - exc, - ctx, - inbound="openai-responses", - log_msg=f"POST /v1/responses chain raised model={model}", - ) - - app.include_router(router, tags=["OpenAI Responses"]) diff --git a/switchyard/lib/endpoints/route_selection.py b/switchyard/lib/endpoints/route_selection.py deleted file mode 100644 index 209110535..000000000 --- a/switchyard/lib/endpoints/route_selection.py +++ /dev/null @@ -1,68 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Route-selection response headers for spend/tokenomics attribution. - -Maps the :data:`CTX_ROUTE_SELECTION` record a routing backend stored on -``ctx`` (see :mod:`switchyard.lib.proxy_context`) to the ``x-switchyard-*`` -response headers a front proxy such as LiteLLM copies into its parent -spend-log row. Shared by the success serializer (``dispatch``) and the -error path (``upstream_error``) — a failure that happens *after* a billed -upstream success must still expose the selection, or the provider spend-log -row's correlation id becomes unjoinable. -""" - -from collections.abc import Mapping - -from switchyard.lib.proxy_context import CTX_ROUTE_SELECTION, ProxyContext - -#: Response headers exposing the route selection behind an upstream call, -#: carrying the same correlation id the provider row received via the -#: outbound ``x-litellm-spend-logs-metadata`` header. -ROUTER_MODEL_HEADER = "x-switchyard-router-model" -SELECTED_MODEL_HEADER = "x-switchyard-selected-model" -SELECTED_PROVIDER_HEADER = "x-switchyard-selected-provider" -ROUTER_CORRELATION_ID_HEADER = "x-switchyard-router-correlation-id" - -_ROUTE_SELECTION_RESPONSE_HEADERS = ( - (ROUTER_MODEL_HEADER, "router_model"), - (SELECTED_MODEL_HEADER, "router_selected_model"), - (SELECTED_PROVIDER_HEADER, "router_selected_provider"), - (ROUTER_CORRELATION_ID_HEADER, "router_correlation_id"), -) - - -def _is_header_value_safe(value: str) -> bool: - """Whether *value* can be emitted as an HTTP/1.1 response-header value. - - ``router_model`` echoes the client-supplied model string, so it must be - re-validated as header material: Starlette encodes response-header values - as latin-1 (a non-encodable value would fail response construction after - the upstream call already succeeded and was billed), and CTL characters — - CR/LF above all — would be a response-splitting vector on permissive - ASGI stacks. - """ - try: - value.encode("latin-1") - except UnicodeEncodeError: - return False - return not any(ord(ch) < 0x20 or ord(ch) == 0x7F for ch in value) - - -def route_selection_headers(ctx: ProxyContext) -> dict[str, str]: - """Response headers for the route selection recorded on *ctx*, if any. - - Empty when no routing backend recorded a selection (passthrough chains, - failures before any upstream success). A recorded field that is absent or - not emittable as a header value is skipped — headers never carry - placeholder or unsafe values. - """ - selection = ctx.metadata.get(CTX_ROUTE_SELECTION) - if not isinstance(selection, Mapping): - return {} - headers: dict[str, str] = {} - for header_name, selection_key in _ROUTE_SELECTION_RESPONSE_HEADERS: - value = selection.get(selection_key) - if isinstance(value, str) and value and _is_header_value_safe(value): - headers[header_name] = value - return headers diff --git a/switchyard/lib/endpoints/routing_log_stats_endpoint.py b/switchyard/lib/endpoints/routing_log_stats_endpoint.py deleted file mode 100644 index 4491f877b..000000000 --- a/switchyard/lib/endpoints/routing_log_stats_endpoint.py +++ /dev/null @@ -1,45 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""HTTP access to session-scoped aggregates from the durable routing log.""" - -from __future__ import annotations - -import asyncio -from typing import TYPE_CHECKING - -from fastapi import APIRouter, HTTPException - -from switchyard.lib.endpoints.base import Endpoint - -if TYPE_CHECKING: - from fastapi import FastAPI - - from switchyard.lib.processors.routing_log_response_processor import ( - RoutingLogResponseProcessor, - ) - - -class RoutingLogStatsEndpoint(Endpoint): - """Expose one trial session's model and token aggregates.""" - - register_once = True - - def __init__(self, processor: RoutingLogResponseProcessor) -> None: - self._processor = processor - - def register(self, app: FastAPI) -> None: - routes = APIRouter() - processor = self._processor - - async def get_session_stats(session_id: str) -> dict[str, object]: - snapshot = await asyncio.to_thread(processor.snapshot_session, session_id) - if snapshot is None: - raise HTTPException(status_code=404, detail="routing session not found") - return snapshot - - routes.get("/v1/routing/session-stats")(get_session_stats) - app.include_router(routes, tags=["Routing log"]) - - -__all__ = ["RoutingLogStatsEndpoint"] diff --git a/switchyard/lib/endpoints/sse_helpers.py b/switchyard/lib/endpoints/sse_helpers.py deleted file mode 100644 index d40224bb8..000000000 --- a/switchyard/lib/endpoints/sse_helpers.py +++ /dev/null @@ -1,207 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Shared SSE serialization helpers for HTTP endpoints. - -Pure async generators that turn typed response streams into the raw -SSE frames FastAPI's ``StreamingResponse`` expects. Each wire format -(OpenAI Chat Completions, Anthropic Messages, Responses API) has its -own helper with the format-specific framing contract. - -Error contract (shared across all helpers): -(:mod:`switchyard.server.endpoints.openai_endpoint`): - -- The endpoint awaits ``switchyard.call()`` *before* handing the stream - to ``StreamingResponse``, so upstream auth / connection / rate-limit - failures raise exceptions that the global handler turns into proper - HTTP error responses. -- Only failures during chunk iteration land in ``except`` here. At - that point HTTP 200 has already been committed, so the best we can - do is emit a final SSE error frame (format-specific shape). -""" - -from __future__ import annotations - -import inspect -import json -import logging -from collections.abc import AsyncGenerator, AsyncIterator, Mapping - -from anthropic.types import RawMessageStreamEvent -from openai.types.chat import ChatCompletionChunk - -log = logging.getLogger(__name__) - - -async def _aclose_stream(stream: object) -> None: - """Best-effort close of an upstream response stream. - - On every exit path — normal completion, mid-stream error, or the client - disconnecting (which makes the ASGI server ``aclose()`` the SSE generator, - raising ``GeneratorExit`` at the suspended ``yield``) — the upstream stream - must be closed so its underlying httpx connection is returned to the pool - instead of leaking. ``ChatResponseStream`` and async generators expose - ``aclose``; SDK ``AsyncStream`` objects expose ``close``; either may be a - coroutine. Closing must never mask the original control flow, so failures - are logged and swallowed. - """ - closer = getattr(stream, "aclose", None) or getattr(stream, "close", None) - if closer is None: - return - try: - result = closer() - if inspect.isawaitable(result): - await result - except Exception as exc: - log.debug("Failed to close upstream stream: %s: %s", type(exc).__name__, exc) - - -async def iter_chat_completion_sse( - stream: AsyncIterator[ChatCompletionChunk], -) -> AsyncGenerator[str, None]: - """Serialize a Chat Completions chunk stream to OpenAI-style SSE frames. - - Accepts any async iterator of objects with ``model_dump()`` (OpenAI - SDK ``ChatCompletionChunk``) or ``to_dict()``; falls back to - ``dict(chunk)`` for dict-likes. This is deliberately duck-typed — - the same helper serves the backend's raw ``ResponseStream`` and - any future transformed stream that still yields chunk-like objects. - - Emits ``data: [DONE]\\n\\n`` after successful completion, matching - the OpenAI streaming contract. - - Args: - stream: Async iterator of ``ChatCompletionChunk`` (or compatible). - - Yields: - SSE-framed strings suitable for ``StreamingResponse``. - """ - try: - async for chunk in stream: - if hasattr(chunk, "model_dump"): - chunk_dict = chunk.model_dump(exclude_none=True) - elif hasattr(chunk, "to_dict"): - chunk_dict = chunk.to_dict() - else: - chunk_dict = ( - dict(chunk) if hasattr(chunk, "__iter__") else {"data": str(chunk)} - ) - yield f"data: {json.dumps(chunk_dict)}\n\n" - - yield "data: [DONE]\n\n" - - except Exception as e: - log.error("Error during chat completions streaming: %s: %s", type(e).__name__, e) - error_data = { - "error": { - "message": repr(e)[:200], - "type": "internal_error", - "code": "internal_chain_error", - } - } - yield f"data: {json.dumps(error_data)}\n\n" - finally: - await _aclose_stream(stream) - - -async def iter_anthropic_sse( - events: AsyncIterator[RawMessageStreamEvent | Mapping[str, object]], -) -> AsyncGenerator[str, None]: - """Frame Anthropic events into ``event: \\ndata: \\n\\n``. - - Anthropic's SSE contract carries a named event per frame (unlike - OpenAI Chat Completions, which only uses ``data:`` lines). The - event name comes from each event's ``"type"`` field — - ``message_start``, ``content_block_delta``, ``message_stop``, etc. - - Accepts two producer shapes: - - * Plain ``dict`` events (from ``stream_openai_to_anthropic`` when - the chain translates OpenAI → Anthropic on the fly). - * Pydantic ``RawMessageStreamEvent`` models (from the Anthropic - SDK's ``AsyncStream`` when the backend speaks Anthropic natively - — see :class:`AnthropicNativeBackend`). Serialized via - ``model_dump(exclude_none=True)``. - - No ``[DONE]`` terminator: Anthropic signals end-of-stream with a - ``message_stop`` event, not a sentinel frame. - - Mid-stream iteration failures emit a final ``event: error`` frame - with an ``{"error": {...}}`` payload and terminate — same error - quarantine pattern as :func:`iter_chat_completion_sse`. - - Args: - events: Async iterator of Anthropic events — dicts or pydantic - models with ``model_dump``. - - Yields: - SSE-framed strings suitable for ``StreamingResponse``. - """ - try: - async for event in events: - if isinstance(event, Mapping): - event_dict = dict(event) - else: - event_dict = event.model_dump(exclude_none=True) - event_type = event_dict.get("type", "message") - yield f"event: {event_type}\ndata: {json.dumps(event_dict)}\n\n" - except Exception as e: - log.error("Error during anthropic streaming: %s: %s", type(e).__name__, e) - error_data = { - "type": "error", - "error": { - "message": repr(e)[:200], - "type": "internal_error", - }, - } - yield f"event: error\ndata: {json.dumps(error_data)}\n\n" - finally: - await _aclose_stream(events) - - -async def iter_preframed_sse( - frames: AsyncIterator[object], -) -> AsyncGenerator[str, None]: - """Forward Responses SSE strings or frame native Responses events. - - The Responses API translator (:func:`stream_chat_to_responses_sse`) - already yields fully-formatted SSE frames (``"event: ...\\ndata: ...\\n\\n"``) - so the endpoint just needs a thin wrapper that preserves the same - mid-stream error contract as the other helpers. Native Responses upstreams - yield JSON events instead, so this helper frames those mappings using each - event's ``type`` field. - - Args: - frames: Async iterator of pre-formatted SSE strings or Responses events. - - Yields: - SSE frames; on exception, a final ``error`` frame. - """ - try: - async for frame in frames: - if isinstance(frame, str): - yield frame - continue - if isinstance(frame, Mapping): - event_dict = dict(frame) - elif hasattr(frame, "model_dump"): - event_dict = frame.model_dump(exclude_none=True) - elif hasattr(frame, "to_dict"): - event_dict = frame.to_dict() - else: - event_dict = {"type": "message", "data": str(frame)} - event_type = event_dict.get("type", "message") - yield f"event: {event_type}\ndata: {json.dumps(event_dict)}\n\n" - except Exception as e: - log.error("Error during responses streaming: %s: %s", type(e).__name__, e) - error_data = { - "type": "error", - "error": { - "message": repr(e)[:200], - "type": "internal_error", - "code": "internal_chain_error", - }, - } - yield f"event: error\ndata: {json.dumps(error_data)}\n\n" - finally: - await _aclose_stream(frames) diff --git a/switchyard/lib/endpoints/stats_endpoint.py b/switchyard/lib/endpoints/stats_endpoint.py deleted file mode 100644 index 3ffdc2674..000000000 --- a/switchyard/lib/endpoints/stats_endpoint.py +++ /dev/null @@ -1,100 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""FastAPI endpoint module exposing stats over HTTP. - -Serves three paths off the same shared stats source: - -- ``GET /v1/stats`` — native JSON snapshot. -- ``GET /v1/routing/stats`` — alias of ``/v1/stats`` for backwards compat. -- ``GET /metrics`` — Prometheus text-format exposition rendered from the - same snapshot via :func:`switchyard.lib.prometheus_exposition.render_prometheus`. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Any - -from fastapi import APIRouter -from fastapi.responses import Response - -from switchyard.lib.endpoints import outcome_metrics -from switchyard.lib.endpoints.base import Endpoint as NemoSwitchyardEndpoint -from switchyard.lib.endpoints.prometheus_emitter import render as render_extra_metrics -from switchyard.lib.prometheus_exposition import render_prometheus -from switchyard.lib.stats_accumulator import StatsAccumulator - -if TYPE_CHECKING: - from fastapi import FastAPI - - -# Prometheus text exposition format 0.0.4 content-type. -PROMETHEUS_CONTENT_TYPE = "text/plain; version=0.0.4; charset=utf-8" - - -class StatsEndpoint(NemoSwitchyardEndpoint): - """Exposes stats via ``GET /v1/stats``, ``/v1/routing/stats`` alias, - and ``GET /metrics`` (Prometheus exposition). - - Contributed automatically by :class:`StatsResponseProcessor.get_endpoint` - — no manual wiring required. - - The ``/v1/routing/stats`` alias exists so existing consumers of the - historical endpoint path (``benchmark/run_terminal_bench_harbor.sh``, external - dashboards) work against passthrough without any config change. - - ``/metrics`` renders the same underlying snapshot for Prometheus scrapers; - JSON behavior on ``/v1/stats`` is untouched. - """ - - register_once = True - - def __init__(self, stats: StatsAccumulator) -> None: - self._stats = stats - - def register(self, app: FastAPI) -> None: - routes = APIRouter() - stats = self._stats - - async def get_stats() -> dict[str, Any]: - """Snapshot of per-model request / token / latency / cost stats.""" - return await stats.snapshot() - - async def reset_stats() -> dict[str, str]: - """Zero all stats counters.""" - await stats.reset() - return {"status": "reset"} - - async def get_metrics() -> Response: - """Prometheus text-format exposition of the shared stats snapshot. - - Components that own non-request-derived state contribute extra - lines via :mod:`switchyard.lib.endpoints.prometheus_emitter` so a - single ``/metrics`` scrape carries both surfaces. - """ - snapshot = await stats.snapshot() - outcome_block = "\n".join(outcome_metrics.render_lines()) + "\n" - return Response( - content=( - render_prometheus(snapshot) - + outcome_block - + render_extra_metrics() - ), - media_type=PROMETHEUS_CONTENT_TYPE, - ) - - # native path. - routes.get("/v1/stats")(get_stats) - routes.post("/v1/stats/reset")(reset_stats) - # Compatibility alias. - routes.get("/v1/routing/stats")(get_stats) - routes.post("/v1/routing/stats/reset")(reset_stats) - - app.include_router(routes, tags=["Stats"]) - - # Prometheus exposition lives at the conventional ``/metrics`` path, - # untagged from the JSON Stats routes so scraper discovery / OpenAPI - # consumers see them separately. - metrics_router = APIRouter() - metrics_router.get("/metrics")(get_metrics) - app.include_router(metrics_router, tags=["Metrics"]) diff --git a/switchyard/lib/endpoints/upstream_error.py b/switchyard/lib/endpoints/upstream_error.py deleted file mode 100644 index 66de361d2..000000000 --- a/switchyard/lib/endpoints/upstream_error.py +++ /dev/null @@ -1,233 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Endpoint-side helper: convert chain exceptions into upstream-status responses. - -Python LLM backends stash the upstream HTTP status / body into -``ctx.metadata`` before raising the upstream provider's exception. Rust -backends attach typed -``status_code`` and ``body`` attributes to ``SwitchyardUpstreamError`` so -endpoints can preserve provider failures without parsing exception text. -""" - -from __future__ import annotations - -import json -import logging -from typing import TYPE_CHECKING, Literal - -from fastapi.responses import JSONResponse - -from switchyard.lib.endpoints import outcome_metrics -from switchyard.lib.endpoints.error_envelope import ( - error_response, - upstream_error_response, -) -from switchyard.lib.endpoints.route_selection import route_selection_headers -from switchyard.lib.proxy_context import ( - CTX_ERROR_SOURCE, - CTX_UPSTREAM_ATTEMPTS_RECORDED, - CTX_UPSTREAM_HTTP_BODY, - CTX_UPSTREAM_HTTP_STATUS, - CTX_UPSTREAM_MODEL, - ERROR_SOURCE_PROVIDER, - ERROR_SOURCE_SWITCHYARD, -) -from switchyard_rust.core import SwitchyardUpstreamError - -if TYPE_CHECKING: - from switchyard.lib.proxy_context import ProxyContext - -Inbound = Literal["anthropic", "openai", "openai-responses"] - -_log = logging.getLogger(__name__) - - -def record_upstream_attempt_success(ctx: ProxyContext) -> None: - """Record one successful (HTTP 200) ``switchyard_upstream_attempts_total``. - - The endpoint-layer fallback that wires the upstream-attempt counter for - every chain whose backend does not record per-attempt itself: the Rust - native / passthrough / multi backends issue exactly one upstream attempt - per call and have no Python retry loop, so one served client request maps - to one successful attempt observable here. - - No-op when a backend already counted its own attempts - (:data:`CTX_UPSTREAM_ATTEMPTS_RECORDED` set) — its retry fan-out must not - be double-counted here. - """ - if ctx.metadata.get(CTX_UPSTREAM_ATTEMPTS_RECORDED): - return - outcome_metrics.record_upstream_attempt(200) - - -def record_upstream_attempt_failure(ctx: ProxyContext, exc: BaseException) -> None: - """Record one failed ``switchyard_upstream_attempts_total`` when attributable. - - Counts an attempt only for failures attributable to the upstream call — a - Python backend that stashed :data:`CTX_UPSTREAM_HTTP_STATUS`, or a Rust - backend's :class:`SwitchyardUpstreamError` (HTTP status carried verbatim; - a status-less upstream error is a network / pre-status failure recorded as - ``None`` → ``retryable_error``). Internal chain failures (translation, - processor, validation) are not upstream attempts and are skipped. - - No-op when a backend already counted its own attempts - (:data:`CTX_UPSTREAM_ATTEMPTS_RECORDED` set). - """ - if ctx.metadata.get(CTX_UPSTREAM_ATTEMPTS_RECORDED): - return - status = ctx.metadata.get(CTX_UPSTREAM_HTTP_STATUS) - if isinstance(status, int): - outcome_metrics.record_upstream_attempt(status) - return - if isinstance(exc, SwitchyardUpstreamError): - rust_status = getattr(exc, "status_code", None) - outcome_metrics.record_upstream_attempt( - rust_status if isinstance(rust_status, int) else None - ) - - -def upstream_response_from_ctx( - ctx: ProxyContext, - *, - inbound: Inbound = "openai", - exc: BaseException | None = None, -) -> JSONResponse | None: - """Return a structured upstream-status response when one can be recovered. - - Python backends store status/body in ``ctx.metadata``. Rust backends attach - typed upstream status/body attributes to ``SwitchyardUpstreamError``. - ``inbound`` is accepted for endpoint compatibility; HTTP errors use one - Switchyard envelope across all LLM routes so clients see stable fields. - Returns ``None`` when neither source carries an upstream status and the - endpoint should re-raise. - """ - status = ctx.metadata.get(CTX_UPSTREAM_HTTP_STATUS) - if isinstance(status, int): - return upstream_error_response( - status, - ctx.metadata.get(CTX_UPSTREAM_HTTP_BODY), - # A stashed status without an explicit source is an upstream - # passthrough; backends that reuse this channel for their own - # rejections (caller_required 401, translation 400) mark the - # stash ``switchyard`` so the header stays truthful. - error_source=_ctx_error_source(ctx, default=ERROR_SOURCE_PROVIDER), - upstream_model=_ctx_upstream_model(ctx), - ) - return upstream_response_from_error(exc, inbound=inbound) - - -def _ctx_error_source(ctx: ProxyContext, *, default: str) -> str: - """Failure origin stamped by the backend, or ``default`` for the path.""" - source = ctx.metadata.get(CTX_ERROR_SOURCE) - return source if isinstance(source, str) and source else default - - -def _ctx_upstream_model(ctx: ProxyContext) -> str | None: - """Upstream model recorded at failure time, when a selection happened.""" - model = ctx.metadata.get(CTX_UPSTREAM_MODEL) - return model if isinstance(model, str) and model else None - - -def upstream_response_from_error( - exc: BaseException | None, - *, - inbound: Inbound = "openai", -) -> JSONResponse | None: - """Return a normalized response for typed Rust upstream HTTP failures.""" - if not isinstance(exc, SwitchyardUpstreamError): - return None - status = getattr(exc, "status_code", None) - raw_body = getattr(exc, "body", None) - if not isinstance(status, int) or not isinstance(raw_body, str): - return None - body = _parse_body(raw_body) - return upstream_error_response(status, body) - - -def _parse_body(raw: str) -> object: - text = raw.strip() - try: - return json.loads(text) - except json.JSONDecodeError: - return text - - -def internal_chain_error_response( - exc: BaseException, - inbound: Inbound, - *, - error_source: str = ERROR_SOURCE_SWITCHYARD, - upstream_model: str | None = None, -) -> JSONResponse: - """Translate an unexpected chain failure into the client error envelope. - - Used when an exception escapes dispatch or response-processing that is not - a known upstream HTTP error (no status stashed in ctx) and not a - ``SwitchyardUpstreamError``. LLM clients expect a JSON error object rather - than FastAPI's plain-text 500; callers must log the traceback before calling - this helper so the full context is preserved server-side. ``inbound`` is - retained in the signature because endpoint callers already pass it, but the - HTTP envelope is intentionally shared across inbound formats. - - ``error_source`` defaults to ``switchyard`` (an unexpected internal - failure) but a backend that failed on a status-less upstream fault (e.g. - a network error after retries) marks ``ctx`` so this 500 is labeled - ``provider`` instead. - """ - message = repr(exc)[:200] - return error_response( - 500, - message, - error_type="internal_error", - code="internal_chain_error", - error_source=error_source, - upstream_model=upstream_model, - ) - - -def handle_chain_exception( - exc: BaseException, - ctx: ProxyContext, - *, - inbound: Inbound, - log_msg: str, -) -> JSONResponse: - """Handle an unexpected chain exception: check for upstream status, log, and return envelope.""" - record_upstream_attempt_failure(ctx, exc) - upstream = upstream_response_from_ctx(ctx, inbound=inbound, exc=exc) - if upstream is not None: - response = upstream - else: - _log.error(log_msg, exc_info=exc) - response = internal_chain_error_response( - exc, - inbound=inbound, - error_source=_ctx_error_source(ctx, default=ERROR_SOURCE_SWITCHYARD), - upstream_model=_ctx_upstream_model(ctx), - ) - # A selection on ctx means an upstream call already SUCCEEDED (and was - # billed, with the spend-logs header stamped) before this failure — e.g. - # response-side translation rejected the 200 payload. Surface the - # selection headers on the error response too, so the front proxy can - # still join its (failed) parent spend-log row to the billed provider row. - for header_name, value in route_selection_headers(ctx).items(): - response.headers[header_name] = value - return response - - -def context_exhausted_response(exc: BaseException, inbound: Inbound) -> JSONResponse: - """Translate :class:`SwitchyardContextPoolExhaustedError` into a 400. - - Raised by the chain executor when every routing target has been evicted - after consecutive context-window overflows; FastAPI endpoints catch it - and call this helper to produce the shared Switchyard HTTP error envelope. - """ - # Chain-executor rejection, not an upstream failure — ``error_response`` - # stamps the ``switchyard`` source header by default. - return error_response( - 400, - str(exc), - error_type="invalid_request_error", - code="context_length_exceeded", - ) diff --git a/switchyard/lib/endpoints/upstream_error_log.py b/switchyard/lib/endpoints/upstream_error_log.py deleted file mode 100644 index 70977506f..000000000 --- a/switchyard/lib/endpoints/upstream_error_log.py +++ /dev/null @@ -1,90 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Structured per-attempt upstream-failure log for Loki ingestion. - -The aggregate ``switchyard_upstream_attempts_total`` counter on ``/metrics`` -answers "how many of each error code" but, by Prometheus' data model, holds -no per-event timestamps. This module is its per-event complement: it emits -one JSON line per *failed* upstream attempt on a dedicated logger, carrying -the exact event timestamp so a Loki/Grafana pipeline can audit, replay, or -plot individual failures. - -The line *is* a JSON object (not a human sentence with structured ``extra``) -on purpose: Switchyard configures plain-text logging via ``logging.basicConfig`` -with no JSON formatter, so embedding the document in the message is what makes -``| json`` work in a Loki query with zero deployment-side formatter config. -The dedicated ``switchyard.upstream_errors`` logger still propagates to the -root handler, so the line also shows on the console. - -The ``code`` and ``outcome`` fields are computed with the same helpers as the -metric labels (:func:`~switchyard.lib.endpoints.outcome_metrics.code_label`, -:func:`~switchyard.lib.endpoints.outcome_metrics.classify`) so the event log -joins cleanly to ``switchyard_upstream_attempts_total``. -""" - -from __future__ import annotations - -import json -import logging -from datetime import UTC, datetime - -from switchyard.lib.endpoints.outcome_metrics import classify, code_label - -#: Dedicated logger so operators can route or level upstream-failure events -#: independently of the (noisier) backend logger. Propagates to root. -log = logging.getLogger("switchyard.upstream_errors") - -#: Structured-log event name — the value a Loki query filters on -#: (``| json | event="upstream_attempt_failed"``). -EVENT_NAME = "upstream_attempt_failed" - -#: Upstream error bodies can be large; cap the logged message so a single -#: pathological error cannot blow up a log line / Loki entry. -_MAX_ERROR_CHARS = 500 - - -def log_upstream_attempt_failure( - *, - model: str, - attempt: int, - status_code: int | None, - error: BaseException, - upstream_model: str | None = None, -) -> None: - """Emit one structured JSON record for a single failed upstream attempt. - - ``status_code`` is the raw upstream HTTP status, or ``None`` for a - non-HTTP failure (network error, pre-status timeout) — recorded as - ``status_code: null`` with ``code="none"``. ``attempt`` is 1-based. - ``upstream_model`` is the model actually sent upstream (``body["model"]``) - when the caller knows it; ``model`` remains the internal route/endpoint id. - - ``code`` and ``outcome`` mirror the labels on - ``switchyard_upstream_attempts_total`` so the event log is joinable to - the aggregate counter. ``error_source`` is always ``provider`` — this - event exists only for failures of actual upstream attempts. The record - is logged at WARNING. - """ - record = { - "event": EVENT_NAME, - "timestamp": datetime.now(UTC).isoformat(), - "model": model, - "upstream_model": upstream_model, - "attempt": attempt, - "status_code": status_code, - "code": code_label(status_code), - # None (non-HTTP failure) is a retryable_error, matching how - # record_upstream_attempt buckets it. - "outcome": "retryable_error" if status_code is None else classify(status_code), - # Attempt failures are upstream-side by definition; the constant field - # keeps the event joinable to the response-header/span vocabulary. - "error_source": "provider", - "error_type": type(error).__name__, - "error": str(error)[:_MAX_ERROR_CHARS], - } - # Compact separators keep the line small; the message is valid JSON. - log.warning(json.dumps(record, separators=(",", ":"))) - - -__all__ = ["EVENT_NAME", "log_upstream_attempt_failure"] diff --git a/switchyard/lib/llm_client.py b/switchyard/lib/llm_client.py deleted file mode 100644 index c01a4e72f..000000000 --- a/switchyard/lib/llm_client.py +++ /dev/null @@ -1,190 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Minimal LLM client wrapper for backends. - -This provides a thin wrapper around the OpenAI SDK to support -OpenAI-compatible backends in the chain. -""" - -from __future__ import annotations - -from typing import Any - -from openai import AsyncOpenAI - -from switchyard.telemetry import get_telemetry_headers - - -class OpenAILLMClient: - """Client that wraps the official OpenAI Python SDK. - - Works with any OpenAI-compatible API (OpenAI, NVIDIA NIM, Azure, - vLLM, etc.) by accepting a custom ``base_url``. - - Used by :class:`~switchyard.lib.backends.openai_llm_backend.OpenAiNativeBackend` - and other OpenAI-compatible backends. - """ - - async_client: AsyncOpenAI - - def __init__( - self, - *, - api_key: str | None = None, - base_url: str | None = None, - timeout: float | None = None, - max_retries: int | None = None, - ) -> None: - """Initialize the async OpenAI client. - - Args: - api_key: API key for authentication. When omitted or empty, - an inert placeholder is used so the SDK can construct; - callers must then supply a real key per-request via - ``acompletion(api_key=...)`` (BYO-key mode). The - placeholder never reaches a real upstream — without a - caller key the SDK call fails with the upstream's 401. - base_url: Custom base URL for OpenAI-compatible APIs (e.g., - Azure, vLLM, NVIDIA NIM). Defaults to OpenAI's standard URL. - timeout: Request timeout in seconds. None means no timeout. - max_retries: Override the OpenAI SDK's default 2-retry budget. - ``None`` (default) keeps the SDK default. Set ``0`` for the - classifier path so a slow-upstream ``ReadTimeout`` falls - through to our own ``fail_open`` fallback at the configured - timeout rather than compounding via SDK exponential backoff. - """ - client_kwargs: dict[str, Any] = {} - if api_key: - client_kwargs["api_key"] = api_key - else: - # BYO-key deployments construct the client with no - # server-side credential and supply the caller's key per - # request via ``acompletion(api_key=...)``. The SDK refuses - # to construct without *some* key, so we inject an inert - # placeholder that is overridden on every real call. If a - # caller forgets to send a key, the upstream sees this - # placeholder and returns 401 — no real secret can leak. - client_kwargs["api_key"] = "switchyard-byo-key-required" - if base_url: - client_kwargs["base_url"] = base_url - if timeout is not None: - client_kwargs["timeout"] = timeout - if max_retries is not None: - client_kwargs["max_retries"] = max_retries - client_kwargs["default_headers"] = get_telemetry_headers() - - # Only the async client is ever used (backends call ``acompletion``). - # A sync ``OpenAI`` client would allocate a second, never-used httpx - # connection pool (1000 connections by default) per instance, so it is - # intentionally not constructed. - self.async_client = AsyncOpenAI(**client_kwargs) - - def _client_for_api_key(self, api_key: str | None) -> AsyncOpenAI: - if api_key and api_key.strip(): - return self.async_client.with_options(api_key=api_key) - return self.async_client - - async def acompletion( - self, - *, - api_key: str | None = None, - **kwargs: Any, - ) -> Any: - """Async wrapper for chat completions. - - When a non-blank ``api_key`` is supplied, the call uses that credential - via the SDK's ``with_options`` override instead of the client's - construction-time key. Used by backends that forward a per-request - caller credential (BYO-key multi-tenant deployments). - - When ``api_key`` is ``None`` or blank (no caller key, or a - whitespace-only header), the override is skipped so the call falls - back to the construction-time key — the per-endpoint ``api_key`` an - operator configured. A blank value must not override a real configured - key with nothing, which would unauthenticate the upstream call (a 401 - even though a valid key was configured). - """ - return await self._client_for_api_key(api_key).chat.completions.create(**kwargs) - - async def aresponses( - self, - *, - api_key: str | None = None, - **kwargs: Any, - ) -> Any: - """Async wrapper for the OpenAI Responses API. - - When a non-blank ``api_key`` is supplied, the call uses that - per-request credential via the SDK's ``with_options`` override. When - ``api_key`` is ``None`` or blank, no override is applied and the - construction-time key configured on the client is used. SDK validation - and upstream errors are intentionally propagated unchanged. - - Non-streaming calls return the upstream's **exact JSON body** (a - ``dict``) rather than the SDK's typed ``Response`` model: round-tripping - through the typed model re-normalizes the payload and its - ``exclude_none`` serialization drops explicit-null fields, eroding - schema fidelity for Responses passthrough. - - Streaming calls return a :class:`RawSSEFrameStream` yielding the - upstream's SSE frames as **verbatim strings** (modulo CRLF → LF line - normalization) for the same reason — the SDK's typed event stream - drops provider extras and explicit nulls per event. The HTTP request - is sent (and error statuses raise) *before* this method returns, so - the caller's retry/failover contract is unchanged. - """ - client = self._client_for_api_key(api_key) - if kwargs.get("stream"): - cm = client.responses.with_streaming_response.create(**kwargs) - # Enter eagerly: the request goes out and non-2xx statuses raise - # ``APIStatusError`` here, not at first iteration — after first - # iteration the endpoint has already committed an HTTP 200. - response = await cm.__aenter__() - return RawSSEFrameStream(cm, response.http_response) - raw = await client.responses.with_raw_response.create(**kwargs) - return raw.http_response.json() - - -class RawSSEFrameStream: - """Async iterator over an SSE response's frames as verbatim strings. - - Each item is one complete frame (all lines up to and including the blank - separator, e.g. ``"event: x\\ndata: {...}\\n\\n"``), byte-equivalent to the - upstream modulo CRLF → LF normalization. Comment/keep-alive frames pass - through unchanged. ``aclose`` releases the underlying HTTP response and is - safe to call at any point, including before the first ``__anext__``. - """ - - def __init__(self, cm: Any, http_response: Any) -> None: - self._cm = cm - self._lines = http_response.aiter_lines() - self._closed = False - - def __aiter__(self) -> RawSSEFrameStream: - return self - - async def __anext__(self) -> str: - buffer: list[str] = [] - try: - async for line in self._lines: - if line == "": - if buffer: - return "\n".join(buffer) + "\n\n" - continue - buffer.append(line) - except BaseException: - await self.aclose() - raise - await self.aclose() - if buffer: - # Upstream closed without a trailing blank line; emit the tail as - # a well-formed frame rather than dropping it. - return "\n".join(buffer) + "\n\n" - raise StopAsyncIteration - - async def aclose(self) -> None: - if self._closed: - return - self._closed = True - await self._cm.__aexit__(None, None, None) diff --git a/switchyard/lib/model_listing.py b/switchyard/lib/model_listing.py deleted file mode 100644 index 83a4b8070..000000000 --- a/switchyard/lib/model_listing.py +++ /dev/null @@ -1,131 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""OpenAI-compatible model listing payload helpers.""" - -from collections.abc import Mapping, Sequence -from typing import Any - -SUPPORTED_INBOUND_FORMATS: tuple[str, ...] = ( - "openai-chat-completions", - "openai-responses", - "anthropic-messages", -) - -DEFAULT_CONTEXT_WINDOW = 128_000 - -_CONTEXT_WINDOW_BY_MODEL_FRAGMENT: tuple[tuple[str, int], ...] = ( - ("nemotron-3-super", 1_000_000), - ("nemotron-3-nano", 262_000), - ("deepseek-v4", 1_000_000), - ("claude", 200_000), - ("kimi-k2", 256_000), -) - - -def _default_capabilities() -> dict[str, Any]: - return { - "streaming": True, - "tool_calling": True, - "context_window": DEFAULT_CONTEXT_WINDOW, - "supported_inbound_formats": list(SUPPORTED_INBOUND_FORMATS), - } - - -def model_capabilities( - model_id: str, - *, - context_window: int | None = None, - tool_calling: bool = True, -) -> dict[str, Any]: - """Infer capability metadata for a Switchyard-advertised model id.""" - return { - "tool_calling": tool_calling, - "context_window": ( - context_window if context_window is not None else inferred_context_window(model_id) - ), - } - - -def combined_model_capabilities(model_ids: Sequence[str]) -> dict[str, Any]: - """Return conservative capabilities for a route spanning multiple models.""" - windows = [inferred_context_window(model_id) for model_id in model_ids] - return { - "tool_calling": True, - "context_window": min(windows) if windows else DEFAULT_CONTEXT_WINDOW, - } - - -def inferred_context_window(model_id: str) -> int: - """Infer a context window from known model-family fragments.""" - normalized = model_id.lower() - for fragment, context_window in _CONTEXT_WINDOW_BY_MODEL_FRAGMENT: - if fragment in normalized: - return context_window - return DEFAULT_CONTEXT_WINDOW - - -def model_entry( - model_id: str, - display_name: str | None = None, - metadata: Mapping[str, Any] | None = None, -) -> dict[str, Any]: - """Build one OpenAI-compatible model object with Switchyard metadata.""" - extra = dict(metadata or {}) - display_name = display_name or str(extra.pop("display_name", model_id)) - extra.pop("id", None) - extra.pop("object", None) - extra.pop("type", None) - - capabilities = _default_capabilities() - raw_capabilities = extra.pop("capabilities", None) - if isinstance(raw_capabilities, Mapping): - capabilities.update(raw_capabilities) - - entry: dict[str, Any] = { - "id": model_id, - "object": "model", - "type": "model", - "created": extra.pop("created", 0), - "owned_by": extra.pop("owned_by", "switchyard"), - "display_name": display_name, - "capabilities": capabilities, - } - entry.update(extra) - return entry - - -def model_list_payload( - entries: Sequence[Mapping[str, Any]], - default_model: str | None = None, - warnings: Sequence[str] = (), -) -> dict[str, Any]: - """Build the ``GET /v1/models`` response envelope.""" - data = [dict(entry) for entry in entries] - model_ids = [str(entry["id"]) for entry in data if "id" in entry] - advertised_default = model_ids[0] if model_ids else None - if default_model in model_ids: - advertised_default = default_model - payload: dict[str, Any] = { - "object": "list", - "data": data, - "first_id": model_ids[0] if model_ids else None, - "last_id": model_ids[-1] if model_ids else None, - "has_more": False, - "default_model": advertised_default, - "model_pool": model_ids, - } - if warnings: - payload["warnings"] = list(warnings) - return payload - - -__all__ = [ - "DEFAULT_CONTEXT_WINDOW", - "SUPPORTED_INBOUND_FORMATS", - "combined_model_capabilities", - "inferred_context_window", - "model_capabilities", - "model_entry", - "model_list_payload", -] diff --git a/switchyard/lib/processors/__init__.py b/switchyard/lib/processors/__init__.py deleted file mode 100644 index c640aa02a..000000000 --- a/switchyard/lib/processors/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Concrete request-side and response-side processor implementations.""" - -from switchyard.lib.processors.model_rewrite_request_processor import ( - ModelRewriteRequestProcessor, -) -from switchyard.lib.processors.stats_request_processor import ( - StatsRequestProcessor, -) -from switchyard.lib.processors.stats_response_processor_accumulator import ( - StatsResponseProcessor, -) - -__all__ = [ - "ModelRewriteRequestProcessor", - "StatsRequestProcessor", - "StatsResponseProcessor", -] diff --git a/switchyard/lib/processors/format_translate.py b/switchyard/lib/processors/format_translate.py deleted file mode 100644 index c4c2d5b23..000000000 --- a/switchyard/lib/processors/format_translate.py +++ /dev/null @@ -1,311 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Router-agnostic format-translation processors for the IGW path. - -When Switchyard owns the LLM call (standalone server), inbound→target -format translation happens *inside* the LLM backend (e.g. -:class:`OpenAiNativeBackend.call` runs -:meth:`TranslationEngine.request_to_any_of` at the top), and -outbound translation happens in :class:`TranslationEngine`. -The host's chain has dedicated slots for both. - -When the host (NMP IGW) owns the LLM call, neither slot is available -— IGW supplies its own backend and there's no -:class:`TranslationEngine` slot. Format translation has to live in -processors instead. The three processors here are the IGW equivalent -of the standalone path, and reusable by *any* router (random -routing today, others later) that needs cross-format dispatch: - -* :class:`StampOriginalFormatProcessor` — runs at the head of the - chain and captures the inbound :class:`ChatRequestType` into - ``ctx.metadata[CTX_ORIGINAL_FORMAT]`` so the response processor can - translate back later. -* :class:`FormatTranslateRequestProcessor` — runs after a router - picks a tier and reads ``ctx.metadata[CTX_TARGET_FORMAT]``. - No-ops when the target is unset or matches - ``request.request_type``; otherwise delegates to - :meth:`TranslationEngine.request_to_any_of`. -* :class:`FormatTranslateResponseProcessor` — runs after the host - backend returns. Reads ``ctx.metadata[CTX_ORIGINAL_FORMAT]`` and - converts the response back to the client's inbound format via - :class:`TranslationEngine`. - -Composability: any router that wants cross-format support stamps a -:class:`ChatRequestType` into ``CTX_TARGET_FORMAT``; these processors -handle the actual conversion. Router and converter are fully -decoupled — neither imports the other. -""" - -from __future__ import annotations - -import logging -from collections.abc import Mapping -from copy import deepcopy -from typing import TYPE_CHECKING, cast - -from pydantic import BaseModel, ConfigDict - -from switchyard.lib.backends.llm_target import BackendFormat -from switchyard.lib.proxy_context import ( - CTX_ORIGINAL_FORMAT, - CTX_ORIGINAL_REQUEST, - CTX_PROXY_ACTUAL_MODEL, - CTX_TARGET_FORMAT, -) -from switchyard_rust.core import ( - ChatRequest, - ChatRequestType, - ChatResponse, - request_type_matches, - request_type_value, - request_with_type, - response_is_streaming, - response_matches_request_type, -) -from switchyard_rust.translation import TranslationEngine - -if TYPE_CHECKING: - from anthropic.types.message_create_params import MessageCreateParamsBase - from openai.types.chat.completion_create_params import CompletionCreateParamsBase - from openai.types.responses.response_create_params import ResponseCreateParamsBase - - from switchyard.lib.proxy_context import ProxyContext - -log = logging.getLogger(__name__) - - -class TranslateConfig(BaseModel): - """Model-to-format map used by IGW-owned translation processors.""" - - model_config = ConfigDict(frozen=True) - - models: list[dict[str, str]] = [] - - -class ModelFormatLookupProcessor: - """Look up the selected model's target format from the translate config. - - After a router picks a tier and stamps its model name into - ``ctx.selected_model``, this processor looks up the model in the - translate config's models list and stamps the corresponding backend - format into ``ctx.metadata[CTX_TARGET_FORMAT]``. The legacy - ``ctx.metadata[CTX_PROXY_ACTUAL_MODEL]`` key remains a fallback for - Python-only routers that have not migrated yet. - - ``BackendFormat.AUTO`` is resolved locally because the IGW path has - no Switchyard-owned backend slot where backend construction can probe - endpoint capabilities. The resolver keeps Anthropic-native inbound - traffic native, and otherwise normalizes to OpenAI Chat as the broad - fallback wire format. - - No-op when the model is not found in the config (passthrough). - """ - - def __init__(self, config: TranslateConfig) -> None: - self._config = config - # Build a lookup dict: model name → backend_format - self._model_to_format: dict[str, BackendFormat] = {} - for entry in config.models: - model_name = entry.get("model") - format_str = entry.get("backend_format") - if model_name and format_str: - try: - self._model_to_format[model_name] = BackendFormat(format_str) - except ValueError: - log.warning( - "Unknown backend_format in translate config: %s", format_str, - ) - - async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: - selected_model = ctx.selected_model or ctx.metadata.get(CTX_PROXY_ACTUAL_MODEL) - if not isinstance(selected_model, str) or not selected_model: - return request - target_format = self._model_to_format.get(selected_model) - if target_format is not None: - target_request_type = _target_request_type_for_backend_format( - target_format, request, - ) - ctx.metadata[CTX_TARGET_FORMAT] = target_request_type - log.info( - "ModelFormatLookupProcessor: model=%s format=%s " - "target_request_type=%s", - selected_model, - target_format.value, - target_request_type.value, - ) - return request - - -class StampOriginalFormatProcessor: - """Capture the inbound request type into ``ctx.metadata``. - - Place at the head of any IGW chain that may rewrite the request - format before the host backend sees it. The matching response - processor reads ``CTX_ORIGINAL_FORMAT`` to know what format to - translate the response back to. - """ - - async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: - ctx.metadata[CTX_ORIGINAL_FORMAT] = request.request_type - if CTX_ORIGINAL_REQUEST not in ctx.metadata: - body = getattr(request, "body", None) - if isinstance(body, Mapping): - ctx.metadata[CTX_ORIGINAL_REQUEST] = deepcopy(dict(body)) - return request - - -class FormatTranslateRequestProcessor: - """Translate the request to the target wire format chosen by a router. - - Reads ``ctx.metadata[CTX_TARGET_FORMAT]`` (a - :class:`ChatRequestType`) and routes through - :meth:`TranslationEngine.request_to_any_of` when the target - differs from ``request.request_type``. Pure passthrough when: - - * No router stamped a target (``CTX_TARGET_FORMAT`` absent). - * Target matches the inbound type (already in the right format). - - Router-agnostic — no notion of *why* a target was chosen, only - what to do with it. Reusable by every routing factory that needs - cross-format dispatch. - """ - - async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: - target = ctx.metadata.get(CTX_TARGET_FORMAT) - if target is None: - return request - if not _is_request_type(target): - raise TypeError( - f"ctx.metadata[{CTX_TARGET_FORMAT!r}] must be a ChatRequestType, " - f"got {type(target).__name__}", - ) - if request_type_matches(request, target): - return request - return TranslationEngine().request_to_any_of(request, [target]) - - -class FormatTranslateResponseProcessor: - """Translate the response back to the inbound format the client expects. - - Reads ``ctx.metadata[CTX_ORIGINAL_FORMAT]`` (stamped by - :class:`StampOriginalFormatProcessor` at the head of the chain) - and converts the response via - :class:`TranslationEngine`. No-ops when the response - is already in the original format or when ``CTX_ORIGINAL_FORMAT`` - is absent. - - Streaming responses take the response model from the routing context so - the translated stream names the model that actually served the call, - while still translating lazily. - """ - - async def process(self, ctx: ProxyContext, response: ChatResponse) -> ChatResponse: - original = ctx.metadata.get(CTX_ORIGINAL_FORMAT) - if original is None: - return response - if not _is_request_type(original): - raise TypeError( - f"ctx.metadata[{CTX_ORIGINAL_FORMAT!r}] must be a ChatRequestType, " - f"got {type(original).__name__}", - ) - if response_is_streaming(response): - return _translate_streaming_response(ctx, response, original) - if _matches_format(response, original): - return response - if request_type_value(original) == request_type_value(ChatRequestType.OPENAI_CHAT): - return TranslationEngine().response_to(ChatRequestType.OPENAI_CHAT, response) - if request_type_value(original) == request_type_value(ChatRequestType.ANTHROPIC): - return TranslationEngine().response_to(ChatRequestType.ANTHROPIC, response) - if request_type_value(original) == request_type_value(ChatRequestType.OPENAI_RESPONSES): - return TranslationEngine().response_to( - ChatRequestType.OPENAI_RESPONSES, - response, - served_model=_served_model(ctx), - ) - raise NotImplementedError( - f"Unsupported original format for response translation: {original!r}", - ) - - -def _translate_streaming_response( - ctx: ProxyContext, - response: ChatResponse, - original: ChatRequestType | str, -) -> ChatResponse: - if _matches_streaming_format(response, original): - return response - return TranslationEngine().response_to( - original, - response, - served_model=_served_model(ctx), - ) - - -def _request_for_original_format( - original: ChatRequestType | str, - body: dict[str, object], -) -> ChatRequest: - if request_type_value(original) == request_type_value(ChatRequestType.OPENAI_CHAT): - return request_with_type(original, cast("CompletionCreateParamsBase", body)) - if request_type_value(original) == request_type_value(ChatRequestType.ANTHROPIC): - return request_with_type(original, cast("MessageCreateParamsBase", body)) - if request_type_value(original) == request_type_value(ChatRequestType.OPENAI_RESPONSES): - return request_with_type(original, cast("ResponseCreateParamsBase", body)) - raise NotImplementedError(f"Unsupported original format: {original!r}") - - -def _served_model(ctx: ProxyContext) -> str | None: - """Return the model the backend actually called, or ``None`` when unknown.""" - model = ctx.selected_model or ctx.metadata.get(CTX_PROXY_ACTUAL_MODEL) - return model if isinstance(model, str) and model else None - - -def _matches_format(response: ChatResponse, target: ChatRequestType | str) -> bool: - """Return ``True`` when *response* is already in *target*'s wire format.""" - return bool(response_matches_request_type(response, target)) and not bool( - response_is_streaming(response) - ) - - -def _matches_streaming_format(response: ChatResponse, target: ChatRequestType | str) -> bool: - return bool(response_matches_request_type(response, target)) and bool( - response_is_streaming(response) - ) - - -def _is_request_type(value: object) -> bool: - try: - request_type_value(cast("ChatRequestType | str", value)) - except (AttributeError, TypeError, ValueError): - return False - return True - - -def _target_request_type_for_backend_format( - backend_format: BackendFormat, - request: ChatRequest, -) -> ChatRequestType: - """Resolve a configured backend wire format to a request format. - - ``BackendFormat.AUTO`` on the processor-only IGW path cannot do the - backend-owned capability probe used by backend factories. It instead - chooses the least surprising concrete request shape: - - * Anthropic inbound remains Anthropic, preserving native Claude - fields instead of translating them away. - * OpenAI Chat inbound remains OpenAI Chat. - * Responses-capable targets preserve OpenAI Responses instead of - normalizing through Chat. - """ - if backend_format == BackendFormat.OPENAI: - return ChatRequestType.OPENAI_CHAT - if backend_format == BackendFormat.RESPONSES: - return ChatRequestType.OPENAI_RESPONSES - if backend_format == BackendFormat.ANTHROPIC: - return ChatRequestType.ANTHROPIC - if backend_format == BackendFormat.AUTO: - if request_type_matches(request, ChatRequestType.ANTHROPIC): - return ChatRequestType.ANTHROPIC - return ChatRequestType.OPENAI_CHAT - raise ValueError(f"Unsupported backend_format: {backend_format!r}") diff --git a/switchyard/lib/processors/model_rewrite_request_processor.py b/switchyard/lib/processors/model_rewrite_request_processor.py deleted file mode 100644 index f218fc3cb..000000000 --- a/switchyard/lib/processors/model_rewrite_request_processor.py +++ /dev/null @@ -1,26 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Request processor that pins every inbound request to one backend model.""" - -from __future__ import annotations - -from switchyard.lib.proxy_context import ProxyContext -from switchyard_rust.core import ChatRequest - - -class ModelRewriteRequestProcessor: - """Force-rewrite ``request.body["model"]`` to a fixed value. - - All request subclasses expose a top-level ``model`` key in their provider - body. Launchers use this as a safety net so the child process can display a - model while Switchyard remains authoritative about the upstream route. - """ - - def __init__(self, model: str) -> None: - self._model = model - - async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: - del ctx - request.set_model(self._model) - return request diff --git a/switchyard/lib/processors/rl_logging_request_processor.py b/switchyard/lib/processors/rl_logging_request_processor.py deleted file mode 100644 index c02af3739..000000000 --- a/switchyard/lib/processors/rl_logging_request_processor.py +++ /dev/null @@ -1,43 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Request-side processor that snapshots the inbound request for RL trace logging.""" - -from __future__ import annotations - -from copy import deepcopy - -from switchyard.lib.proxy_context import ProxyContext -from switchyard_rust.core import ChatRequest, ChatRequestType -from switchyard_rust.translation import TranslationEngine - -#: Context metadata key holding the OpenAI-Chat-shaped snapshot of the inbound -#: request body. Written by :class:`RlLoggingRequestProcessor`; read by -#: :class:`~switchyard.lib.processors.rl_logging_response_processor.RlLoggingResponseProcessor` -#: to reconstruct the logged conversation. -CTX_RL_LOGGING_REQUEST = "_rl_logging_request" - - -class RlLoggingRequestProcessor: - """Snapshot the inbound request as an OpenAI-Chat body for RL trace logging. - - The response-side logger only receives ``(ctx, response)``, so the request - has to be captured on the request side. We store the *translated* body (a - plain dict) rather than the request wrapper so the snapshot is both - format-normalized (every inbound format becomes OpenAI Chat) and immune to - any in-place mutation later processors might perform. - """ - - def __init__(self) -> None: - self._translation = TranslationEngine() - - async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: - """Snapshot the OpenAI-Chat-translated request body onto ``ctx``. - - Writes a deep copy under :data:`CTX_RL_LOGGING_REQUEST` (so the snapshot - stays stable even if later processors mutate the request) and returns - the request unchanged. - """ - openai_request = self._translation.request_to(ChatRequestType.OPENAI_CHAT, request) - ctx.metadata[CTX_RL_LOGGING_REQUEST] = deepcopy(dict(openai_request.body)) - return request diff --git a/switchyard/lib/processors/rl_logging_response_processor.py b/switchyard/lib/processors/rl_logging_response_processor.py deleted file mode 100644 index 529a8c1c2..000000000 --- a/switchyard/lib/processors/rl_logging_response_processor.py +++ /dev/null @@ -1,187 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Response-side processor that writes per-turn RL training traces to local JSON files.""" - -from __future__ import annotations - -import json -import logging -import uuid as uuid_lib -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -from switchyard.lib.chat_response.streaming_response_accumulator import ( - attach_final_response_callback, -) -from switchyard.lib.processors.rl_logging_request_processor import ( - CTX_RL_LOGGING_REQUEST, - RlLoggingRequestProcessor, -) -from switchyard.lib.proxy_context import CTX_PROXY_ACTUAL_MODEL, ProxyContext -from switchyard_rust.core import ( - ChatRequestType, - ChatResponse, - ChatResponseType, - response_type_matches, -) -from switchyard_rust.translation import TranslationEngine - -logger = logging.getLogger(__name__) - -JsonObject = dict[str, Any] - - -class RlLoggingResponseProcessor: - """Write one ``message_history`` JSON trace per completed turn to ``log_dir``. - - Restores the pre-1.0 ``--enable-rl-logging`` local trace format: each - request/response pair is written to its own file as - ``{uuid, messages, tools, tool_choice, token_count, is_valid}``. - - Streaming responses are captured via - :func:`attach_final_response_callback`, which accumulates the native stream - and fires once it drains; non-streaming responses are logged inline. The - response is always returned unchanged — this processor only observes. - """ - - def __init__(self, log_dir: Path | str) -> None: - self._log_dir = Path(log_dir) - self._log_dir.mkdir(parents=True, exist_ok=True) - self._translation = TranslationEngine() - - async def process(self, ctx: ProxyContext, response: ChatResponse) -> ChatResponse: - """Log one ``message_history`` trace for the completed turn; return the response. - - Streaming responses are captured when the stream drains; non-streaming - responses log inline. Write failures are swallowed (logged, never - raised) so trace logging can never break the proxied response. - """ - served_model: str = ctx.selected_model or ctx.metadata.get( - CTX_PROXY_ACTUAL_MODEL, "unknown", - ) - - async def _emit(final: ChatResponse) -> None: - self._write_trace(ctx, final) - - # Streaming responses log on stream completion; everything else is - # already complete and logs inline. - attached = attach_final_response_callback( - response, served_model=served_model, callback=_emit, - ) - if not attached: - await _emit(response) - return response - - def _write_trace(self, ctx: ProxyContext, response: ChatResponse) -> None: - request = ctx.metadata.get(CTX_RL_LOGGING_REQUEST) - if not isinstance(request, dict): - return - entry = self._build_entry(request, response) - if entry is None: - return - try: - self._write_entry(entry) - except OSError as exc: - logger.warning("RL logging: failed to write trace to %s: %s", self._log_dir, exc) - - def _build_entry(self, request: JsonObject, response: ChatResponse) -> JsonObject | None: - translated = self._translation.response_to(ChatRequestType.OPENAI_CHAT, response) - if not response_type_matches(translated, ChatResponseType.OPENAI_COMPLETION): - return None - body = dict(translated.body) - choices = body.get("choices") - if not isinstance(choices, list) or not choices or not isinstance(choices[0], dict): - return None - message = choices[0].get("message") - if not isinstance(message, dict): - return None - - messages = [dict(m) for m in request.get("messages", []) if isinstance(m, dict)] - assistant: JsonObject = {"role": "assistant"} - content = message.get("content") - if content is not None: - assistant["content"] = content - tool_calls = message.get("tool_calls") - if tool_calls: - assistant["tool_calls"] = tool_calls - messages.append(assistant) - - usage = body.get("usage") - usage = usage if isinstance(usage, dict) else {} - return { - "uuid": str(uuid_lib.uuid4()), - "messages": messages, - "tools": _format_tools(request.get("tools", [])), - "tool_choice": _format_tool_choice(request.get("tool_choice")), - "token_count": { - "prompt_tokens": usage.get("prompt_tokens", 0), - "completion_tokens": usage.get("completion_tokens", 0), - "total_tokens": usage.get("total_tokens", 0), - }, - "is_valid": True, - } - - def _write_entry(self, entry: JsonObject) -> None: - path = self._log_dir / _trace_filename() - with open(path, "w") as handle: - json.dump(entry, handle, indent=2) - - -def build_rl_logging_processors( - rl_log_dir: Path | None, -) -> tuple[list[Any], list[Any]]: - """Request/response processor lists for local RL trace logging. - - Returns ``([], [])`` when ``rl_log_dir`` is ``None`` (logging disabled), or - the paired snapshot + writer processors otherwise. Shared by the ``launch`` - and ``serve`` wiring. - """ - if rl_log_dir is None: - return [], [] - return [RlLoggingRequestProcessor()], [RlLoggingResponseProcessor(rl_log_dir)] - - -def _trace_filename() -> str: - """File-safe ``{timestamp}_trace_{trace_id}_{suffix}.json`` (one file per turn).""" - timestamp = datetime.now(UTC).strftime("%Y-%m-%dT%H-%M-%S-%f")[:-3] + "Z" - trace_id = uuid_lib.uuid4().hex[:8] - suffix_id = uuid_lib.uuid4().hex[:8] - return f"{timestamp}_trace_{trace_id}_{suffix_id}.json" - - -def _format_tools(raw_tools: object) -> list[JsonObject]: - """Port the V1 message_history tool shape: ``{id, description, inputSchema}``.""" - if not isinstance(raw_tools, list): - return [] - tools: list[JsonObject] = [] - for tool in raw_tools: - if not isinstance(tool, dict): - continue - entry: JsonObject = {} - function = tool.get("function") - if isinstance(function, dict): - entry["id"] = function.get("name", "") - entry["description"] = function.get("description", "") - if "parameters" in function: - entry["inputSchema"] = {"jsonSchema": function["parameters"]} - else: - entry["id"] = tool.get("name", tool.get("id", "")) - entry["description"] = tool.get("description", "") - if "input_schema" in tool: - entry["inputSchema"] = {"jsonSchema": tool["input_schema"]} - elif "parameters" in tool: - entry["inputSchema"] = {"jsonSchema": tool["parameters"]} - tools.append(entry) - return tools - - -def _format_tool_choice(tool_choice: object) -> str: - if isinstance(tool_choice, str): - return tool_choice - if isinstance(tool_choice, dict): - choice_type = tool_choice.get("type") - if isinstance(choice_type, str): - return choice_type - return "auto" diff --git a/switchyard/lib/processors/routing_log_response_processor.py b/switchyard/lib/processors/routing_log_response_processor.py deleted file mode 100644 index cc87c0183..000000000 --- a/switchyard/lib/processors/routing_log_response_processor.py +++ /dev/null @@ -1,205 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Response-side processor that appends one JSONL routing record per request.""" - -from __future__ import annotations - -import asyncio -import json -import logging -import threading -from collections.abc import Mapping -from datetime import UTC, datetime -from pathlib import Path -from typing import TYPE_CHECKING, Any - -from switchyard.lib.chat_response.streaming_response_accumulator import ( - attach_final_response_callback, -) -from switchyard.lib.proxy_context import CTX_PROXY_ACTUAL_MODEL, ProxyContext -from switchyard.lib.request_metadata import CTX_REQUEST_HEADERS, CTX_REQUEST_METADATA -from switchyard_rust.core import ChatResponse - -if TYPE_CHECKING: - from switchyard.lib.endpoints.base import Endpoint - -logger = logging.getLogger(__name__) - - -class RoutingLogResponseProcessor: - """Append one JSON line per completed request to ``log_file``. - - Each record carries the routing decision (selected model and tier), the - caller-supplied task and session identity headers, and token usage, so a - benchmark harness can attribute router traffic to individual tasks. - Streaming responses log once the stream drains; write failures are logged - and never break the proxied response. - """ - - def __init__(self, log_file: Path | str) -> None: - self._log_file = Path(log_file) - self._log_file.parent.mkdir(parents=True, exist_ok=True) - self._lock = threading.Lock() - - async def process(self, ctx: ProxyContext, response: ChatResponse) -> ChatResponse: - """Log one routing record for the completed request; return the response unchanged.""" - served_model: str = ( - ctx.metadata.get(CTX_PROXY_ACTUAL_MODEL) or ctx.selected_model or "unknown" - ) - - async def _emit(final: ChatResponse) -> None: - await asyncio.to_thread(self._write_record, ctx, served_model, final) - - attached = attach_final_response_callback( - response, served_model=served_model, callback=_emit, - ) - if not attached: - await _emit(response) - return response - - def _write_record(self, ctx: ProxyContext, served_model: str, response: ChatResponse) -> None: - metadata = ctx.metadata.get(CTX_REQUEST_METADATA) - headers = ctx.metadata.get(CTX_REQUEST_HEADERS) or {} - # Prefer the model id the backend actually served so buckets reconcile - # with the global routing_stats schema, which keys on the served id. - actual_model = _field(response.body, "model") - record = { - "ts": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z", - "task": getattr(metadata, "task", None), - "trial_id": headers.get("x-switchyard-trial-id"), - "session_id": getattr(metadata, "session_id", None), - "model": actual_model if isinstance(actual_model, str) and actual_model else served_model, - "tier": ctx.selected_target or "", - **_usage_tokens(response.body), - } - try: - line = json.dumps(record, separators=(",", ":")) - with self._lock, self._log_file.open("a", encoding="utf-8") as handle: - handle.write(line + "\n") - except OSError as exc: - logger.warning("Routing log: failed to append to %s: %s", self._log_file, exc) - - def snapshot_session(self, session_id: str) -> dict[str, object] | None: - """Aggregate the durable request log for one exact trial session. - - This re-reads and re-parses the whole log on every call. It is sized for - benchmark runs (thousands of records), not a long-lived production server - with millions of sessions, where it would be O(records) per query. - """ - models: dict[str, dict[str, int]] = {} - totals = { - "total_calls": 0, - "total_prompt_tokens": 0, - "total_cached_tokens": 0, - "total_cache_creation_tokens": 0, - "total_completion_tokens": 0, - } - try: - if not self._log_file.is_file(): - return None - lines = self._log_file.read_text(encoding="utf-8").splitlines() - except OSError as exc: - logger.warning("Routing log: failed to read %s: %s", self._log_file, exc) - return None - - for line in lines: - try: - record = json.loads(line) - except json.JSONDecodeError: - continue - if not isinstance(record, Mapping) or record.get("session_id") != session_id: - continue - model_value = record.get("model") - model = model_value if isinstance(model_value, str) and model_value else "unknown" - bucket = models.setdefault( - model, - { - "calls": 0, - "prompt_tokens": 0, - "cached_tokens": 0, - "cache_creation_tokens": 0, - "completion_tokens": 0, - }, - ) - bucket["calls"] += 1 - totals["total_calls"] += 1 - for record_key, bucket_key, total_key in ( - ("prompt_tokens", "prompt_tokens", "total_prompt_tokens"), - ("cached_tokens", "cached_tokens", "total_cached_tokens"), - ( - "cache_creation_tokens", - "cache_creation_tokens", - "total_cache_creation_tokens", - ), - ("completion_tokens", "completion_tokens", "total_completion_tokens"), - ): - value = record.get(record_key) - if isinstance(value, int) and not isinstance(value, bool) and value >= 0: - bucket[bucket_key] += value - totals[total_key] += value - - if not totals["total_calls"]: - return None - return {"session_id": session_id, **totals, "models": models} - - def get_endpoint(self) -> Endpoint: - """Contribute the session-scoped routing-stat snapshot endpoint.""" - from switchyard.lib.endpoints.routing_log_stats_endpoint import ( - RoutingLogStatsEndpoint, - ) - - return RoutingLogStatsEndpoint(self) - - -def _usage_tokens(body: object) -> dict[str, int]: - """Six-field token breakdown matching the global routing-stats schema. - - cached/cache_creation are subsets of prompt_tokens, reasoning a subset of - completion_tokens, total = prompt + completion. - """ - usage = _field(body, "usage") - prompt = _int_field(usage, "prompt_tokens") - completion = _int_field(usage, "completion_tokens") - cached = 0 - cache_creation = 0 - reasoning = 0 - if prompt or completion: - # OpenAI Chat Completions. - prompt_details = _field(usage, "prompt_tokens_details") - cached = _int_field(prompt_details, "cached_tokens") - cache_creation = _int_field(prompt_details, "cache_creation_tokens") - reasoning = _int_field(_field(usage, "completion_tokens_details"), "reasoning_tokens") - else: - completion = _int_field(usage, "output_tokens") - reasoning = _int_field(_field(usage, "output_tokens_details"), "reasoning_tokens") - input_details = _field(usage, "input_tokens_details") - if input_details is not None: - # OpenAI Responses API: prompt_tokens already includes cached. - prompt = _int_field(usage, "input_tokens") - cached = _int_field(input_details, "cached_tokens") - cache_creation = _int_field(input_details, "cache_creation_tokens") - else: - # Anthropic Messages: cache tokens are prompt siblings, so fold in. - cached = _int_field(usage, "cache_read_input_tokens") - cache_creation = _int_field(usage, "cache_creation_input_tokens") - prompt = _int_field(usage, "input_tokens") + cached + cache_creation - return { - "prompt_tokens": prompt, - "cached_tokens": cached, - "cache_creation_tokens": cache_creation, - "completion_tokens": completion, - "reasoning_tokens": reasoning, - "total_tokens": prompt + completion, - } - - -def _field(value: object, name: str) -> Any: - if isinstance(value, Mapping): - return value.get(name) - return getattr(value, name, None) - - -def _int_field(value: object, name: str) -> int: - field = _field(value, name) - return field if isinstance(field, int) else 0 diff --git a/switchyard/lib/processors/stats_request_processor.py b/switchyard/lib/processors/stats_request_processor.py deleted file mode 100644 index cf6c3c8a9..000000000 --- a/switchyard/lib/processors/stats_request_processor.py +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Rust-owned stats request processor export.""" - -from switchyard_rust.components import StatsRequestProcessor - -__all__ = ["StatsRequestProcessor"] diff --git a/switchyard/lib/processors/stats_response_processor_accumulator.py b/switchyard/lib/processors/stats_response_processor_accumulator.py deleted file mode 100644 index 48526a927..000000000 --- a/switchyard/lib/processors/stats_response_processor_accumulator.py +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Rust-owned stats response processor export.""" - -from switchyard_rust.components import StatsResponseProcessor - -__all__ = ["StatsResponseProcessor"] diff --git a/switchyard/lib/prometheus_exposition.py b/switchyard/lib/prometheus_exposition.py deleted file mode 100644 index 146ad0a26..000000000 --- a/switchyard/lib/prometheus_exposition.py +++ /dev/null @@ -1,216 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Render a :class:`StatsAccumulator` snapshot as Prometheus exposition text. - -Hand-rolled so the metrics surface ships without adding the ``prometheus-client`` -dependency. The exposition follows Prometheus text format 0.0.4: ``# HELP`` / -``# TYPE`` headers, one sample per line, label values escaped per the spec. - -Metric design (all names prefixed ``switchyard_``): - -- ``requests_total{model,tier}`` — counter; selected model / tier traffic. -- ``errors_total{model,tier}`` — counter; backend-call errors. -- ``prompt_tokens_total`` / ``completion_tokens_total`` / - ``cached_tokens_total`` / ``cache_creation_tokens_total`` / - ``reasoning_tokens_total`` — counters per model/tier. -- ``model_call_latency_ms`` — summary; backend-only call latency, - per model/tier, with ``quantile="0.5"`` and ``quantile="0.99"``. -- ``total_latency_ms`` — summary; end-to-end request latency, per model/tier. -- ``routing_overhead_ms`` — summary; global router decision overhead - (``total_latency - backend_latency``). -- ``total_requests`` / ``total_errors`` — gauges; running totals across all - models, mirroring the ``/v1/stats`` JSON top-level fields. - -Label cardinality is intentionally bounded: model names and tier ids are -both small fixed sets in any real deployment; nothing per-request (no -request ids, no prompts) is ever emitted. -""" - -from __future__ import annotations - -from importlib.metadata import version as _pkg_version -from typing import Any - -try: - _SWITCHYARD_VERSION = _pkg_version("nemo-switchyard") -except Exception: - _SWITCHYARD_VERSION = "unknown" - - -def render_prometheus(snapshot: dict[str, Any]) -> str: - """Render a stats snapshot as Prometheus text-format exposition. - - Args: - snapshot: The dict returned by :meth:`StatsAccumulator.snapshot_sync`. - - Returns: - A string containing the full exposition payload (UTF-8, LF-terminated - lines, trailing newline). - """ - lines: list[str] = [] - - _emit_gauge( - lines, - "switchyard_build_info", - "Switchyard build information.", - [({"version": _SWITCHYARD_VERSION}, 1)], - ) - - total_requests = snapshot.get("total_requests", 0) - total_errors = snapshot.get("total_errors", 0) - _emit_gauge( - lines, - "switchyard_total_requests", - "Total chain-level requests recorded.", - [({}, total_requests)], - ) - _emit_gauge( - lines, - "switchyard_total_errors", - "Total chain-level backend errors recorded.", - [({}, total_errors)], - ) - - models: dict[str, dict[str, Any]] = snapshot.get("models", {}) or {} - - # Counter families keyed by (description, attribute on per-model dict). - counters: list[tuple[str, str, str]] = [ - ("switchyard_requests_total", "Calls per selected model/tier.", "calls"), - ("switchyard_errors_total", "Backend errors per selected model/tier.", "errors"), - ("switchyard_prompt_tokens_total", "Prompt tokens billed per model/tier.", "prompt_tokens"), - ( - "switchyard_completion_tokens_total", - "Completion tokens generated per model/tier.", - "completion_tokens", - ), - ("switchyard_cached_tokens_total", "Cached prompt tokens per model/tier.", "cached_tokens"), - ( - "switchyard_cache_creation_tokens_total", - "Cache-creation tokens per model/tier.", - "cache_creation_tokens", - ), - ( - "switchyard_reasoning_tokens_total", - "Reasoning tokens per model/tier.", - "reasoning_tokens", - ), - ] - for metric, help_text, attr in counters: - samples = [ - (_labels_for(model, m), m.get(attr, 0)) for model, m in sorted(models.items()) - ] - _emit_counter(lines, metric, help_text, samples) - - # Summary families: backend-call latency and total-latency per model/tier. - summaries: list[tuple[str, str, str]] = [ - ( - "switchyard_model_call_latency_ms", - "Backend-call latency per model/tier (ms).", - "model_call_latency", - ), - ( - "switchyard_total_latency_ms", - "End-to-end request latency per model/tier (ms).", - "total_latency", - ), - ] - for metric, help_text, attr in summaries: - per_model = [ - (_labels_for(model, m), m.get(attr) or _empty_histogram()) - for model, m in sorted(models.items()) - ] - _emit_summary(lines, metric, help_text, per_model) - - # Global routing-overhead summary (router decision time minus backend time). - overhead = snapshot.get("routing_overhead") or _empty_histogram() - _emit_summary( - lines, - "switchyard_routing_overhead_ms", - "Router decision latency overhead across all requests (ms).", - [({}, overhead)], - ) - - return "\n".join(lines) + "\n" - - -def _labels_for(model: str, m: dict[str, Any]) -> dict[str, str]: - labels: dict[str, str] = {"model": model} - tier = m.get("tier") - if tier: - labels["tier"] = str(tier) - return labels - - -def _empty_histogram() -> dict[str, float | int]: - return {"count": 0, "total_ms": 0.0, "p50_ms": 0.0, "p99_ms": 0.0} - - -def _emit_gauge( - lines: list[str], - name: str, - help_text: str, - samples: list[tuple[dict[str, str], float | int]], -) -> None: - lines.append(f"# HELP {name} {help_text}") - lines.append(f"# TYPE {name} gauge") - for labels, value in samples: - lines.append(f"{name}{render_labels(labels)} {format_number(value)}") - - -def _emit_counter( - lines: list[str], - name: str, - help_text: str, - samples: list[tuple[dict[str, str], float | int]], -) -> None: - lines.append(f"# HELP {name} {help_text}") - lines.append(f"# TYPE {name} counter") - for labels, value in samples: - lines.append(f"{name}{render_labels(labels)} {format_number(value)}") - - -def _emit_summary( - lines: list[str], - name: str, - help_text: str, - samples: list[tuple[dict[str, str], dict[str, float | int]]], -) -> None: - lines.append(f"# HELP {name} {help_text}") - lines.append(f"# TYPE {name} summary") - for labels, hist in samples: - for quantile in ("0.5", "0.99"): - q_labels = dict(labels) - q_labels["quantile"] = quantile - field = "p50_ms" if quantile == "0.5" else "p99_ms" - lines.append( - f"{name}{render_labels(q_labels)} {format_number(hist.get(field, 0.0))}" - ) - lines.append( - f"{name}_sum{render_labels(labels)} {format_number(hist.get('total_ms', 0.0))}" - ) - lines.append( - f"{name}_count{render_labels(labels)} {format_number(hist.get('count', 0))}" - ) - - -def render_labels(labels: dict[str, str]) -> str: - if not labels: - return "" - parts = [f'{k}="{_escape_label_value(v)}"' for k, v in labels.items()] - return "{" + ",".join(parts) + "}" - - -def _escape_label_value(value: str) -> str: - # Per Prometheus exposition spec: escape backslash, double-quote, newline. - return value.replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n") - - -def format_number(value: float | int) -> str: - if isinstance(value, int): - return str(value) - if value != value or value in (float("inf"), float("-inf")): # NaN / inf - return "0" - if value.is_integer(): - return str(int(value)) - return f"{value:g}" diff --git a/switchyard/lib/proxy_context.py b/switchyard/lib/proxy_context.py deleted file mode 100644 index 9b715a7f7..000000000 --- a/switchyard/lib/proxy_context.py +++ /dev/null @@ -1,123 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""ProxyContext re-export and shared metadata keys.""" - -from switchyard_rust.core import ProxyContext - -# --------------------------------------------------------------------------- -# Metadata key constants -# --------------------------------------------------------------------------- -# Use these instead of bare strings to avoid silent typos and to make -# cross-component key contracts discoverable at import time. - -#: Stores a deep-copy of the incoming request dict. -#: Written by RequestBufferProcessor. -CTX_ORIGINAL_REQUEST = "original_request" - -#: Model that was actually used for the LLM call, after any routing/override. -#: Written by routing backends and processors. -CTX_PROXY_ACTUAL_MODEL = "_proxy_actual_model" - -#: Routing metadata dict produced by tier-routing request processors. -CTX_ROUTING = "_routing" - -#: Original inbound format stored by translation layers. -CTX_ORIGINAL_FORMAT = "_original_format" - -#: Original model name stored by translation layers. -CTX_ORIGINAL_MODEL = "_original_model" - -#: Target wire format chosen by a router (e.g. random routing). -#: Written by router request-side components; read by -#: FormatTranslateRequestProcessor. -CTX_TARGET_FORMAT = "_target_format" - -#: Caller-supplied API key extracted from the inbound request's -#: ``x-switchyard-api-key`` header (preferred — survives proxies that strip -#: ``Authorization``), or ``Authorization: Bearer `` / ``x-api-key``. Set by -#: the HTTP endpoint after header parsing; consumed by backends that support -#: opt-in per-caller credential forwarding. -#: Absent when the caller did not supply a credential or supplied a known -#: launcher-sentinel placeholder. -CTX_CALLER_API_KEY = "_caller_api_key" - -#: Upstream HTTP status code recorded by a Python backend when an LLM -#: provider returns a non-2xx response. Endpoints inspect this on the -#: error path to passthrough the upstream status (e.g. 401 from a bad -#: API key) instead of masking it as a 500. Rust backend errors don't -#: round-trip structurally — they are reported as opaque strings via -#: ``SwitchyardError::Upstream`` — so this is currently the Python-side -#: channel only. -CTX_UPSTREAM_HTTP_STATUS = "_upstream_http_status" - -#: Upstream HTTP response body recorded alongside -#: :data:`CTX_UPSTREAM_HTTP_STATUS`. May be a string or a JSON-decodable -#: dict — endpoints pass it through to the caller verbatim. -CTX_UPSTREAM_HTTP_BODY = "_upstream_http_body" - -#: Set truthy by a backend that records its own per-attempt -#: ``switchyard.lib.endpoints.outcome_metrics`` upstream-attempt counters -#: (e.g. a backend that retries across endpoints and must count each -#: attempt). When present, the endpoint layer skips its -#: single-attempt fallback recording for this request so retry fan-out is not -#: double-counted. Absent for the Rust native / passthrough / multi backends, -#: which issue exactly one upstream attempt per call and have no Python retry -#: loop — those rely on the endpoint fallback. -CTX_UPSTREAM_ATTEMPTS_RECORDED = "_upstream_attempts_recorded" - -#: Which layer originated the failure being surfaced to the client: -#: ``"provider"`` for an upstream LLM failure passed through, ``"switchyard"`` -#: for an error this proxy synthesized (credential rejection, translation -#: rejection, routing failure). Written by backends on the error path; read by -#: the endpoint layer to stamp the ``x-switchyard-error-source`` response -#: header. Unset means the endpoint's per-path default applies (``provider`` -#: for a stashed upstream status, ``switchyard`` for synthesized envelopes). -CTX_ERROR_SOURCE = "_error_source" - -#: :data:`CTX_ERROR_SOURCE` value for errors Switchyard itself originated. -#: Defined here (not in the endpoints layer) so backends can stamp it without -#: importing FastAPI-dependent modules. -ERROR_SOURCE_SWITCHYARD = "switchyard" - -#: :data:`CTX_ERROR_SOURCE` value for upstream provider failures passed through. -ERROR_SOURCE_PROVIDER = "provider" - -#: Upstream model actually attempted when the surfaced failure happened, when -#: a routing selection took place. Written alongside -#: :data:`CTX_ERROR_SOURCE`; read by the endpoint layer to stamp the -#: ``x-switchyard-upstream-model`` response header. -CTX_UPSTREAM_MODEL = "_upstream_model" - -#: Route-selection record for spend/tokenomics attribution, written by a -#: routing backend after a successful upstream call. A dict with the keys -#: ``router_model`` (client-facing route id), ``router_strategy``, -#: ``router_selected_endpoint``, ``router_selected_model``, -#: ``router_selected_provider``, and ``router_correlation_id``. The same -#: payload is stamped on the outbound upstream call as the -#: ``x-litellm-spend-logs-metadata`` header (one per attempt, so provider -#: spend-log rows record the endpoint they actually hit); the endpoint layer -#: reads this key to return the ``x-switchyard-*`` route-selection response -#: headers, letting a front proxy enrich its own spend-log row with the same -#: correlation id. -CTX_ROUTE_SELECTION = "_route_selection" - - -__all__ = [ - "CTX_CALLER_API_KEY", - "CTX_ERROR_SOURCE", - "CTX_ORIGINAL_FORMAT", - "CTX_ORIGINAL_MODEL", - "CTX_ORIGINAL_REQUEST", - "CTX_PROXY_ACTUAL_MODEL", - "CTX_ROUTE_SELECTION", - "CTX_ROUTING", - "CTX_TARGET_FORMAT", - "CTX_UPSTREAM_ATTEMPTS_RECORDED", - "CTX_UPSTREAM_HTTP_BODY", - "CTX_UPSTREAM_HTTP_STATUS", - "CTX_UPSTREAM_MODEL", - "ERROR_SOURCE_PROVIDER", - "ERROR_SOURCE_SWITCHYARD", - "ProxyContext", -] diff --git a/switchyard/lib/request_metadata.py b/switchyard/lib/request_metadata.py deleted file mode 100644 index de4eed7be..000000000 --- a/switchyard/lib/request_metadata.py +++ /dev/null @@ -1,142 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""request metadata helpers for HTTP endpoint context.""" - -from collections.abc import Mapping -from dataclasses import dataclass -from typing import Any - -from switchyard.lib.proxy_context import CTX_CALLER_API_KEY - -CTX_REQUEST_METADATA = "_request_metadata" -CTX_REQUEST_HEADERS = "_request_headers" - -# Existing Switchyard session header. Do not add aliases here unless a -# concrete client requires one; keeping one spelling avoids ambiguity. -PROXY_SESSION_ID_HEADER = "proxy_x_session_id" -INTAKE_TASK_HEADER = "x-switchyard-intake-task" - -# Sentinel values our own launchers send as the ``Authorization`` / -# ``OPENAI_API_KEY`` value so coding agents satisfy their "no key set" -# preconditions. Treat as if no key was supplied. The codex launcher -# sets ``OPENAI_API_KEY="switchyard"`` (see codex_cli_launcher.py). -_CALLER_KEY_SENTINELS = frozenset({"switchyard", ""}) - -# Dedicated forwarded credential header. Preferred over ``Authorization`` -# because a proxy in front of Switchyard (e.g. LiteLLM) consumes the -# ``Authorization`` header for its own auth and strips it before the upstream -# call, while a custom header passes through untouched — so a BYO-key caller -# behind such a proxy stays correctly attributed for upstream inference spend. -CALLER_API_KEY_HEADER = "x-switchyard-api-key" # pragma: allowlist secret - -# Request headers whose values carry a caller credential. Values are redacted -# in the map retained as ``CTX_REQUEST_HEADERS``. The raw key remains separately -# in ``CTX_CALLER_API_KEY`` for upstream forwarding and is not serialized by -# logging or tracing. -_SENSITIVE_HEADERS = frozenset({"authorization", "x-api-key", CALLER_API_KEY_HEADER}) -_REDACTED = "[REDACTED]" - - -@dataclass(frozen=True, slots=True) -class RequestMetadata: - """Session and task identifiers retained for compatibility routing logs.""" - - session_id: str | None = None - task: str | None = None - - @classmethod - def from_headers(cls, headers: Mapping[str, str]) -> "RequestMetadata": - """Extract request metadata from a case-insensitive header mapping.""" - normalized = {name.lower(): value for name, value in headers.items()} - return cls( - session_id=_nonempty_header(normalized, PROXY_SESSION_ID_HEADER), - task=_nonempty_header(normalized, INTAKE_TASK_HEADER), - ) - - -def attach_request_metadata( - ctx: Any, - metadata: RequestMetadata, - headers: Mapping[str, str] | None = None, -) -> None: - """Attach request metadata and redacted headers to the Python context.""" - ctx.metadata[CTX_REQUEST_METADATA] = metadata - if headers is not None: - # Redact credential headers before retaining the map: the caller key is - # already extracted into ``CTX_CALLER_API_KEY`` for upstream forwarding, - # so nothing downstream needs the raw value, and a retained/logged header - # map must not expose it. - ctx.metadata[CTX_REQUEST_HEADERS] = redact_sensitive_headers(headers) - - -def _nonempty_header(headers: Mapping[str, str], name: str) -> str | None: - value = headers.get(name) - return value if value else None - - -def redact_sensitive_headers(headers: Mapping[str, str]) -> dict[str, str]: - """Return a copy of *headers* with credential-bearing values redacted. - - Headers named in :data:`_SENSITIVE_HEADERS` (``Authorization``, ``x-api-key``, - ``x-switchyard-api-key``) have their values replaced with ``"[REDACTED]"`` so a - retained or logged header map can never expose the caller's API key. - Header-name matching is case-insensitive. - """ - return { - name: (_REDACTED if name.lower() in _SENSITIVE_HEADERS else value) - for name, value in headers.items() - } - - -def attach_caller_api_key(ctx: Any, headers: Mapping[str, str]) -> None: - """Attach the caller-supplied API key to *ctx* when the request carries one.""" - caller_key = extract_caller_api_key(headers) - if caller_key is not None: - ctx.metadata[CTX_CALLER_API_KEY] = caller_key - - -def extract_caller_api_key(headers: Mapping[str, str]) -> str | None: - """Pull the caller-supplied API key out of an HTTP request's headers. - - Precedence: the dedicated ``x-switchyard-api-key`` header first, then - ``Authorization: Bearer ``, then ``x-api-key``. The dedicated header is - preferred because a proxy in front of Switchyard (e.g. LiteLLM) strips - ``Authorization`` before the upstream call; the custom header survives, so the - caller's key — not a service key — is the credential billed for upstream - inference. Returns ``None`` when no usable header is present, the bearer - scheme is missing, or the value is a known launcher sentinel (so coding-agent - placeholder keys do not get forwarded upstream as real credentials). - """ - forwarded = headers.get(CALLER_API_KEY_HEADER) or headers.get("X-Switchyard-Api-Key") - if forwarded: - candidate = forwarded.strip() - if candidate.lower() not in _CALLER_KEY_SENTINELS: - return candidate - auth = headers.get("authorization") or headers.get("Authorization") - if auth: - scheme, _, value = auth.partition(" ") - if scheme.lower() == "bearer" and value: - candidate = value.strip() - if candidate.lower() not in _CALLER_KEY_SENTINELS: - return candidate - api_key = headers.get("x-api-key") or headers.get("X-Api-Key") - if api_key: - candidate = api_key.strip() - if candidate.lower() not in _CALLER_KEY_SENTINELS: - return candidate - return None - - -__all__ = [ - "CALLER_API_KEY_HEADER", - "CTX_REQUEST_METADATA", - "CTX_REQUEST_HEADERS", - "INTAKE_TASK_HEADER", - "PROXY_SESSION_ID_HEADER", - "RequestMetadata", - "attach_caller_api_key", - "attach_request_metadata", - "extract_caller_api_key", - "redact_sensitive_headers", -] diff --git a/switchyard/lib/roles.py b/switchyard/lib/roles.py deleted file mode 100644 index 397b9f2e9..000000000 --- a/switchyard/lib/roles.py +++ /dev/null @@ -1,48 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Python backend role class and translated-response aliases. - -Request-side and response-side components are plain objects with async -``process(...)`` methods. The backend remains nominal because it owns upstream -transport behavior and request-format support; native implementations register -with the same Python role class. -""" - -from __future__ import annotations - -from collections.abc import AsyncIterable, Mapping -from typing import TypeAlias - -from anthropic.types import Message as AnthropicMessage -from anthropic.types import RawMessageStreamEvent -from openai.types.chat import ChatCompletion, ChatCompletionChunk -from openai.types.responses import Response as OpenAIResponse -from openai.types.responses import ResponseStreamEvent - -from switchyard_rust.core import LLMBackend - -# The final translated response surfaced by TranslationEngine.translate(). -# Union covers all three formats x (non-streaming | streaming) plus the -# dict-returning converters. -TranslatedStream: TypeAlias = ( - AsyncIterable[ChatCompletionChunk] - | AsyncIterable[RawMessageStreamEvent] - | AsyncIterable[ResponseStreamEvent] - | AsyncIterable[Mapping[str, object]] - | AsyncIterable[str] -) - -TranslatedResponse: TypeAlias = ( - ChatCompletion - | OpenAIResponse - | AnthropicMessage - | Mapping[str, object] - | TranslatedStream -) - -__all__ = [ - "LLMBackend", - "TranslatedResponse", - "TranslatedStream", -] diff --git a/switchyard/lib/route_table.py b/switchyard/lib/route_table.py deleted file mode 100644 index 1ea7b6251..000000000 --- a/switchyard/lib/route_table.py +++ /dev/null @@ -1,170 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""RouteTable — route inbound requests to per-model runtimes. - -Stores a static mapping of model name → callable runtime. The launcher -populates the table at startup after building all configured runtimes. The -three HTTP endpoints read ``body["model"]``, call -:meth:`lookup_switchyard`, and dispatch to the returned chain before making any -backend calls. -""" - -import logging -from collections.abc import Iterator, Mapping -from typing import Any, ClassVar, Protocol, TypeAlias - -from switchyard.lib.model_listing import model_entry -from switchyard.lib.proxy_context import ProxyContext -from switchyard.lib.roles import TranslatedResponse -from switchyard_rust.core import ChatRequest - -log = logging.getLogger(__name__) - - -#: Type accepted by app factories and launcher runtimes — either a single -#: :class:`Switchyard` chain or a :class:`RouteTable` that dispatches -#: by inbound model id. -class ChainRuntime(Protocol): - """Runtime accepted by the table and FastAPI dispatcher.""" - - async def call( - self, - request: ChatRequest, - *, - ctx: ProxyContext | None = None, - ) -> TranslatedResponse: - """Execute one request through the runtime.""" - ... - - def iter_components(self) -> list[Any]: - """Return lifecycle components in startup order.""" - ... - - -SwitchyardApp: TypeAlias = "ChainRuntime | RouteTable" - - -class RouteTable: - """Table that maps model names to pre-built runtimes. - - Register one chain per model the proxy should handle explicitly. Unknown - models raise ``KeyError`` so endpoint handlers can return ``model_not_found`` - instead of silently forwarding a request to the wrong backend. - """ - - #: Same key as :class:`Switchyard` so app factories store this under the - #: attribute the V2 endpoint handlers already read. - state_key: ClassVar[str] = "switchyard" - - def __init__(self) -> None: - self._by_model: dict[str, ChainRuntime] = {} - self._metadata_by_model: dict[str, dict[str, Any]] = {} - self._model_listing_warnings: list[str] = [] - self._default_model: str | None = None - # Last model id that `lookup_switchyard` successfully resolved. - # Updated on every request ingress (each endpoint calls - # `lookup_switchyard` once per request before any backend work). - # Read by the launcher's live stats footer so the displayed model - # tracks what the user actually picked via /model, with no delay - # for streaming responses. - self._last_looked_up: str | None = None - - def register( - self, - model: str, - switchyard: ChainRuntime, - metadata: Mapping[str, Any] | None = None, - default: bool = False, - ) -> None: - """Register *switchyard* as the exact-match chain for *model*.""" - self._by_model[model] = switchyard - self._metadata_by_model[model] = dict(metadata or {}) - if default: - self._default_model = model - log.debug("RouteTable: registered chain for model=%r", model) - - def registered_models(self) -> list[str]: - """Return registered model ids in registration order.""" - return list(self._by_model) - - def set_default_model(self, model: str) -> None: - """Mark *model* as the default entry advertised by ``/v1/models``.""" - if model not in self._by_model: - raise KeyError(model) - self._default_model = model - - def default_model(self) -> str | None: - """Return the advertised default model id, falling back to first entry.""" - if self._default_model in self._by_model: - return self._default_model - return next(iter(self._by_model), None) - - def items(self) -> Iterator[tuple[str, ChainRuntime, dict[str, Any]]]: - """Iterate ``(model_id, chain, metadata)`` triples in registration order. - - Used when a caller needs to merge one table into another — e.g. a - caller that composes multiple route tables. - """ - for model in self._by_model: - yield model, self._by_model[model], dict(self._metadata_by_model.get(model, {})) - - def lookup_switchyard(self, model: str) -> ChainRuntime: - """Return the chain for *model*. - - Records *model* as the last successfully resolved id (see - :attr:`last_looked_up`). - - Raises: - KeyError: *model* is unregistered. - """ - chain = self._by_model.get(model) - if chain is not None: - log.debug("RouteTable: model=%r → registered chain", model) - self._last_looked_up = model - return chain - raise KeyError(model) - - @property - def last_looked_up(self) -> str | None: - """Model id from the most recent successful :meth:`lookup_switchyard`. - - ``None`` until the first request arrives. Set at request ingress, so - the value reflects the model the user is *currently* sending traffic - for — useful for live launcher TUIs that want to display which - table entry the client most recently picked. - """ - return self._last_looked_up - - def registered_model_entries(self) -> list[dict[str, Any]]: - """Return OpenAI-compatible model entries with optional metadata.""" - entries: list[dict[str, Any]] = [] - for model in self._by_model: - metadata = dict(self._metadata_by_model.get(model, {})) - entries.append(model_entry(model, metadata=metadata)) - return entries - - def add_model_listing_warning(self, warning: str) -> None: - """Record non-fatal model catalog discovery warnings for ``/v1/models``.""" - if warning not in self._model_listing_warnings: - self._model_listing_warnings.append(warning) - - def model_listing_warnings(self) -> list[str]: - """Return non-fatal model catalog warnings in discovery order.""" - return list(self._model_listing_warnings) - - def iter_components(self) -> list[Any]: - """Return all chain components across registered chains, deduplicated. - - Components shared by object identity are returned once so endpoint and - shutdown hooks are not double-registered. - """ - seen: set[int] = set() - result: list[Any] = [] - for switchyard in self._by_model.values(): - for component in switchyard.iter_components(): - if id(component) in seen: - continue - seen.add(id(component)) - result.append(component) - return result diff --git a/switchyard/lib/startup_timing.py b/switchyard/lib/startup_timing.py deleted file mode 100644 index bff550e42..000000000 --- a/switchyard/lib/startup_timing.py +++ /dev/null @@ -1,56 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Per-stage startup timing for ``switchyard launch``. - -Turn it on with ``--startup-timing`` or ``SWITCHYARD_STARTUP_TIMING=1``; it does -nothing when off. ``mark()`` records a checkpoint during startup, and ``dump()`` -prints the time between checkpoints to stderr just before the agent starts. It -times only switchyard's own startup work, not Python's one-time import cost. -""" - -import os -import sys -import time -from typing import TextIO - -# On unless SWITCHYARD_STARTUP_TIMING is unset or "0", or once --startup-timing calls -# enable(). Read once at import — the env var does not change during a launch. -enabled: bool = os.environ.get("SWITCHYARD_STARTUP_TIMING", "0") != "0" - -# (label, perf_counter timestamp) for each point reached during startup. -_marks: list[tuple[str, float]] = [] - - -def enable() -> None: - """Turn timing on for this process (called by the ``--startup-timing`` flag).""" - global enabled - enabled = True - - -def mark(label: str) -> None: - """Record that startup reached *label*. No-op unless timing is enabled.""" - if enabled: - _marks.append((label, time.perf_counter())) - - -def dump(stream: TextIO | None = None) -> None: - """Print the per-stage breakdown to stderr, then reset. No-op when disabled. - - Each line is the time between consecutive marks; the last line is the total - from the first mark to the last. - """ - if not enabled or len(_marks) < 2: - _marks.clear() - return - out = stream if stream is not None else sys.stderr - start = _marks[0][1] - prev = start - lines = ["switchyard startup timing:"] - for label, stamp in _marks[1:]: - lines.append(f" {(stamp - prev) * 1000:8.1f} ms {label}") - prev = stamp - lines.append(f" {'-' * 8}") - lines.append(f" {(prev - start) * 1000:8.1f} ms total (launch invoked -> child spawn)") - print("\n".join(lines), file=out) - _marks.clear() diff --git a/switchyard/lib/stats_accumulator.py b/switchyard/lib/stats_accumulator.py deleted file mode 100644 index 0b062681c..000000000 --- a/switchyard/lib/stats_accumulator.py +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Rust-owned stats accumulator export.""" - -from switchyard_rust.components import StatsAccumulator - -__all__ = ["StatsAccumulator"] diff --git a/switchyard/lib/switchyard.py b/switchyard/lib/switchyard.py deleted file mode 100644 index e4515ef0e..000000000 --- a/switchyard/lib/switchyard.py +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Python compatibility Switchyard chain export.""" - -from __future__ import annotations - -from switchyard_rust.core import Switchyard - -__all__ = ["Switchyard"] diff --git a/switchyard/lib/tracing.py b/switchyard/lib/tracing.py deleted file mode 100644 index 5f80a7f15..000000000 --- a/switchyard/lib/tracing.py +++ /dev/null @@ -1,72 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Optional Datadog (ddtrace) spans for the proxy routing path. - -Switchyard runs inside a proxy that owns the Datadog APM trace. These helpers -add child spans and tags around the route-decision and upstream-attempt blocks -so routing behaviour is visible in APM. - -``ddtrace`` is an **optional** dependency (the ``tracing`` extra). When it is -not installed the helpers are complete no-ops with no overhead, so the default -install and non-Datadog deployments are unaffected. When it is installed, the -spans created here nest under whatever span the surrounding proxy already has -active, so they appear inline in the proxy's trace. -""" - -from __future__ import annotations - -from collections.abc import Iterator, Mapping -from contextlib import contextmanager -from typing import Any, Protocol, runtime_checkable - -try: # ddtrace >= 2.0 exposes the global tracer here - from ddtrace.trace import tracer as _dd_tracer -except Exception: # pragma: no cover - import shape varies / absent in dev - try: # ddtrace < 2.0 fallback - from ddtrace import tracer as _dd_tracer - except Exception: - _dd_tracer = None - - -@runtime_checkable -class Span(Protocol): - """Minimal span surface the routing instrumentation relies on.""" - - def set_tag(self, key: str, value: Any) -> None: ... - - -class _NoopSpan: - """Span stand-in used when no tracer is available; drops every tag.""" - - def set_tag(self, key: str, value: Any) -> None: - return None - - -_NOOP_SPAN = _NoopSpan() - - -@contextmanager -def routing_span(name: str) -> Iterator[Span]: - """Open a child span named *name*, or yield a no-op span when ddtrace is absent. - - The span nests under whatever span the surrounding proxy has active, so the - routing instrumentation shows up inline in the proxy's Datadog trace and is - finished automatically when the ``with`` block exits. - """ - if _dd_tracer is None: - yield _NOOP_SPAN - return - with _dd_tracer.trace(name) as span: - yield span - - -def set_tags(span: Span, tags: Mapping[str, Any]) -> None: - """Set each non-``None`` tag on *span*. - - ``None`` values are skipped so an unavailable signal (e.g. a signal that - has not been recorded yet) leaves no empty/misleading tag on the span. - """ - for key, value in tags.items(): - if value is not None: - span.set_tag(key, value) diff --git a/switchyard/server/__init__.py b/switchyard/server/__init__.py deleted file mode 100644 index 1482e4502..000000000 --- a/switchyard/server/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""HTTP serving primitives. - -Exposes: - -* ``server_util`` — CLI helpers. -* ``switchyard_app`` — FastAPI app factory for a ``Switchyard`` chain. -""" - -from switchyard.server.server_util import ( - DEFAULT_SECRETS_FILE, - REPO_ROOT, - add_common_args, - add_transport_args, - build_and_serve, - ensure_openai_api_key_env, - load_secrets, - resolve_config_with_secrets, - resolve_credentials_from_env, -) - -__all__ = [ - "DEFAULT_SECRETS_FILE", - "REPO_ROOT", - "add_common_args", - "add_transport_args", - "build_and_serve", - "ensure_openai_api_key_env", - "load_secrets", - "resolve_config_with_secrets", - "resolve_credentials_from_env", -] diff --git a/switchyard/server/server_util.py b/switchyard/server/server_util.py deleted file mode 100644 index b62275cc6..000000000 --- a/switchyard/server/server_util.py +++ /dev/null @@ -1,389 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Shared CLI helpers for ``switchyard.cli.switchyard_cli``. - -Provides: - -- secrets-file loading -- common argparse argument surface -- credential resolution from CLI / env -- :func:`build_and_serve`, which wraps a :class:`~switchyard.lib.switchyard.Switchyard` - in a FastAPI app via :func:`~switchyard.server.switchyard_app.build_switchyard_app` - and starts uvicorn -""" - -from __future__ import annotations - -import argparse -import json -import logging -import os -from collections.abc import Mapping -from enum import Enum -from pathlib import Path -from typing import TYPE_CHECKING - -from switchyard.lib.endpoints.base import Endpoint as NemoSwitchyardEndpoint -from switchyard.lib.switchyard import Switchyard - -if TYPE_CHECKING: - from switchyard.lib.route_table import RouteTable - - -class InboundFormat(Enum): - """Inbound wire format accepted by the proxy.""" - OPENAI = "openai" - ANTHROPIC = "anthropic" - BOTH = "both" - - -logger = logging.getLogger(__name__) - -# Four parents up: foundation/server/server_util.py → foundation/server → -# foundation → switchyard → repo root. -REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent -DEFAULT_SECRETS_FILE = REPO_ROOT / "secrets" / "secrets.json" - - -# --------------------------------------------------------------------------- -# Secrets -# --------------------------------------------------------------------------- - - -# secrets.json is a flat map of section name → section body. Each section -# body is itself a dict (provider creds or server config). Values are -# typed ``object`` so the narrow ``isinstance(section, dict)`` checks below -# stay honest — the alternative ``Any`` would silently absorb type errors. -SecretsFile = Mapping[str, Mapping[str, object]] - - -def load_secrets(secrets_file: Path | None = None) -> SecretsFile: - """Load secrets from ``secrets.json``, returning ``{}`` if not found.""" - path = secrets_file or DEFAULT_SECRETS_FILE - if path.exists(): - with open(path) as f: - loaded: SecretsFile = json.load(f) - return loaded - return {} - - -# --------------------------------------------------------------------------- -# Argparse surfaces -# --------------------------------------------------------------------------- - - -def add_transport_args(parser: argparse.ArgumentParser) -> None: - """Register transport-layer arguments: ``--host``, ``--port``, ``--inbound``, ``--reload``. - - Subcommands that define their own credential args with non-standard - names (e.g. ``--api-base`` instead of ``--base-url``) use this - helper to avoid the full :func:`add_common_args` surface. - """ - parser.add_argument("--host", type=str, default="0.0.0.0", help="Host to bind to") - parser.add_argument( - "--port", "-p", type=int, default=None, - help="Port to bind to (default: 4000, or server.port from secrets.json)", - ) - parser.add_argument( - "--inbound", - type=str, - default=None, - choices=[f.value for f in InboundFormat], - help="Inbound API format: openai, anthropic, or both", - ) - parser.add_argument("--reload", action="store_true", help="Enable auto-reload") - - -def add_common_args( - parser: argparse.ArgumentParser, - *, - include_model: bool = True, -) -> None: - """Register arguments shared by every subcommand. - - Adds the transport args (:func:`add_transport_args`) plus backend - credentials (``--api-key`` / ``--base-url``), the optional model - override (``--model``), and uvicorn worker count (``--workers``). - - Args: - parser: Target parser (subcommand parser, typically). - include_model: Whether to expose ``--model`` for backend model - override. Subcommands that don't yet support a model-override - request processor should pass ``False``. - """ - add_transport_args(parser) - parser.add_argument( - "--api-key", type=str, default=None, - help="API key for the backend LLM (falls back to env vars / secrets.json)", - ) - parser.add_argument( - "--base-url", type=str, default=None, - help="Base URL for the backend LLM API", - ) - if include_model: - parser.add_argument( - "--model", type=str, default=None, - help="Model name override (replaces model from incoming requests)", - ) - parser.add_argument( - "--workers", "-w", type=int, - default=int(os.environ.get("SWITCHYARD_WORKERS", "1")), - help="Number of uvicorn worker processes (default: 1, or SWITCHYARD_WORKERS env var)", - ) - - -def resolve_rl_log_dir(args: argparse.Namespace) -> Path | None: - """Resolve the RL trace-log directory from the global rl-logging flags. - - Returns ``None`` unless the global ``--enable-rl-logging`` flag is set, in - which case the directory is ``--rl-log-dir`` (default ``./rl_data``). Shared - by the ``launch`` and ``serve`` entry points. - """ - if not getattr(args, "enable_rl_logging", False): - return None - return Path(getattr(args, "rl_log_dir", None) or "./rl_data").expanduser() - - -# --------------------------------------------------------------------------- -# Credential resolution -# --------------------------------------------------------------------------- - - -def resolve_credentials_from_env( - args: argparse.Namespace, - *, - check_anthropic: bool = False, -) -> tuple[str | None, str | None]: - """Return ``(api_key, base_url)`` resolved from ``args`` + env vars. - - Resolution order for *api_key*: - 1. ``args.api_key`` - 2. ``ANTHROPIC_API_KEY`` (only when ``check_anthropic=True``) - 3. ``OPENAI_API_KEY`` - - Resolution order for *base_url*: - 1. ``args.base_url`` - 2. ``OPENAI_BASE_URL`` - 3. ``OPENAI_API_BASE`` - - Does not read ``secrets.json`` — use :func:`resolve_config_with_secrets` - when secrets-file fallback is also needed. - """ - if check_anthropic: - api_key = ( - args.api_key - or os.environ.get("ANTHROPIC_API_KEY") - or os.environ.get("OPENAI_API_KEY") - ) - else: - api_key = args.api_key or os.environ.get("OPENAI_API_KEY") - base_url = ( - args.base_url - or os.environ.get("OPENAI_BASE_URL") - or os.environ.get("OPENAI_API_BASE") - ) - return api_key, base_url - - -def resolve_config_with_secrets( - args: argparse.Namespace, - *, - api_key_env_vars: tuple[str, ...] = ("OPENAI_API_KEY",), - base_url_env_vars: tuple[str, ...] = (), - base_url_arg: str = "api_base", - secrets_section_priority: tuple[str, ...] = (), -) -> tuple[str | None, str | None]: - """Resolve ``(api_key, base_url)`` from CLI → env → ``secrets.json``. - - Also mutates ``args.port`` in place when the CLI didn't set it and - ``secrets.json`` has a ``server.port`` entry. - - Resolution order: - * api_key: ``args.api_key`` → *api_key_env_vars* (in order) → - secrets sections (``secrets_section_priority`` in order, then - first provider as a final fallback) - * base_url: ``getattr(args, base_url_arg)`` → *base_url_env_vars* - (in order) → same secrets-section traversal as api_key - * port: ``args.port`` (if set) else ``secrets["server"]["port"]`` - - The secrets-section traversal mirrors the pattern used by the - legacy subcommands: a priority-list of sections (e.g. - ``("nvidia",)``) is checked first, and if none of them yielded an - api_key we fall all the way back to the first provider entry — - whichever section happens to come first in ``secrets.json``. - - Args: - args: Parsed CLI namespace. Must have ``api_key`` and - ``port`` attributes; ``base_url_arg`` must also resolve. - api_key_env_vars: Env var names checked in order for the - api_key fallback. - base_url_env_vars: Env var names checked in order for the - base_url fallback. Empty means "CLI + secrets only". - base_url_arg: Attribute name on ``args`` that holds the CLI - base URL — typically ``"api_base"`` (for ``--api-base``) - or ``"base_url"`` (for ``--base-url``). - secrets_section_priority: Top-level section names in - ``secrets.json`` to consult first (e.g. ``("nvidia",)``). - Empty defaults to "first provider only". - - Returns: - ``(api_key, base_url)`` — either element may be ``None``. - """ - secrets = load_secrets() - secrets_api_key: str | None = None - secrets_base_url: str | None = None - secrets_port: int | None = None - - if secrets: - for section_name in secrets_section_priority: - section = secrets.get(section_name, {}) - if isinstance(section, dict): - api_key_value = section.get("api_key") - if secrets_api_key is None and isinstance(api_key_value, str): - secrets_api_key = api_key_value - base_url_value = section.get("base_url") - if secrets_base_url is None and isinstance(base_url_value, str): - secrets_base_url = base_url_value - - # Mirror the legacy behavior: fall back to the first provider - # section only when the priority sections didn't yield an api_key. - if not secrets_api_key: - first_provider: Mapping[str, object] = next(iter(secrets.values()), {}) - if isinstance(first_provider, dict): - api_key_value = first_provider.get("api_key") - if secrets_api_key is None and isinstance(api_key_value, str): - secrets_api_key = api_key_value - base_url_value = first_provider.get("base_url") - if secrets_base_url is None and isinstance(base_url_value, str): - secrets_base_url = base_url_value - - server_section = secrets.get("server", {}) - if isinstance(server_section, dict): - port_value = server_section.get("port") - if isinstance(port_value, (int, str)): - secrets_port = int(port_value) - - # api_key: CLI > env vars (in order) > secrets - api_key: str | None = args.api_key - for env_var in api_key_env_vars: - if api_key: - break - api_key = os.environ.get(env_var) - api_key = api_key or secrets_api_key - - # base_url: CLI > env vars (in order) > secrets - base_url: str | None = getattr(args, base_url_arg, None) - for env_var in base_url_env_vars: - if base_url: - break - base_url = os.environ.get(env_var) - base_url = base_url or secrets_base_url - - if args.port is None and secrets_port is not None: - args.port = secrets_port - - return api_key, base_url - - -def ensure_openai_api_key_env(api_key: str | None) -> None: - """Set ``OPENAI_API_KEY`` env var iff unset and *api_key* is provided. - - Some downstream libraries read - ``OPENAI_API_KEY`` directly from the environment, so subcommands - that resolve credentials from CLI / secrets.json pin them into the - environment for those libraries to see. - """ - if api_key and not os.environ.get("OPENAI_API_KEY"): - os.environ["OPENAI_API_KEY"] = api_key - - -def resolve_port(default: int = 4000) -> int: - """Resolve the server port from ``secrets.json`` with a fixed fallback.""" - server_section = load_secrets().get("server", {}) - if isinstance(server_section, Mapping): - value = server_section.get("port") - if isinstance(value, int): - return value - if isinstance(value, str): - try: - return int(value) - except ValueError: - logger.warning("Ignoring non-integer secrets.json server.port=%r", value) - return default - - -# --------------------------------------------------------------------------- -# Build + serve -# --------------------------------------------------------------------------- - - -def build_and_serve( - args: argparse.Namespace, - switchyard: Switchyard | RouteTable, - *, - inbound_default: str = "openai", - disable_backend_streaming: bool = False, - extra_endpoints: list[NemoSwitchyardEndpoint] | None = None, - strategy_summary: str | None = None, -) -> None: - """Wire a Switchyard runtime object into a FastAPI app and serve it. - - Builds the app via :func:`~switchyard.server.switchyard_app.build_switchyard_app` - (which registers all three inbound formats — OpenAI Chat, Anthropic Messages, and - OpenAI Responses API), optionally appends *extra_endpoints*, then starts uvicorn. - - Expected attributes on *args*: - * ``host`` (str), ``port`` (int | None), ``inbound`` (str | None), - ``reload`` (bool), ``workers`` (int, optional; defaults to 1). - - Args: - args: Parsed CLI namespace. - switchyard: Already-built chain or table to serve. - inbound_default: Unused — always registers all inbound formats. - disable_backend_streaming: Unused — kept for signature compatibility. - extra_endpoints: Additional endpoint modules to register after the defaults. - """ - del inbound_default, disable_backend_streaming - - import threading - - import uvicorn - - from switchyard.server.switchyard_app import build_switchyard_app - - app = build_switchyard_app(switchyard) - if extra_endpoints: - for endpoint in extra_endpoints: - endpoint.register(app) - - port = args.port if isinstance(args.port, int) else resolve_port() - - def _print_banner() -> None: - from switchyard.cli.launchers.launcher_runtime import ( - print_ready_banner, - wait_for_proxy_ready, - ) - from switchyard.lib.route_table import RouteTable as _RT - if not wait_for_proxy_ready(port, timeout_s=15.0): - return - table = switchyard if isinstance(switchyard, _RT) else None - default_model = table.default_model() if table else None - print_ready_banner( - port=port, - display_model=default_model or "switchyard", - strategy_summary=strategy_summary, - routes=table.registered_models() if table else None, - default_route=default_model, - ) - - threading.Thread(target=_print_banner, daemon=True).start() - - workers = getattr(args, "workers", 1) - uvicorn.run( - app, - host=args.host, - port=port, - reload=args.reload, - workers=workers, - ) diff --git a/switchyard/server/switchyard_app.py b/switchyard/server/switchyard_app.py deleted file mode 100644 index 404b77b95..000000000 --- a/switchyard/server/switchyard_app.py +++ /dev/null @@ -1,214 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Convenience factory for serving a ``Switchyard`` or model table. - -The default setup registers all three inbound endpoints so the app -can serve OpenAI Chat Completions, Anthropic Messages, and OpenAI -Responses API clients simultaneously — the chain handles format -translation internally. -""" - -from __future__ import annotations - -import inspect -from collections.abc import AsyncIterator, Callable, Iterable -from contextlib import asynccontextmanager -from typing import TYPE_CHECKING, cast - -from fastapi import FastAPI, Request -from fastapi.exception_handlers import request_validation_exception_handler -from fastapi.exceptions import RequestValidationError -from fastapi.responses import Response - -from switchyard.lib.endpoints import outcome_metrics -from switchyard.lib.endpoints.anthropic_messages_endpoint import ( - AnthropicMessagesEndpoint, -) -from switchyard.lib.endpoints.base import Endpoint -from switchyard.lib.endpoints.dispatch import invalid_request_response -from switchyard.lib.endpoints.models_endpoint import ModelsEndpoint -from switchyard.lib.endpoints.openai_chat_endpoint import ( - OpenAIChatEndpoint, -) -from switchyard.lib.endpoints.responses_endpoint import ( - ResponsesEndpoint, -) -from switchyard_rust.core import SwitchyardInvalidRequestError - -#: Inbound LLM-serving paths whose response status codes feed the -#: client-side outcome counter. Other routes (/v1/models, /v1/stats, -#: /metrics, /health) are excluded — they don't represent router-served -#: LLM traffic and would distort the error-rate ratio. -_LLM_ROUTES: frozenset[str] = frozenset({ - "/v1/chat/completions", - "/v1/messages", - "/v1/responses", -}) - -if TYPE_CHECKING: - from switchyard.lib.route_table import SwitchyardApp - - -async def _run_lifecycle_method(component: object, method_name: str) -> None: - method = getattr(component, method_name, None) - if not callable(method): - return - result = method() - if inspect.isawaitable(result): - await result - - -async def _shutdown_components(components: Iterable[object]) -> None: - for component in components: - await _run_lifecycle_method(component, "shutdown") - - -def build_switchyard_app(switchyard: SwitchyardApp) -> FastAPI: - """Create a FastAPI app serving *switchyard* over all inbound formats. - - Registers three LLM endpoints plus a liveness probe: - - - ``POST /v1/chat/completions`` (OpenAI Chat Completions) - - ``POST /v1/messages`` (Anthropic Messages) - - ``POST /v1/responses`` (OpenAI Responses API) - - ``GET /v1/models`` (local model discovery) - - ``GET /health`` (liveness — always 200 when the process is up) - - All three LLM routes go through the same chain — translation - between wire formats is handled by ``TranslationEngine`` - inside the backend and ``TranslationEngine`` inside the - translator. - - Example:: - - from switchyard.lib.backends.llm_target import BackendFormat, LlmTarget - from switchyard.lib.backends import OpenAiNativeBackend - from switchyard_rust.translation import TranslationEngine - from switchyard.lib.switchyard import Switchyard - from switchyard import build_switchyard_app - import uvicorn - - switchyard = Switchyard( - backend=OpenAiNativeBackend(LlmTarget(model="gpt-4o", format=BackendFormat.OPENAI)), - translator=TranslationEngine(), - ) - uvicorn.run(build_switchyard_app(switchyard), port=4000) - """ - components = _switchyard_components(switchyard) - - @asynccontextmanager - async def _lifespan(_app: FastAPI) -> AsyncIterator[None]: - started: list[object] = [] - startup_complete = False - try: - for component in components: - await _run_lifecycle_method(component, "startup") - started.append(component) - startup_complete = True - yield - finally: - shutdown_targets = components if startup_complete else started - await _shutdown_components(reversed(shutdown_targets)) - - app = FastAPI(title="Switchyard", lifespan=_lifespan) - - @app.exception_handler(RequestValidationError) - async def _request_validation_error_handler( - request: Request, exc: RequestValidationError - ) -> Response: - """Map body validation failures on LLM routes to the Switchyard 400 envelope. - - FastAPI raises RequestValidationError for both malformed JSON and wrong - body types (e.g. array instead of object) when a route declares a typed - body parameter. Non-LLM routes and non-body errors fall through to - FastAPI's default 422 handler. - """ - if request.url.path in _LLM_ROUTES: - body_errors = [e for e in exc.errors() if e.get("loc", (None,))[0] == "body"] - if body_errors: - is_json_parse = any(e["type"] == "json_invalid" for e in body_errors) - message = ( - "Request body is not valid JSON" - if is_json_parse - else "Request body must be a JSON object" - ) - return invalid_request_response(message, code="invalid_body") - return await request_validation_exception_handler(request, exc) - - @app.exception_handler(SwitchyardInvalidRequestError) - async def _invalid_request_handler( - _request: Request, exc: SwitchyardInvalidRequestError - ) -> Response: - """Map request validation failures to the 400 envelope. - - ``ChatRequest.validate()`` (called by the inbound endpoints) raises - this when a body is structurally valid but semantically invalid. The - only such check today is a present-but-empty - ``messages`` array, so the envelope uses ``code="empty_messages"``; - revisit if more validations start sharing this error. - """ - return invalid_request_response(str(exc), code="empty_messages") - - app.state.switchyard = switchyard - - @app.middleware("http") - async def _record_client_outcome(request, call_next): # type: ignore[no-untyped-def] - """Tally every LLM-route response into the outcome counters. - - Runs after the endpoint produces its response, so it sees the - final status code regardless of how it was generated (success, - upstream-error passthrough, internal exception, model-not-found). - """ - response = await call_next(request) - if request.url.path in _LLM_ROUTES: - outcome_metrics.record_client_response(response.status_code) - return response - - # Route tables can contain hundreds of per-model components that contribute - # the same fixed-path endpoint. Registering each copy creates unreachable - # duplicate routes and recursively nests FastAPI lifespan contexts. - registered_once_endpoint_types: set[type[Endpoint]] = set() - for endpoint in [ - OpenAIChatEndpoint(), - AnthropicMessagesEndpoint(), - ResponsesEndpoint(), - ModelsEndpoint(), - ]: - endpoint.register(app) - if endpoint.register_once: - registered_once_endpoint_types.add(type(endpoint)) - - for component in components: - get_endpoint = getattr(component, "get_endpoint", None) - if not callable(get_endpoint): - continue - contributed = cast(Endpoint | None, get_endpoint()) - if contributed is None: - continue - endpoint_type = type(contributed) - if contributed.register_once and endpoint_type in registered_once_endpoint_types: - continue - contributed.register(app) - if contributed.register_once: - registered_once_endpoint_types.add(endpoint_type) - - @app.get("/health", include_in_schema=False) - async def _health() -> dict[str, str]: - return {"status": "ok"} - - return app - - -def _switchyard_components( - switchyard: SwitchyardApp, -) -> list[object]: - iter_components = getattr(switchyard, "iter_components", None) - if not callable(iter_components): - if callable(getattr(switchyard, "startup", None)) or callable( - getattr(switchyard, "shutdown", None) - ): - return [switchyard] - return [] - component_iter = cast(Callable[[], Iterable[object]], iter_components) - return list(component_iter()) diff --git a/switchyard/telemetry.py b/switchyard/telemetry.py deleted file mode 100644 index fd33635cc..000000000 --- a/switchyard/telemetry.py +++ /dev/null @@ -1,64 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Lightweight telemetry headers for outbound LLM SDK calls. - -Switchyard sends its package version as a single HTTP header on upstream LLM -requests. This lets downstream request logs attribute traffic to a Switchyard -version without a side channel or additional reporting infrastructure. - -Set ``SWITCHYARD_TELEMETRY_OPT_OUT=1`` to suppress the header. The legacy -``NEMO_SWITCHYARD_TELEMETRY_OPT_OUT`` name is also honored for compatibility -with pre-rename environments. -""" - -from __future__ import annotations - -import importlib.metadata -import logging -import os -from functools import lru_cache - -log = logging.getLogger(__name__) - -HEADER_NAME = "X-Switchyard-Version" -OPT_OUT_ENVVAR = "SWITCHYARD_TELEMETRY_OPT_OUT" -LEGACY_OPT_OUT_ENVVAR = "NEMO_SWITCHYARD_TELEMETRY_OPT_OUT" - -_FALSEY_VALUES = {"", "0", "false", "no"} - - -def _is_truthy_opt_out_value(value: str | None) -> bool: - """Return whether *value* should opt out of telemetry headers.""" - if value is None: - return False - return value.strip().lower() not in _FALSEY_VALUES - - -def _is_opted_out() -> bool: - """Return whether telemetry headers are disabled by environment.""" - return any( - _is_truthy_opt_out_value(os.environ.get(name)) - for name in (OPT_OUT_ENVVAR, LEGACY_OPT_OUT_ENVVAR) - ) - - -@lru_cache(maxsize=1) -def _get_version() -> str: - """Read the installed ``nemo-switchyard`` package version once.""" - try: - return importlib.metadata.version("nemo-switchyard") - except Exception: - log.debug("telemetry: could not read switchyard package version", exc_info=True) - return "unknown" - - -def get_telemetry_headers() -> dict[str, str]: - """Return headers to attach to outbound LLM SDK clients. - - Returns an empty dict when telemetry is opted out, so callers can pass or - merge the result unconditionally. - """ - if _is_opted_out(): - return {} - return {HEADER_NAME: _get_version()} diff --git a/switchyard_rust/__init__.py b/switchyard_rust/__init__.py index 2603ef237..a18dd8c97 100644 --- a/switchyard_rust/__init__.py +++ b/switchyard_rust/__init__.py @@ -1,155 +1,4 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Python companion wrappers for the ``crates/switchyard-py`` bindings.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -from switchyard_rust.translation import ( - TranslationEngine, - is_native_translation_available, -) - -if TYPE_CHECKING: - from switchyard_rust.components import AnthropicNativeBackend as AnthropicNativeBackend - from switchyard_rust.components import BackendFormat as BackendFormat - from switchyard_rust.components import EndpointConfig as EndpointConfig - from switchyard_rust.components import LlmTarget as LlmTarget - from switchyard_rust.components import LlmTargetBackend as LlmTargetBackend - from switchyard_rust.components import MultiLlmBackend as MultiLlmBackend - from switchyard_rust.components import OpenAiNativeBackend as OpenAiNativeBackend - from switchyard_rust.components import OpenAiPassthroughBackend as OpenAiPassthroughBackend - from switchyard_rust.components import ( - RandomRoutingProcessorConfig as RandomRoutingProcessorConfig, - ) - from switchyard_rust.components import StatsAccumulator as StatsAccumulator - from switchyard_rust.components import StatsLlmBackend as StatsLlmBackend - from switchyard_rust.components import StatsRequestProcessor as StatsRequestProcessor - from switchyard_rust.components import StatsResponseProcessor as StatsResponseProcessor - from switchyard_rust.core import ChatRequest as ChatRequest - from switchyard_rust.core import ChatRequestType as ChatRequestType - from switchyard_rust.core import ChatResponse as ChatResponse - from switchyard_rust.core import ChatResponseStream as ChatResponseStream - from switchyard_rust.core import ChatResponseType as ChatResponseType - from switchyard_rust.core import LLMBackend as LLMBackend - from switchyard_rust.core import ProxyContext as ProxyContext - from switchyard_rust.core import ProxyMetadata as ProxyMetadata - from switchyard_rust.core import SwitchyardBackendError as SwitchyardBackendError - from switchyard_rust.core import SwitchyardConfigError as SwitchyardConfigError - from switchyard_rust.core import ( - SwitchyardDuplicateRegistrationError as SwitchyardDuplicateRegistrationError, - ) - from switchyard_rust.core import SwitchyardInvalidIdError as SwitchyardInvalidIdError - from switchyard_rust.core import SwitchyardModelNotFoundError as SwitchyardModelNotFoundError - from switchyard_rust.core import SwitchyardProcessorError as SwitchyardProcessorError - from switchyard_rust.core import SwitchyardRuntimeError as SwitchyardRuntimeError - from switchyard_rust.core import ( - SwitchyardUnsupportedRequestTypeError as SwitchyardUnsupportedRequestTypeError, - ) - from switchyard_rust.core import SwitchyardUpstreamError as SwitchyardUpstreamError - - -def __getattr__(name: str) -> object: - if name in { - "AnthropicNativeBackend", - "BackendFormat", - "EndpointConfig", - "LlmTarget", - "LlmTargetBackend", - "MultiLlmBackend", - "OpenAiNativeBackend", - "OpenAiPassthroughBackend", - "RandomRoutingProcessorConfig", - "StatsAccumulator", - "StatsLlmBackend", - "StatsRequestProcessor", - "StatsResponseProcessor", - }: - from switchyard_rust import components - - return getattr(components, name) - if name == "ChatRequest": - from switchyard_rust.core import ChatRequest - - return ChatRequest - if name == "ChatRequestType": - from switchyard_rust.core import ChatRequestType - - return ChatRequestType - if name == "ChatResponse": - from switchyard_rust.core import ChatResponse - - return ChatResponse - if name == "ChatResponseStream": - from switchyard_rust.core import ChatResponseStream - - return ChatResponseStream - if name == "ChatResponseType": - from switchyard_rust.core import ChatResponseType - - return ChatResponseType - if name == "LLMBackend": - from switchyard_rust.core import LLMBackend - - return LLMBackend - if name == "ProxyMetadata": - from switchyard_rust.core import ProxyMetadata - - return ProxyMetadata - if name == "ProxyContext": - from switchyard_rust.core import ProxyContext - - return ProxyContext - if name in { - "SwitchyardRuntimeError", - "SwitchyardConfigError", - "SwitchyardInvalidIdError", - "SwitchyardDuplicateRegistrationError", - "SwitchyardModelNotFoundError", - "SwitchyardUnsupportedRequestTypeError", - "SwitchyardProcessorError", - "SwitchyardBackendError", - "SwitchyardUpstreamError", - }: - from switchyard_rust import core - - return getattr(core, name) - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - - -__all__ = [ - "AnthropicNativeBackend", - "BackendFormat", - "ChatRequest", - "ChatRequestType", - "ChatResponse", - "ChatResponseStream", - "ChatResponseType", - "EndpointConfig", - "LLMBackend", - "LlmTarget", - "LlmTargetBackend", - "MultiLlmBackend", - "OpenAiNativeBackend", - "OpenAiPassthroughBackend", - "ProxyMetadata", - "ProxyContext", - "RandomRoutingProcessorConfig", - "StatsAccumulator", - "StatsLlmBackend", - "StatsRequestProcessor", - "StatsResponseProcessor", - "SwitchyardBackendError", - "SwitchyardConfigError", - "SwitchyardDuplicateRegistrationError", - "SwitchyardInvalidIdError", - "SwitchyardModelNotFoundError", - "SwitchyardProcessorError", - "SwitchyardRuntimeError", - "SwitchyardUnsupportedRequestTypeError", - "SwitchyardUpstreamError", - "TranslationEngine", - "is_native_translation_available", -] +"""Python wrappers for Switchyard's native libsy and server bindings.""" diff --git a/switchyard_rust/_native.py b/switchyard_rust/_native.py new file mode 100644 index 000000000..3ddd376e3 --- /dev/null +++ b/switchyard_rust/_native.py @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Load the private native extension shared by the public wrappers.""" + +from __future__ import annotations + +import importlib +import importlib.metadata +import os +from typing import Any + + +def _ensure_switchyard_version_env() -> None: + if os.environ.get("SWITCHYARD_VERSION", "").strip(): + return + try: + version = importlib.metadata.version("nemo-switchyard") + except importlib.metadata.PackageNotFoundError: + return + if version.strip(): + os.environ["SWITCHYARD_VERSION"] = version + + +def load_native() -> Any: + """Load the extension or report how to repair a source checkout.""" + _ensure_switchyard_version_env() + try: + return importlib.import_module("switchyard_rust._switchyard_rust") + except ImportError as exc: # pragma: no cover - broken install guard + raise RuntimeError( + "The Switchyard native extension is required. Run `uv run maturin develop` " + "or install a built switchyard wheel." + ) from exc diff --git a/switchyard_rust/components.py b/switchyard_rust/components.py deleted file mode 100644 index e34cadea6..000000000 --- a/switchyard_rust/components.py +++ /dev/null @@ -1,90 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Direct bindings for Rust-owned Switchyard components.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Any - -from switchyard_rust.core import _load_native - -_COMPONENT_EXPORTS = frozenset( - { - "AnthropicNativeBackend", - "BackendFormat", - "DimensionCollector", - "EndpointConfig", - "LlmTarget", - "LlmTargetBackend", - "MultiLlmBackend", - "OpenAiNativeBackend", - "OpenAiPassthroughBackend", - "PickOutcome", - "RandomRoutingProcessorConfig", - "ResponseFlag", - "ResponseSignalCollector", - "ResponseSignals", - "StatsAccumulator", - "StatsLlmBackend", - "StatsRequestProcessor", - "StatsResponseProcessor", - "ToolResultSignal", - "extract_response_signals", - "get_response_signals", - "get_tool_result_signal", - "set_stats_route_label", - "stage_pick_tier", - "stage_score_signal", - } -) -_BACKEND_EXPORTS = frozenset( - { - "AnthropicNativeBackend", - "MultiLlmBackend", - "OpenAiNativeBackend", - "OpenAiPassthroughBackend", - "StatsLlmBackend", - } -) - -if TYPE_CHECKING: - AnthropicNativeBackend: type[Any] - BackendFormat: type[Any] - DimensionCollector: type[Any] - EndpointConfig: type[Any] - LlmTarget: type[Any] - LlmTargetBackend: type[Any] - MultiLlmBackend: type[Any] - OpenAiNativeBackend: type[Any] - OpenAiPassthroughBackend: type[Any] - PickOutcome: type[Any] - RandomRoutingProcessorConfig: type[Any] - ResponseFlag: type[Any] - ResponseSignalCollector: type[Any] - ResponseSignals: type[Any] - StatsAccumulator: type[Any] - StatsLlmBackend: type[Any] - StatsRequestProcessor: type[Any] - StatsResponseProcessor: type[Any] - ToolResultSignal: type[Any] - extract_response_signals: Any - get_response_signals: Any - get_tool_result_signal: Any - set_stats_route_label: Any - stage_pick_tier: Any - stage_score_signal: Any - - -def __getattr__(name: str) -> object: - if name in _COMPONENT_EXPORTS: - value = getattr(_load_native(), name) - if name in _BACKEND_EXPORTS: - from switchyard_rust.core import LLMBackend - - LLMBackend.register(value) - return value - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - - -__all__ = sorted(_COMPONENT_EXPORTS) diff --git a/switchyard_rust/components.pyi b/switchyard_rust/components.pyi deleted file mode 100644 index 36714b8fc..000000000 --- a/switchyard_rust/components.pyi +++ /dev/null @@ -1,270 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -from collections.abc import Iterable -from typing import Any, ClassVar - -from switchyard_rust.core import ( - ChatRequestType, - LLMBackend, - ProxyContext, -) - -class BackendFormat: - AUTO: ClassVar[BackendFormat] - OPENAI: ClassVar[BackendFormat] - RESPONSES: ClassVar[BackendFormat] - ANTHROPIC: ClassVar[BackendFormat] - - value: str - - def __init__(self, value: str = "auto") -> None: ... - - -class EndpointConfig: - base_url: str | None - api_key: str | None - timeout_secs: float | None - - def __init__( - self, - base_url: str | None = None, - api_key: str | None = None, - timeout_secs: float | None = None, - ) -> None: ... - def to_dict(self) -> dict[str, Any]: ... - - -class LlmTarget: - id: str - model: str - format: BackendFormat - backend_format: BackendFormat - endpoint: EndpointConfig - base_url: str | None - api_key: str | None - timeout: float | None - extra_body: dict[str, Any] | None - extra_headers: dict[str, str] - - def __init__( - self, - id: str | None = None, - model: str | None = None, - format: BackendFormat | str | None = None, - backend_format: BackendFormat | str | None = None, - endpoint: EndpointConfig | dict[str, Any] | None = None, - base_url: str | None = None, - api_key: str | None = None, - timeout_secs: float | None = None, - timeout: float | None = None, - extra_body: dict[str, Any] | None = None, - extra_headers: dict[str, str] | None = None, - ) -> None: ... - def to_dict(self) -> dict[str, Any]: ... - def model_dump(self) -> dict[str, Any]: ... - - -class RandomRoutingProcessorConfig: - strong: LlmTarget - weak: LlmTarget - strong_probability: float - rng_seed: int | None - - def __init__( - self, - strong: LlmTarget, - weak: LlmTarget, - strong_probability: float = 0.5, - rng_seed: int | None = None, - ) -> None: ... - def to_dict(self) -> dict[str, Any]: ... - - -class StatsAccumulator: - def __init__(self) -> None: ... - async def record_success( - self, - model: str, - backend_latency_ms: float | None = None, - tier: str | None = None, - ) -> None: ... - async def record_error(self, model: str, tier: str | None = None) -> None: ... - async def record_usage( - self, - model: str, - prompt_tokens: int = 0, - completion_tokens: int = 0, - cached_tokens: int = 0, - cache_creation_tokens: int = 0, - reasoning_tokens: int = 0, - total_latency_ms: float | None = None, - routing_overhead_ms: float | None = None, - tier: str | None = None, - ) -> None: ... - async def record_classifier_usage( - self, - model: str, - prompt_tokens: int = 0, - completion_tokens: int = 0, - cached_tokens: int = 0, - cache_creation_tokens: int = 0, - reasoning_tokens: int = 0, - latency_ms: float | None = None, - ) -> None: ... - async def record_classifier_error(self, model: str) -> None: ... - async def snapshot(self) -> dict[str, Any]: ... - def snapshot_sync(self) -> dict[str, Any]: ... - async def reset(self) -> None: ... - def reset_sync(self) -> None: ... - - -def set_stats_route_label(ctx: Any, label: str) -> None: ... - - -class LlmTargetBackend: - target: LlmTarget - - def __init__(self, target: LlmTarget, backend: LLMBackend) -> None: ... - - -class OpenAiNativeBackend(LLMBackend): - target: LlmTarget - - def __init__(self, target: LlmTarget) -> None: ... - - -class OpenAiPassthroughBackend(LLMBackend): - endpoint: EndpointConfig - - def __init__( - self, - endpoint: EndpointConfig | dict[str, Any] | None = None, - api_key: str | None = None, - base_url: str | None = None, - timeout_secs: float | None = None, - timeout: float | None = None, - ) -> None: ... - - -class AnthropicNativeBackend(LLMBackend): - target: LlmTarget - - def __init__(self, target: LlmTarget) -> None: ... - - -class MultiLlmBackend(LLMBackend): - def __init__( - self, - targets: Iterable[LlmTargetBackend | tuple[LlmTarget, LLMBackend]], - supported_request_types: Iterable[ChatRequestType | str] | None = None, - default_target_id: str | None = None, - ) -> None: ... - def target_ids(self) -> list[str]: ... - def default_target_id(self) -> str | None: ... - - -class StatsLlmBackend(LLMBackend): - accumulator: StatsAccumulator - - def __init__(self, inner: LLMBackend, accumulator: StatsAccumulator) -> None: ... - - -class StatsRequestProcessor: - def __init__(self) -> None: ... - async def process(self, ctx: ProxyContext, request: Any) -> Any: ... - async def startup(self) -> None: ... - async def shutdown(self) -> None: ... - - -class StatsResponseProcessor: - accumulator: StatsAccumulator - - def __init__(self, accumulator: StatsAccumulator) -> None: ... - async def process(self, ctx: ProxyContext, response: Any) -> Any: ... - async def startup(self) -> None: ... - async def shutdown(self) -> None: ... - def get_endpoint(self) -> object: ... - - -class DimensionCollector: - def __init__( - self, - *, - recent_window: int | None = None, - ) -> None: ... - async def process(self, ctx: ProxyContext, request: Any) -> Any: ... - async def startup(self) -> None: ... - async def shutdown(self) -> None: ... - - -class ToolResultSignal: - severity: float - turn_depth: int - write_count: int - edit_count: int - read_count: int - todowrite_count: int - recent_write_count: int - recent_edit_count: int - recent_read_count: int - recent_todowrite_count: int - pure_bash_streak: int - no_error_streak: int - tests_passed: bool - compacted: bool - - -def get_tool_result_signal(ctx: ProxyContext) -> ToolResultSignal | None: ... - - -class PickOutcome: - resolved: bool - tier: str | None - source: str | None - default_tier: str - score: float - confidence: float | None - - -def stage_pick_tier( - signal: ToolResultSignal, picker_mode: str, confidence_threshold: float -) -> PickOutcome: ... - - -def stage_score_signal(signal: ToolResultSignal) -> tuple[float, float]: ... - - -class ResponseFlag: - MALFORMED_TOOL_CALL_JSON: ClassVar[ResponseFlag] - EMPTY_RESPONSE: ClassVar[ResponseFlag] - TRUNCATED_COMPLETION: ClassVar[ResponseFlag] - MISSING_REQUIRED_ARGS: ClassVar[ResponseFlag] - - def __eq__(self, other: object) -> bool: ... - def __hash__(self) -> int: ... - - -class ResponseSignals: - flags: list[ResponseFlag] - - def has_failures(self) -> bool: ... - def contains(self, flag: ResponseFlag) -> bool: ... - - -class ResponseSignalCollector: - def __init__(self) -> None: ... - async def process(self, ctx: ProxyContext, response: Any) -> Any: ... - async def startup(self) -> None: ... - async def shutdown(self) -> None: ... - - -def get_response_signals(ctx: ProxyContext) -> ResponseSignals | None: ... - - -def extract_response_signals(body: dict[str, Any] | None) -> ResponseSignals: ... - - -__all__: list[str] diff --git a/switchyard_rust/core.py b/switchyard_rust/core.py deleted file mode 100644 index f9ed33bfd..000000000 --- a/switchyard_rust/core.py +++ /dev/null @@ -1,793 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Python core values and compatibility chain wrappers.""" - -from __future__ import annotations - -import importlib -import importlib.metadata -import json -import os -from abc import ABCMeta -from collections.abc import Iterable, Mapping -from enum import Enum -from typing import TYPE_CHECKING, Any, ClassVar, Protocol, TypeAlias, cast - -JsonScalar: TypeAlias = bool | int | float | str | None -JsonValue: TypeAlias = JsonScalar | list["JsonValue"] | dict[str, "JsonValue"] - - -class ChatRequestType(Enum): - """Wire format carried by a :class:`ChatRequest`.""" - - OPENAI_CHAT = "openai_chat" - OPENAI_RESPONSES = "openai_responses" - ANTHROPIC = "anthropic" - - def __str__(self) -> str: - return self.value - - -class ChatResponseType(Enum): - """Wire format and delivery mode carried by a :class:`ChatResponse`.""" - - OPENAI_COMPLETION = "openai_completion" - OPENAI_STREAM = "openai_stream" - OPENAI_RESPONSES_COMPLETION = "openai_responses_completion" - OPENAI_RESPONSES_STREAM = "openai_responses_stream" - ANTHROPIC_COMPLETION = "anthropic_completion" - ANTHROPIC_STREAM = "anthropic_stream" - - def __str__(self) -> str: - return self.value - - -def _json_copy(value: object) -> JsonValue: - """Normalize an object to an owned JSON value.""" - model_dump = getattr(value, "model_dump", None) - if callable(model_dump): - value = model_dump(mode="json", exclude_none=True) - else: - to_dict = getattr(value, "to_dict", None) - if callable(to_dict): - value = to_dict() - try: - return cast(JsonValue, json.loads(json.dumps(value, allow_nan=False))) - except (TypeError, ValueError) as error: - raise ValueError(str(error)) from error - - -class ChatRequest: - """Owned provider request body with an explicit wire format.""" - - def __init__(self, request_type: ChatRequestType, body: object) -> None: - self.request_type = request_type - self._body = _json_copy(body) - - @classmethod - def openai_chat(cls, body: object) -> ChatRequest: - """Build an OpenAI Chat Completions request.""" - return cls(ChatRequestType.OPENAI_CHAT, body) - - @classmethod - def openai_responses(cls, body: object) -> ChatRequest: - """Build an OpenAI Responses request.""" - return cls(ChatRequestType.OPENAI_RESPONSES, body) - - @classmethod - def anthropic(cls, body: object) -> ChatRequest: - """Build an Anthropic Messages request.""" - return cls(ChatRequestType.ANTHROPIC, body) - - @property - def body(self) -> Any: - """Return an owned copy of the request body.""" - return _json_copy(self._body) - - @property - def model(self) -> str | None: - """Return the request model when it is a string.""" - if isinstance(self._body, dict): - model = self._body.get("model") - return model if isinstance(model, str) else None - return None - - def validate(self) -> None: - """Reject semantically invalid message-based requests.""" - if self.request_type in {ChatRequestType.OPENAI_CHAT, ChatRequestType.ANTHROPIC}: - if isinstance(self._body, dict) and self._body.get("messages") == []: - raise _load_native().SwitchyardInvalidRequestError( - "messages must be a non-empty array" - ) - - def set_model(self, model: str) -> None: - """Set the request model, replacing a malformed non-object body.""" - if not isinstance(self._body, dict): - self._body = {} - self._body["model"] = model - - def replace_body(self, body: object) -> None: - """Replace the body without changing its wire format.""" - self._body = _json_copy(body) - - def to_body(self) -> JsonValue: - """Return an owned copy of the request body.""" - return _json_copy(self._body) - - def __repr__(self) -> str: - model = f", model={self.model!r}" if self.model is not None else "" - return f"ChatRequest(request_type={self.request_type.value!r}{model})" - - -class ChatResponseStream: - """Single-consumer async response stream with callback transforms.""" - - def __init__(self, source: object) -> None: - self._native = _load_native()._NativeChatResponseStream(source) - - @classmethod - def _from_native(cls, stream: Any) -> ChatResponseStream: - instance = cls.__new__(cls) - instance._native = stream - return instance - - def tap(self, callback: object) -> ChatResponseStream: - """Observe each event before mapping.""" - self._native.tap(callback) - return self - - def map(self, callback: object) -> ChatResponseStream: - """Transform each event in registration order.""" - self._native.map(callback) - return self - - def on_complete(self, callback: object) -> ChatResponseStream: - """Run a callback once after normal stream exhaustion.""" - self._native.on_complete(callback) - return self - - def __aiter__(self) -> ChatResponseStream: - self._native.__aiter__() - return self - - async def __anext__(self) -> Any: - return await self._native.__anext__() - - async def aclose(self) -> None: - """Close the upstream source and release its connection.""" - await self._native.aclose() - - def __repr__(self) -> str: - return "ChatResponseStream()" - - -class ChatResponse: - """Owned buffered response or live response stream.""" - - def __init__( - self, - response_type: ChatResponseType, - *, - body: object | None = None, - stream: object | None = None, - ) -> None: - self.response_type = response_type - self._body: JsonValue | None - self._stream: ChatResponseStream | None - if response_type.value.endswith("_stream"): - if stream is None: - raise TypeError("streaming ChatResponse requires a stream") - self._stream = ( - stream if isinstance(stream, ChatResponseStream) else ChatResponseStream(stream) - ) - self._body = None - else: - self._body = _json_copy(body) - self._stream = None - - @classmethod - def openai_completion(cls, body: object) -> ChatResponse: - """Build a buffered OpenAI Chat Completions response.""" - return cls(ChatResponseType.OPENAI_COMPLETION, body=body) - - @classmethod - def openai_stream(cls, stream: object) -> ChatResponse: - """Build a streaming OpenAI Chat Completions response.""" - return cls(ChatResponseType.OPENAI_STREAM, stream=stream) - - @classmethod - def openai_responses_completion(cls, body: object) -> ChatResponse: - """Build a buffered OpenAI Responses response.""" - return cls(ChatResponseType.OPENAI_RESPONSES_COMPLETION, body=body) - - @classmethod - def openai_responses_stream(cls, stream: object) -> ChatResponse: - """Build a streaming OpenAI Responses response.""" - return cls(ChatResponseType.OPENAI_RESPONSES_STREAM, stream=stream) - - @classmethod - def anthropic_completion(cls, body: object) -> ChatResponse: - """Build a buffered Anthropic response.""" - return cls(ChatResponseType.ANTHROPIC_COMPLETION, body=body) - - @classmethod - def anthropic_stream(cls, stream: object) -> ChatResponse: - """Build a streaming Anthropic response.""" - return cls(ChatResponseType.ANTHROPIC_STREAM, stream=stream) - - @property - def body(self) -> Any: - """Return an owned buffered body.""" - if self._stream is not None: - raise AttributeError("streaming ChatResponse values do not have a buffered body") - return _json_copy(self._body) - - @property - def stream(self) -> ChatResponseStream: - """Return the live stream.""" - if self._stream is None: - raise AttributeError("buffered ChatResponse values do not have a stream") - return self._stream - - def replace_body(self, body: object) -> None: - """Replace a buffered body without changing its wire shape.""" - if self._stream is not None: - raise ValueError("streaming ChatResponse values do not have a replaceable body") - self._body = _json_copy(body) - - def to_body(self) -> JsonValue: - """Return an owned buffered body.""" - if self._stream is not None: - raise AttributeError("streaming ChatResponse values do not have a buffered body") - return _json_copy(self._body) - - def __repr__(self) -> str: - return f"ChatResponse(response_type={self.response_type.value!r})" - - -class ProxyMetadata(dict[str, Any]): - """Mutable per-request metadata owned by Python.""" - - -class ProxyContext: - """Per-request state shared by Python and native components.""" - - def __init__( - self, - metadata: Mapping[str, Any] | None = None, - request_id: str | None = None, - ) -> None: - self._native = _load_native()._NativeProxyContext(metadata, request_id) - self.metadata = ProxyMetadata(metadata or {}) - - @property - def request_id(self) -> str | None: - return cast(str | None, self._native.request_id) - - @request_id.setter - def request_id(self, value: str | None) -> None: - self._native.request_id = value - - @property - def inbound_format(self) -> ChatRequestType | None: - value = self._native.inbound_format - return request_type_enum(value) if value is not None else None - - @inbound_format.setter - def inbound_format(self, value: object | None) -> None: - self._native.inbound_format = value - - @property - def selected_model(self) -> str | None: - return cast(str | None, self._native.selected_model) - - @selected_model.setter - def selected_model(self, value: str | None) -> None: - self._native.selected_model = value - - @property - def selected_target(self) -> str | None: - return cast(str | None, self._native.selected_target) - - @selected_target.setter - def selected_target(self, value: str | None) -> None: - self._native.selected_target = value - - @property - def evicted_targets(self) -> list[str] | None: - return cast(list[str] | None, self._native.evicted_targets) - - @evicted_targets.setter - def evicted_targets(self, value: list[str] | None) -> None: - self._native.evicted_targets = value - - @property - def backend_call_latency_ms(self) -> float | None: - return cast(float | None, self._native.backend_call_latency_ms) - - @backend_call_latency_ms.setter - def backend_call_latency_ms(self, value: float | None) -> None: - self._native.backend_call_latency_ms = value - - def __repr__(self) -> str: - return repr(self._native) - - -class LLMBackend(metaclass=ABCMeta): # noqa: B024 - """Base class for Python and registered native LLM backends.""" - - def __new__(cls, *args: object, **kwargs: object) -> LLMBackend: - if cls is LLMBackend: - raise TypeError("can't instantiate abstract role LLMBackend") - return super().__new__(cls) - - @property - def supported_request_types(self) -> list[ChatRequestType]: - """Return request formats accepted by this backend.""" - raise NotImplementedError("LLMBackend.supported_request_types must be implemented") - - async def startup(self) -> None: - """Start backend resources.""" - return None - - async def shutdown(self) -> None: - """Stop backend resources.""" - return None - - async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: - """Call the upstream model.""" - raise NotImplementedError("LLMBackend.call must be implemented") - - -class _NativeModule(Protocol): - _NativeChatResponseStream: Any - _NativeProxyContext: Any - is_subagent_request: Any - SwitchyardRuntimeError: type[RuntimeError] - SwitchyardConfigError: type[RuntimeError] - SwitchyardInvalidIdError: type[RuntimeError] - SwitchyardDuplicateRegistrationError: type[RuntimeError] - SwitchyardModelNotFoundError: type[RuntimeError] - SwitchyardUnsupportedRequestTypeError: type[RuntimeError] - SwitchyardInvalidRequestError: type[RuntimeError] - SwitchyardProcessorError: type[RuntimeError] - SwitchyardBackendError: type[RuntimeError] - SwitchyardUpstreamError: type[RuntimeError] - SwitchyardContextWindowExceededError: type[RuntimeError] - SwitchyardContextPoolExhaustedError: type[RuntimeError] - - -def _ensure_switchyard_version_env() -> None: - if os.environ.get("SWITCHYARD_VERSION", "").strip(): - return - for distribution in ("switchyard", "nemo-switchyard"): - try: - version = importlib.metadata.version(distribution) - except importlib.metadata.PackageNotFoundError: - continue - if version.strip(): - os.environ["SWITCHYARD_VERSION"] = version - return - - -def _load_native() -> _NativeModule: - _ensure_switchyard_version_env() - try: - return cast(_NativeModule, importlib.import_module("switchyard_rust._switchyard_rust")) - except ImportError as exc: # pragma: no cover - broken install guard - raise RuntimeError( - "The Switchyard native extension is required. Run `uv run maturin develop` " - "or install a built switchyard wheel." - ) from exc - - -if TYPE_CHECKING: - class SwitchyardRuntimeError(RuntimeError): - """Base class for native Switchyard runtime errors.""" - - class SwitchyardConfigError(SwitchyardRuntimeError): - """Raised for invalid Switchyard configuration.""" - - class SwitchyardInvalidIdError(SwitchyardRuntimeError): - """Raised when a Switchyard identifier is invalid.""" - - class SwitchyardDuplicateRegistrationError(SwitchyardRuntimeError): - """Raised when a registry receives a duplicate ID.""" - - class SwitchyardModelNotFoundError(SwitchyardRuntimeError): - """Raised when a route table cannot find a model.""" - - class SwitchyardUnsupportedRequestTypeError(SwitchyardRuntimeError): - """Raised when a component rejects a request format.""" - - class SwitchyardInvalidRequestError(SwitchyardRuntimeError): - """Raised when a request body fails semantic validation.""" - - class SwitchyardProcessorError(SwitchyardRuntimeError): - """Raised when a processor fails.""" - - class SwitchyardBackendError(SwitchyardRuntimeError): - """Raised when a backend fails.""" - - class SwitchyardUpstreamError(SwitchyardRuntimeError): - """Raised when an upstream call fails.""" - - status_code: int - body: str - - class SwitchyardContextWindowExceededError(SwitchyardBackendError): - """Raised when an upstream request exceeds the context window.""" - - class SwitchyardContextPoolExhaustedError(SwitchyardBackendError): - """Raised when every attempted routing target was evicted.""" -else: - SwitchyardRuntimeError = _load_native().SwitchyardRuntimeError - SwitchyardConfigError = _load_native().SwitchyardConfigError - SwitchyardInvalidIdError = _load_native().SwitchyardInvalidIdError - SwitchyardDuplicateRegistrationError = _load_native().SwitchyardDuplicateRegistrationError - SwitchyardModelNotFoundError = _load_native().SwitchyardModelNotFoundError - SwitchyardUnsupportedRequestTypeError = _load_native().SwitchyardUnsupportedRequestTypeError - SwitchyardInvalidRequestError = _load_native().SwitchyardInvalidRequestError - SwitchyardProcessorError = _load_native().SwitchyardProcessorError - SwitchyardBackendError = _load_native().SwitchyardBackendError - SwitchyardUpstreamError = _load_native().SwitchyardUpstreamError - SwitchyardContextWindowExceededError = _load_native().SwitchyardContextWindowExceededError - SwitchyardContextPoolExhaustedError = _load_native().SwitchyardContextPoolExhaustedError - - -def is_subagent_request(headers: Mapping[str, str]) -> bool: - """Return whether canonical protocol metadata marks delegated agent work.""" - return bool(_load_native().is_subagent_request(headers)) - - -def _role_type_name(value: object) -> str: - """Return a stable display name for role validation errors.""" - return type(value).__name__ - - -def _processor_error(error: BaseException) -> RuntimeError: - """Wrap processor failures in the public Switchyard processor error.""" - native = _load_native() - if isinstance(error, native.SwitchyardRuntimeError): - return error - return native.SwitchyardProcessorError(str(error)) - - -def _backend_error(error: BaseException) -> RuntimeError: - """Wrap backend failures in the public Switchyard backend error.""" - native = _load_native() - if isinstance(error, native.SwitchyardRuntimeError): - return error - return native.SwitchyardBackendError(str(error)) - - -class Switchyard: - """Python-only compatibility chain for the current FastAPI/recipe surface.""" - - state_key: ClassVar[str] = "switchyard" - - def __init__( - self, - *, - request_processors: Iterable[Any] | None = None, - backend: Any, - response_processors: Iterable[Any] | None = None, - translator: Any, - fallback_target_on_evict: str | None = None, - ) -> None: - if not isinstance(backend, LLMBackend): - actual = _role_type_name(backend) - raise TypeError(f"Switchyard backend must be LLMBackend, got {actual}") - self._request_components = tuple(request_processors or ()) - self._backend = backend - self._response_components = tuple(response_processors or ()) - self._translator = translator - self._fallback_target_on_evict = fallback_target_on_evict - - def iter_components(self) -> list[Any]: - """Return lifecycle components in startup order.""" - return [ - *self._request_components, - self._backend, - *self._response_components, - self._translator, - ] - - async def call( - self, - request: Any, - *, - ctx: Any | None = None, - ) -> Any: - """Run the compatibility chain and translate the final response.""" - context = ctx if ctx is not None else ProxyContext() - processed_request = await self._process_request_components(context, request) - try: - response = await self._call_backend_stage(context, processed_request) - except _load_native().SwitchyardContextWindowExceededError as error: - if self._fallback_target_on_evict is None: - raise - response = await self._retry_after_context_overflow( - context, - processed_request, - error, - ) - return await self._translator.translate(context, processed_request, response) - - async def _call_backend_stage( - self, - ctx: Any, - request: Any, - ) -> Any: - """Call the backend and then response processors.""" - native = _load_native() - try: - response: object = await cast(Any, self._backend).call(ctx, request) - except native.SwitchyardContextWindowExceededError: - raise - except Exception as error: - raise _backend_error(error) from error - if not isinstance(response, ChatResponse): - actual = _role_type_name(response) - raise native.SwitchyardBackendError( - f"Switchyard backend returned {actual}, expected ChatResponse", - ) - return await self._process_response_components(ctx, response) - - async def _process_request_components(self, ctx: Any, request: Any) -> Any: - """Run request-side compatibility components in order.""" - native = _load_native() - current = request - for component in self._request_components: - process = getattr(component, "process", None) - if not callable(process): - actual = _role_type_name(component) - raise native.SwitchyardProcessorError( - f"Request component {actual} must define process(ctx, request)", - ) - try: - current = await process(ctx, current) - except Exception as error: - raise _processor_error(error) from error - if not isinstance(current, ChatRequest): - actual = _role_type_name(current) - raise native.SwitchyardProcessorError( - f"Request component returned {actual}, expected ChatRequest", - ) - return current - - async def _process_response_components(self, ctx: Any, response: Any) -> Any: - """Run response-side compatibility components in order.""" - native = _load_native() - current = response - for component in self._response_components: - process = getattr(component, "process", None) - if not callable(process): - actual = _role_type_name(component) - raise native.SwitchyardProcessorError( - f"Response component {actual} must define process(ctx, response)", - ) - try: - current = await process(ctx, current) - except Exception as error: - raise _processor_error(error) from error - if not isinstance(current, ChatResponse): - actual = _role_type_name(current) - raise native.SwitchyardProcessorError( - f"Response component returned {actual}, expected ChatResponse", - ) - return current - - async def _retry_after_context_overflow( - self, - ctx: Any, - request: Any, - error: BaseException, - ) -> Any: - """Record the evicted target, rewrite the selection, and retry once.""" - native = _load_native() - target_id = self._overflow_target_id(ctx, error) - if target_id is not None: - evicted = set(ctx.evicted_targets or []) - evicted.add(target_id) - ctx.evicted_targets = sorted(evicted) - self._rewrite_evicted_pick(ctx) - try: - return await self._call_backend_stage(ctx, request) - except native.SwitchyardContextWindowExceededError as second: - last_target = self._overflow_target_id(ctx, second) or "unknown" - reason = "all attempted targets returned context-window overflow" - pool_error = native.SwitchyardContextPoolExhaustedError( - f"context pool exhausted after target {last_target}: {reason}", - ) - cast(Any, pool_error).last_target_id = last_target - cast(Any, pool_error).reason = reason - raise pool_error from second - - def _overflow_target_id( - self, - ctx: Any, - error: BaseException, - ) -> str | None: - """Return the target id carried by an overflow error or current context.""" - target_id = getattr(error, "target_id", None) - if isinstance(target_id, str) and target_id: - return target_id - selected = ctx.selected_target - return selected if isinstance(selected, str) and selected else None - - def _rewrite_evicted_pick(self, ctx: Any) -> None: - """Rewrite an evicted or exception-only target to the configured fallback.""" - selected = ctx.selected_target - evicted = set(ctx.evicted_targets or []) - if (selected is not None and selected in evicted) or (not selected and evicted): - ctx.selected_target = self._fallback_target_on_evict - - -def request_type_value(value: object) -> str: - """Normalize request-format tags and wire strings.""" - raw = value.value if hasattr(value, "value") else value - if not isinstance(raw, str): - raise TypeError(f"Request type must be a string-like value, got {type(raw).__name__}") - if raw == "anthropic_messages": - return "anthropic" - if raw in {"openai_chat", "openai_responses", "anthropic"}: - return raw - raise ValueError(f"Unknown request type: {value!r}") - - -def request_type_enum(value: object) -> ChatRequestType: - """Normalize a request-format tag to the Python enum.""" - return ChatRequestType(request_type_value(value)) - - -def request_type_matches(request: object, request_type: object) -> bool: - """Return whether a request has the requested wire format.""" - return request_type_value(cast(Any, request).request_type) == request_type_value(request_type) - - -def request_with_type(request_type: object, body: Mapping[str, Any] | JsonValue) -> ChatRequest: - """Build a request with the given wire format.""" - normalized = request_type_value(request_type) - if normalized == "openai_chat": - return ChatRequest.openai_chat(body) - if normalized == "openai_responses": - return ChatRequest.openai_responses(body) - if normalized == "anthropic": - return ChatRequest.anthropic(body) - raise ValueError(f"Unknown request type: {request_type!r}") - - -def response_type_value(value: object) -> str: - """Normalize response-format tags and wire strings.""" - raw = value.value if hasattr(value, "value") else value - if not isinstance(raw, str): - raise TypeError(f"Response type must be a string-like value, got {type(raw).__name__}") - legacy_aliases = { - "completion": "openai_completion", - "stream": "openai_stream", - "responses_api_completion": "openai_responses_completion", - "responses_api_stream": "openai_responses_stream", - } - raw = legacy_aliases.get(raw, raw) - if raw in { - "openai_completion", - "openai_stream", - "openai_responses_completion", - "openai_responses_stream", - "anthropic_completion", - "anthropic_stream", - }: - return raw - raise ValueError(f"Unknown response type: {value!r}") - - -def response_type_enum(value: object) -> ChatResponseType: - """Normalize a response-format tag to the Python enum.""" - return ChatResponseType(response_type_value(value)) - - -def response_type_matches(response: object, response_type: object) -> bool: - """Return whether a Rust-backed response has the requested wire shape.""" - return response_type_value(cast(Any, response).response_type) == response_type_value(response_type) - - -def response_with_type( - response_type: object, - body_or_stream: Mapping[str, Any] | JsonValue | object, -) -> ChatResponse: - """Build a response with the given wire shape.""" - normalized = response_type_value(response_type) - if normalized == "openai_completion": - return ChatResponse.openai_completion(body_or_stream) - if normalized == "openai_stream": - return ChatResponse.openai_stream(body_or_stream) - if normalized == "openai_responses_completion": - return ChatResponse.openai_responses_completion(body_or_stream) - if normalized == "openai_responses_stream": - return ChatResponse.openai_responses_stream(body_or_stream) - if normalized == "anthropic_completion": - return ChatResponse.anthropic_completion(body_or_stream) - if normalized == "anthropic_stream": - return ChatResponse.anthropic_stream(body_or_stream) - raise ValueError(f"Unknown response type: {response_type!r}") - - -def response_type_for_request_type( - request_type: object, - *, - stream: bool, -) -> ChatResponseType: - """Return the response shape that corresponds to a request format.""" - normalized = request_type_value(request_type) - if normalized == "openai_chat": - return response_type_enum("openai_stream" if stream else "openai_completion") - if normalized == "openai_responses": - return response_type_enum( - "openai_responses_stream" if stream else "openai_responses_completion" - ) - if normalized == "anthropic": - return response_type_enum("anthropic_stream" if stream else "anthropic_completion") - raise ValueError(f"Unknown request type: {request_type!r}") - - -def response_matches_request_type( - response: object, - request_type: object, -) -> bool: - """Return whether a response's provider format matches a request format.""" - response_type = response_type_value(cast(Any, response).response_type) - request_type_normalized = request_type_value(request_type) - if request_type_normalized == "openai_chat": - return response_type in {"openai_completion", "openai_stream"} - if request_type_normalized == "openai_responses": - return response_type in { - "openai_responses_completion", - "openai_responses_stream", - } - if request_type_normalized == "anthropic": - return response_type in {"anthropic_completion", "anthropic_stream"} - raise ValueError(f"Unknown request type: {request_type!r}") - - -def response_is_streaming(response: object) -> bool: - """Return whether a response carries a live stream.""" - return response_type_value(cast(Any, response).response_type).endswith("_stream") - - -__all__ = [ - "ChatRequest", - "ChatRequestType", - "ChatResponse", - "ChatResponseStream", - "ChatResponseType", - "LLMBackend", - "ProxyMetadata", - "ProxyContext", - "Switchyard", - "SwitchyardBackendError", - "SwitchyardConfigError", - "SwitchyardContextPoolExhaustedError", - "SwitchyardContextWindowExceededError", - "SwitchyardDuplicateRegistrationError", - "SwitchyardInvalidIdError", - "SwitchyardInvalidRequestError", - "SwitchyardModelNotFoundError", - "SwitchyardProcessorError", - "SwitchyardRuntimeError", - "SwitchyardUnsupportedRequestTypeError", - "SwitchyardUpstreamError", - "is_subagent_request", - "request_type_enum", - "request_type_matches", - "request_type_value", - "request_with_type", - "response_is_streaming", - "response_matches_request_type", - "response_type_enum", - "response_type_for_request_type", - "response_type_matches", - "response_type_value", - "response_with_type", -] diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py index dd01ed5d4..a7eb5c85a 100644 --- a/switchyard_rust/libsy.py +++ b/switchyard_rust/libsy.py @@ -8,7 +8,7 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Protocol -from switchyard_rust.core import _load_native +from switchyard_rust._native import load_native _EXPORTS = frozenset( { @@ -40,9 +40,7 @@ async def call( from collections.abc import Sequence from typing import final - from switchyard_rust.core import SwitchyardRuntimeError - - class LibsyError(SwitchyardRuntimeError): ... + class LibsyError(RuntimeError): ... @final class LlmTarget: @@ -112,7 +110,7 @@ def stage_router( def __getattr__(name: str) -> object: if name in _EXPORTS: - native: Any = _load_native() + native: Any = load_native() return getattr(native.libsy, name) raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/switchyard_rust/server.py b/switchyard_rust/server.py index 46b5d6036..193aef353 100644 --- a/switchyard_rust/server.py +++ b/switchyard_rust/server.py @@ -8,7 +8,7 @@ from os import PathLike from typing import TYPE_CHECKING, Any, final -from switchyard_rust.core import _load_native +from switchyard_rust._native import load_native if TYPE_CHECKING: @@ -38,7 +38,7 @@ def __exit__( def __getattr__(name: str) -> object: if name == "Server": - native: Any = _load_native() + native: Any = load_native() return native.server.Server raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/switchyard_rust/translation.py b/switchyard_rust/translation.py deleted file mode 100644 index b7b5c9843..000000000 --- a/switchyard_rust/translation.py +++ /dev/null @@ -1,527 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Python-side Switchyard wrappers over the Rust translation engine.""" - -from __future__ import annotations - -import dataclasses -import importlib -import inspect -import json -import logging -from collections.abc import AsyncGenerator, AsyncIterable, Iterable, Mapping, Sequence -from typing import TYPE_CHECKING, Any, Literal, Protocol, TypeVar, cast - -from openai.types.chat import ChatCompletionChunk - -if TYPE_CHECKING: - from switchyard.lib.proxy_context import ProxyContext - from switchyard.lib.roles import TranslatedResponse, TranslatedStream - from switchyard_rust.core import ChatRequest, ChatRequestType, ChatResponse - - -class _NativeStreamTranslation(Protocol): - def translate_event(self, event: Mapping[str, Any]) -> list[dict[str, Any]]: ... - def finish(self) -> list[dict[str, Any]]: ... - - -class _NativeTranslationEngine(Protocol): - def translate_request( - self, - source: str, - target: str, - body: Mapping[str, Any], - ) -> dict[str, Any]: ... - - def translate_response( - self, - source: str, - target: str, - body: Mapping[str, Any], - ) -> dict[str, Any]: ... - - def stream( - self, - source: str, - target: str, - model: str | None = None, - message_id: str | None = None, - ) -> _NativeStreamTranslation: ... - - def normalize_anthropic_tool_use_ids(self, messages: object) -> object: ... - - -class _NativeModule(Protocol): - TranslationEngine: type[_NativeTranslationEngine] - - -_T = TypeVar("_T") -_NativeFormat = Literal["openai_chat", "openai_responses", "anthropic_messages"] -_StreamOutput = Literal["objects", "chat_chunks", "responses_sse"] -_native: _NativeModule | None = None -_log = logging.getLogger(__name__) - - -def _load_native() -> _NativeModule: - global _native - if _native is None: - try: - _native = cast( - _NativeModule, - importlib.import_module("switchyard_rust._switchyard_rust"), - ) - except ImportError as exc: # pragma: no cover - broken install guard - raise RuntimeError( - "Rust translation extension is required. Run `uv run maturin develop` " - "or install a built switchyard wheel." - ) from exc - return _native - - -class TranslationEngine: - """Single Python-facing engine for request, response, and stream translation.""" - - def __init__(self) -> None: - self._inner = _load_native().TranslationEngine() - - def translate_request( - self, - source: str | ChatRequestType, - target: str | ChatRequestType, - body: Mapping[str, Any] | object, - ) -> dict[str, Any]: - """Translate a JSON request body between provider wire formats.""" - return self._inner.translate_request( - _format_name(source), - _format_name(target), - _jsonable_mapping(body), - ) - - def translate_response( - self, - source: str | ChatRequestType, - target: str | ChatRequestType, - body: Mapping[str, Any] | object, - ) -> dict[str, Any]: - """Translate a JSON response body between provider wire formats.""" - return self._inner.translate_response( - _format_name(source), - _format_name(target), - _jsonable_mapping(body), - ) - - def request_to( - self, - target: str | ChatRequestType, - request: ChatRequest, - ) -> ChatRequest: - """Return *request* in the target wire format.""" - source = _request_format(request) - target_format = _format_name(target) - if source == target_format: - return request - return _wrap_request( - target_format, - self.translate_request(source, target_format, request.body), - ) - - def request_to_any_of( - self, - request: ChatRequest, - supported: Sequence[ChatRequestType], - ) -> ChatRequest: - """Passthrough when possible, otherwise translate to the first supported type.""" - if not supported: - raise ValueError("supported must be non-empty") - source = _request_format(request) - supported_formats = [_format_name(item) for item in supported] - if source in supported_formats: - return request - return self.request_to(supported_formats[0], request) - - def response_to( - self, - target: str | ChatRequestType, - response: ChatResponse, - *, - served_model: str | None = None, - ) -> ChatResponse: - """Return a ChatResponse wrapper in the target wire format.""" - source = _response_format(response) - target_format = _format_name(target) - if source == target_format: - return response - if _is_streaming_response(response): - return _wrap_streaming_response( - target_format, - self._translate_response_stream(source, target_format, response, served_model), - ) - return _wrap_response( - target_format, - self.translate_response(source, target_format, _response_body(response)), - ) - - def response_for_request( - self, - request: ChatRequest, - response: ChatResponse, - *, - served_model: str | None = None, - ) -> TranslatedResponse: - """Translate a backend response to the original client's wire format.""" - if _is_streaming_response(response): - return self.stream_for_request(request, response, served_model=served_model) - source = _response_format(response) - target = _request_format(request) - if source == target: - return cast("TranslatedResponse", _response_body(response)) - return cast( - "TranslatedResponse", - self.translate_response(source, target, _response_body(response)), - ) - - def stream_for_request( - self, - request: ChatRequest, - response: ChatResponse, - *, - served_model: str | None = None, - ) -> TranslatedStream: - """Translate a backend stream to the original client's stream contract.""" - source = _response_format(response) - target = _request_format(request) - if source == target: - return cast("TranslatedStream", _response_stream(response)) - return cast( - "TranslatedStream", - self._translate_response_stream(source, target, response, served_model), - ) - - async def translate( - self, - ctx: ProxyContext, - request: ChatRequest, - response: ChatResponse, - ) -> TranslatedResponse: - """Implement Switchyard's terminal TranslationEngine role. - - The served model comes from the routing context, never from a request - body: routers rewrite the model on their own copy of the request, so the - request reaching this role can still name the route the client addressed. - """ - return self.response_for_request(request, response, served_model=ctx.selected_model) - - async def translate_stream( - self, - source: str | ChatRequestType, - target: str | ChatRequestType, - stream: AsyncIterable[Any], - *, - model: str | None = None, - message_id: str | None = None, - output: _StreamOutput = "objects", - ) -> AsyncGenerator[Any, None]: - """Translate an async stream between provider event formats. - - Closes *stream* on every exit path. When the client disconnects, the - ASGI server ``aclose()``-es the SSE generator consuming this one, which - raises ``GeneratorExit`` at the suspended ``yield``; without the - ``finally`` the upstream input stream (and the pooled connection it - holds) would never be released. - """ - target_format = _format_name(target) - translator = self._inner.stream( - _format_name(source), - target_format, - model, - message_id, - ) - try: - async for event in stream: - for payload in _stream_event_payloads(event): - for translated in translator.translate_event(payload): - yield _coerce_stream_output(target_format, translated, output) - for translated in translator.finish(): - yield _coerce_stream_output(target_format, translated, output) - finally: - await _aclose_input_stream(stream) - - def _translate_response_stream( - self, - source: _NativeFormat, - target: _NativeFormat, - response: ChatResponse, - served_model: str | None, - ) -> AsyncGenerator[Any, None]: - return self.translate_stream( - source, - target, - _response_stream(response), - model=served_model, - output=_stream_wire_output_for_target(target), - ) - - def normalize_anthropic_tool_use_ids(self, messages: object) -> object: - """Normalize Anthropic tool IDs without breaking result references.""" - return self._inner.normalize_anthropic_tool_use_ids(_jsonable(messages, set())) - - -async def _aclose_input_stream(stream: object) -> None: - """Best-effort close of a translated stream's upstream input. - - ``ChatResponseStream`` and async generators expose ``aclose``; SDK - ``AsyncStream`` objects expose ``close``; either may be a coroutine. - Closing must never mask the control flow that triggered it, so failures - are logged and swallowed. - """ - closer = getattr(stream, "aclose", None) or getattr(stream, "close", None) - if closer is None: - return - try: - result = closer() - if inspect.isawaitable(result): - await result - except Exception as exc: - _log.debug("Failed to close translated input stream: %s: %s", type(exc).__name__, exc) - - -def is_native_translation_available() -> bool: - """Return whether the required native translation extension loaded.""" - _load_native() - return True - - -def _format_name(value: str | ChatRequestType) -> _NativeFormat: - raw = value.value if hasattr(value, "value") else str(value) - if raw == "anthropic": - raw = "anthropic_messages" - if raw in {"openai_chat", "openai_responses", "anthropic_messages"}: - return cast(_NativeFormat, raw) - raise ValueError(f"Unknown translation format: {value!r}") - - -def _request_format(request: ChatRequest) -> _NativeFormat: - from switchyard_rust.core import request_type_value - - try: - return _format_name(request_type_value(request.request_type)) - except (AttributeError, ValueError) as exc: - raise NotImplementedError( - f"Request translation not implemented for {type(request).__name__}" - ) from exc - - -def _response_format(response: ChatResponse) -> _NativeFormat: - from switchyard_rust.core import response_type_value - - response_type = response_type_value(response.response_type) - if response_type in {"openai_completion", "openai_stream"}: - return "openai_chat" - if response_type in {"openai_responses_completion", "openai_responses_stream"}: - return "openai_responses" - if response_type in {"anthropic_completion", "anthropic_stream"}: - return "anthropic_messages" - raise NotImplementedError( - f"Response translation not implemented for {type(response).__name__}" - ) - - -def _wrap_request(format_name: _NativeFormat, body: dict[str, Any]) -> ChatRequest: - from switchyard_rust.core import request_with_type - - if format_name == "openai_chat": - return request_with_type("openai_chat", body) - if format_name == "openai_responses": - return request_with_type("openai_responses", body) - if format_name == "anthropic_messages": - return request_with_type("anthropic", body) - raise ValueError(f"Unknown request format: {format_name!r}") - - -def _wrap_response(format_name: _NativeFormat, body: dict[str, Any]) -> ChatResponse: - from switchyard_rust.core import ChatResponse - - if format_name == "openai_chat": - return ChatResponse.openai_completion(body) - if format_name == "openai_responses": - return ChatResponse.openai_responses_completion(body) - if format_name == "anthropic_messages": - return ChatResponse.anthropic_completion(body) - raise ValueError(f"Unknown response format: {format_name!r}") - - -def _wrap_streaming_response( - format_name: _NativeFormat, - stream: AsyncIterable[Any], -) -> ChatResponse: - from switchyard.lib.chat_response.anthropic import AnthropicResponseStream - from switchyard.lib.chat_response.openai_chat import ResponseStream - from switchyard.lib.chat_response.openai_responses import ResponsesApiStream - from switchyard_rust.core import ChatResponse - - if format_name == "openai_chat": - return ChatResponse.openai_stream(ResponseStream(cast(Any, stream))) - if format_name == "openai_responses": - return ChatResponse.openai_responses_stream(ResponsesApiStream(cast(Any, stream))) - if format_name == "anthropic_messages": - return ChatResponse.anthropic_stream(AnthropicResponseStream(cast(Any, stream))) - raise ValueError(f"Unknown streaming response format: {format_name!r}") - - -def _stream_wire_output_for_target(target: _NativeFormat) -> _StreamOutput: - if target == "openai_chat": - return "chat_chunks" - if target == "openai_responses": - return "responses_sse" - return "objects" - - -def _response_body(response: ChatResponse) -> Any: - return response.body - - -def _response_stream(response: ChatResponse) -> AsyncIterable[Any]: - return cast(AsyncIterable[Any], response.stream) - - -def _is_streaming_response(response: ChatResponse) -> bool: - from switchyard_rust.core import response_is_streaming - - return response_is_streaming(response) - - -def _coerce_stream_output( - target: _NativeFormat, - event: Mapping[str, Any], - output: _StreamOutput, -) -> Any: - if output == "responses_sse": - return _sse_frame(event) - if output == "chat_chunks": - return _validate_or_construct(ChatCompletionChunk, dict(event)) - _ = target - return dict(event) - - -def _validate_or_construct(model_cls: type[_T], body: dict[str, Any]) -> _T: - validator = getattr(model_cls, "model_validate", None) - if callable(validator): - try: - return cast(_T, validator(body)) - except Exception: - pass - constructor = getattr(model_cls, "model_construct", None) - if callable(constructor): - return cast(_T, constructor(**body)) - return model_cls(**body) - - -def _sse_frame(event: Mapping[str, Any]) -> str: - event_type = event.get("type", "message") - return f"event: {event_type}\ndata: {json.dumps(dict(event))}\n\n" - - -def _jsonable_mapping(value: Any) -> dict[str, Any]: - value = _jsonable(value, set()) - return dict(value) if isinstance(value, Mapping) else {} - - -def _stream_event_payloads(event: Any) -> list[dict[str, Any]]: - """JSON payload(s) of one stream item, for cross-format translation. - - Backends that preserve wire fidelity yield raw SSE frame *strings* - (see ``RawSSEFrameStream``); those are parsed here so the translator - still sees event dicts. Every other item shape keeps the existing - jsonable-mapping coercion. - """ - if isinstance(event, str): - return sse_frame_payloads(event) - return [_jsonable_mapping(event)] - - -def sse_frame_payloads(frame: str) -> list[dict[str, Any]]: - """Parse an SSE frame string into its JSON ``data:`` payload(s). - - Follows the SSE contract: a frame's ``data:`` lines are joined with - newlines to form one payload; an optional single leading space after the - colon is stripped. Comment/keep-alive frames, the ``[DONE]`` sentinel, - and non-JSON or non-object payloads yield nothing. Accepts a string - containing multiple ``\\n\\n``-separated frames and returns payloads in - order. - """ - payloads: list[dict[str, Any]] = [] - for block in frame.split("\n\n"): - data_lines: list[str] = [] - for line in block.split("\n"): - if line.startswith("data:"): - value = line[5:] - data_lines.append(value[1:] if value.startswith(" ") else value) - if not data_lines: - continue - data = "\n".join(data_lines) - if data.strip() == "[DONE]": - continue - try: - parsed = json.loads(data) - except ValueError: - continue - if isinstance(parsed, dict): - payloads.append(parsed) - return payloads - - -def _jsonable(value: Any, seen: set[int]) -> Any: - if hasattr(value, "model_dump"): - try: - return _jsonable(value.model_dump(exclude_none=True), seen) - except TypeError: - return _jsonable(value.model_dump(), seen) - if hasattr(value, "to_dict"): - return _jsonable(value.to_dict(), seen) - if dataclasses.is_dataclass(value) and not isinstance(value, type): - return _jsonable(dataclasses.asdict(value), seen) - if isinstance(value, Mapping): - obj_id = id(value) - if obj_id in seen: - return str(value) - seen.add(obj_id) - try: - if any(id(item) in seen for item in value.values()): - return str(value) - return {str(key): _jsonable(item, seen) for key, item in value.items()} - finally: - seen.remove(obj_id) - if isinstance(value, (list, tuple)): - obj_id = id(value) - if obj_id in seen: - return str(value) - seen.add(obj_id) - try: - if any(id(item) in seen for item in value): - return str(value) - return [_jsonable(item, seen) for item in value] - finally: - seen.remove(obj_id) - if isinstance(value, Iterable) and not isinstance(value, (str, bytes, bytearray)): - return [_jsonable(item, seen) for item in value] - if hasattr(value, "__dict__") and not isinstance(value, type): - obj_id = id(value) - if obj_id in seen: - return str(value) - seen.add(obj_id) - try: - return { - key: _jsonable(item, seen) - for key, item in vars(value).items() - if not key.startswith("_") - } - finally: - seen.remove(obj_id) - return value - - -__all__ = [ - "TranslationEngine", - "is_native_translation_available", -] diff --git a/tests/_chain_test_helpers.py b/tests/_chain_test_helpers.py deleted file mode 100644 index 29f290999..000000000 --- a/tests/_chain_test_helpers.py +++ /dev/null @@ -1,234 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Shared test infrastructure for classifier chain tests. - -Private (underscore-prefixed) module so pytest doesn't try to collect -it as a test file. Symbols inside also keep the underscore-private -convention used by the test files that import them. - -The owner is :mod:`tests.test_llm_classifier_e2e`, which exercises the -classifier-only routing chain. -""" - -from __future__ import annotations - -import json -import threading -from dataclasses import dataclass -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from typing import cast - -import httpx - -# --------------------------------------------------------------------------- -# URL + model constants shared between classifier and chain harnesses. -# --------------------------------------------------------------------------- - -#: respx-intercepted classifier endpoint. Distinct from the backend URL so an -#: unexpected hit is caught by missing-mock errors. -_CLASSIFIER_BASE = "https://classifier.test/v1" -_CLASSIFIER_URL = f"{_CLASSIFIER_BASE}/chat/completions" - -_CLASSIFIER_MODEL = "router-classifier-llm" - -#: Tier model IDs the deterministic backend exposes to its OpenAI-compatible -#: stubs. The two-tier (simple / complex) setup is enough to validate -#: classifier-driven dispatch without proliferating tiers. -_SIMPLE_MODEL = "tier/simple-model" -_COMPLEX_MODEL = "tier/complex-model" - - -# --------------------------------------------------------------------------- -# Classifier payloads (shared between classifier-only and chain tests). -# --------------------------------------------------------------------------- - - -def _classifier_payload(content: str) -> dict[str, object]: - """An OpenAI Chat Completion JSON body whose content is the classifier output.""" - return { - "id": "chatcmpl-classifier", - "object": "chat.completion", - "created": 1700000000, - "model": _CLASSIFIER_MODEL, - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": content}, - "finish_reason": "stop", - } - ], - "usage": {"prompt_tokens": 5, "completion_tokens": 10, "total_tokens": 15}, - } - - -def _signals_json(**overrides: object) -> str: - payload: dict[str, object] = { - "task_type": "debugging", - "complexity": "complex", - "reasoning_depth": "multi_step", - "tool_planning_required": False, - "precision_requirement": "high", - "context_dependency": "conversation", - "structured_output_risk": "low", - "recommended_tier": "complex", - "confidence": 0.88, - "reason_code": "debugging", - "abstain": False, - } - payload.update(overrides) - return json.dumps(payload) - - -# --------------------------------------------------------------------------- -# Generic backend stub bodies (tier-agnostic). -# --------------------------------------------------------------------------- - - -def _backend_payload(*, content: str, model: str) -> dict[str, object]: - return { - "id": "chatcmpl-backend", - "object": "chat.completion", - "created": 1700000001, - "model": model, - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": content}, - "finish_reason": "stop", - } - ], - "usage": {"prompt_tokens": 8, "completion_tokens": 4, "total_tokens": 12}, - } - - -def _stream_chunk(*, content: str = "", finish: str | None = None) -> dict[str, object]: - delta: dict[str, object] = {} - if content: - delta["content"] = content - return { - "id": "chatcmpl-backend-stream", - "object": "chat.completion.chunk", - "created": 1700000002, - "model": _COMPLEX_MODEL, - "choices": [{"index": 0, "delta": delta, "finish_reason": finish}], - } - - -def _sse_body(chunks: list[dict[str, object]]) -> bytes: - return ( - "".join(f"data: {json.dumps(c)}\n\n" for c in chunks) + "data: [DONE]\n\n" - ).encode("utf-8") - - -def _last_body(stub: _OpenAICompatStub) -> dict[str, object]: - return cast(dict[str, object], stub.requests[-1]["body"]) - - -# --------------------------------------------------------------------------- -# OpenAI-compatible loopback stub. Used for tier backends because Rust -# reqwest bypasses respx; a real local HTTP server lets us assert on the -# exact body the backend emitted. -# --------------------------------------------------------------------------- - - -class _OpenAICompatStub: - def __init__(self) -> None: - self._server: ThreadingHTTPServer | None = None - self._thread: threading.Thread | None = None - self._lock = threading.Lock() - self._requests: list[dict[str, object]] = [] - self._responses: list[tuple[int, bytes, str]] = [] - - def __enter__(self) -> _OpenAICompatStub: - owner = self - - class Handler(BaseHTTPRequestHandler): - protocol_version = "HTTP/1.1" - - def do_POST(self) -> None: - """Record the request (path, body, headers) and pop the queued reply.""" - length = int(self.headers.get("content-length", "0")) - raw = self.rfile.read(length) - body = json.loads(raw.decode("utf-8")) - with owner._lock: - # Header names lower-cased so tests can look them up - # without caring how the client cased them on the wire. - owner._requests.append({ - "path": self.path, - "body": body, - "headers": {k.lower(): v for k, v in self.headers.items()}, - }) - if owner._responses: - status, content, content_type = owner._responses.pop(0) - else: - status = 500 - content = b'{"error":{"message":"no stub response queued"}}' - content_type = "application/json" - - self.send_response(status) - self.send_header("content-type", content_type) - self.send_header("content-length", str(len(content))) - self.send_header("connection", "close") - self.end_headers() - self.wfile.write(content) - - def log_message(self, _format: str, *args: object) -> None: - return None - - self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) - # ``shutdown()`` blocks up to serve_forever's poll interval; the 0.5s - # default adds half a second of idle teardown to every test using the - # stub, so poll frequently. - server = self._server - self._thread = threading.Thread( - target=lambda: server.serve_forever(poll_interval=0.05), daemon=True - ) - self._thread.start() - return self - - def __exit__(self, *args: object) -> None: - if self._server is not None: - self._server.shutdown() - self._server.server_close() - if self._thread is not None: - self._thread.join(timeout=2) - - @property - def base_url(self) -> str: - if self._server is None: - raise RuntimeError("stub server is not running") - host, port = self._server.server_address - return f"http://{host}:{port}/v1" - - @property - def requests(self) -> list[dict[str, object]]: - with self._lock: - return list(self._requests) - - @property - def called(self) -> bool: - return bool(self.requests) - - def respond_json(self, body: dict[str, object], *, status: int = 200) -> None: - content = json.dumps(body).encode("utf-8") - with self._lock: - self._responses.append((status, content, "application/json")) - - def respond_sse(self, body: bytes) -> None: - with self._lock: - self._responses.append((200, body, "text/event-stream")) - - -# --------------------------------------------------------------------------- -# Harness dataclass returned by chain-test fixtures. -# --------------------------------------------------------------------------- - - -@dataclass(frozen=True) -class _ClassifierHarness: - """ASGI-driven client + per-tier loopback stubs for chain composition tests.""" - - client: httpx.AsyncClient - simple: _OpenAICompatStub - complex: _OpenAICompatStub diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index 324b7cca9..000000000 --- a/tests/conftest.py +++ /dev/null @@ -1,23 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Shared pytest fixtures.""" - -from __future__ import annotations - -import pytest - - -@pytest.fixture(autouse=True) -def _stub_anthropic_messages_probe(monkeypatch: pytest.MonkeyPatch) -> None: - """Keep unit tests hermetic. - - ``format=auto`` resolution probes ``/v1/messages`` over the network at - backend-build time, so presets that default a Claude tier to ``auto`` would - otherwise make live calls from unit tests. Stub it to a no-network default; - tests that exercise the probe set their own value, which overrides this. - """ - monkeypatch.setattr( - "switchyard.lib.backends.backend_format_resolver.probe_anthropic_messages_support_sync", - lambda **_kwargs: False, - ) diff --git a/tests/contract/__init__.py b/tests/contract/__init__.py deleted file mode 100644 index 0260e26e2..000000000 --- a/tests/contract/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Contract tests for the API surface that downstream consumers (e.g. the -NeMo Platform `nemo-switchyard` middleware plugin) depend on. - -These tests intentionally restate import paths and class shapes verbatim so a -PR that renames, deletes, or refactors a symbol fails *here* with a clear -message instead of breaking downstream at integration time. - -If a test in this suite fails because you intentionally changed the contract, -that's the signal to coordinate the migration with downstream consumers -*before* merging — don't silently update the test to match the new shape. -""" diff --git a/tests/contract/test_platform_imports.py b/tests/contract/test_platform_imports.py deleted file mode 100644 index d44081ee9..000000000 --- a/tests/contract/test_platform_imports.py +++ /dev/null @@ -1,68 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Import-surface contract: every symbol the NeMo Platform `nemo-switchyard` -plugin imports from switchyard must resolve. - -A failure here means a downstream IGW middleware plugin will hit -``ModuleNotFoundError`` or ``ImportError`` at process startup. The single -source of truth for this list is the platform plugin source tree at -``plugins/nemo-switchyard/src/nemo_switchyard/`` in the NVIDIA-NeMo/Platform -repo — keep these two in lockstep when adding or removing imports there. -""" - -from __future__ import annotations - -import importlib - -import pytest - -# (module path, [attribute names that must exist on the module]) -# -# Listed by source file in the downstream plugin so reviewers can grep the -# Platform side easily: -# _format.py — chat_request.{anthropic,base,openai_chat,openai_responses} -# _bridge.py — chat_request.base, chat_response.*, proxy_context.ProxyContext -# _processors.py — proxy_context.CTX_TARGET_FORMAT -# middleware.py — proxy_context -PLATFORM_IMPORT_SURFACE: list[tuple[str, list[str]]] = [ - ("switchyard.lib.chat_request.anthropic", ["AnthropicChatRequest"]), - ("switchyard.lib.chat_request.base", ["ChatRequest"]), - ("switchyard.lib.chat_request.openai_chat", ["OpenAIChatRequest"]), - ("switchyard.lib.chat_request.openai_responses", ["ResponsesChatRequest"]), - ( - "switchyard.lib.chat_response.anthropic", - ["AnthropicChatResponse", "AnthropicStreamingChatResponse", "AnthropicResponseStream"], - ), - ("switchyard.lib.chat_response.base", ["ChatResponse"]), - ( - "switchyard.lib.chat_response.openai_chat", - ["CompletionChatResponse", "StreamingChatResponse", "ResponseStream"], - ), - ( - "switchyard.lib.chat_response.openai_responses", - ["ResponsesApiChatResponse", "ResponsesApiStreamingChatResponse", "ResponsesApiStream"], - ), - ("switchyard.lib.proxy_context", ["CTX_TARGET_FORMAT", "ProxyContext"]), - ("switchyard.lib.processors.format_translate", ["TranslateConfig"]), -] - - -@pytest.mark.parametrize( - ("module_path", "attr"), - [(mod, attr) for mod, attrs in PLATFORM_IMPORT_SURFACE for attr in attrs], -) -def test_platform_import_resolves(module_path: str, attr: str) -> None: - """Each symbol the Platform plugin imports must resolve from upstream switchyard. - - Failure mode: a switchyard PR that deletes ``module_path`` or renames - ``attr`` ships a broken contract. Downstream IGW startup will fail with - ``ModuleNotFoundError`` or ``ImportError`` at the plugin entry-point load - step. - """ - module = importlib.import_module(module_path) - assert hasattr(module, attr), ( - f"Platform's nemo-switchyard plugin imports `{attr}` from `{module_path}`, " - f"but it is no longer exported. Coordinate the migration with the Platform team " - f"before merging." - ) diff --git a/tests/contract/test_proxy_context.py b/tests/contract/test_proxy_context.py deleted file mode 100644 index 57db68400..000000000 --- a/tests/contract/test_proxy_context.py +++ /dev/null @@ -1,67 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""ProxyContext + metadata-key constant contract. - -Platform reads ``CTX_TARGET_FORMAT`` and writes ``CTX_ORIGINAL_FORMAT`` into -``ProxyContext.metadata`` to bridge state across IGW request/response hooks -(see ``plugins/nemo-switchyard/src/nemo_switchyard/_processors.py`` and -``middleware.py`` in the Platform repo). - -If the constant is renamed or its sentinel value changes, request- and -response-side translate stops sharing state — the user sees broken responses -at runtime with no obvious error. -""" - -from __future__ import annotations - -import pytest - -from switchyard.lib.proxy_context import CTX_TARGET_FORMAT, ProxyContext - - -def test_ctx_target_format_is_a_stable_key() -> None: - """``CTX_TARGET_FORMAT`` must be a hashable value usable as a dict key. - - Platform stores it verbatim under ``ctx.metadata[CTX_TARGET_FORMAT]`` and - reads it back later. Any change here (string rename, enum migration) breaks - cross-phase metadata bridging silently. - """ - # Must be hashable (used as dict key) - probe = {CTX_TARGET_FORMAT: "sentinel"} - assert probe[CTX_TARGET_FORMAT] == "sentinel" - # Must not be None (Platform branches on key-present-vs-missing) - assert CTX_TARGET_FORMAT is not None - - -def test_proxy_context_metadata_round_trip() -> None: - """ProxyContext.metadata must accept arbitrary keys + survive round-trip. - - Platform's _bridge.py copies metadata between an IGW context and a - switchyard ProxyContext on each phase. A mapping that drops unknown keys or - enforces a typed schema breaks Platform. - """ - ctx = ProxyContext() - ctx.metadata[CTX_TARGET_FORMAT] = "openai_chat" - ctx.metadata["arbitrary_string_key"] = {"nested": "value"} - assert ctx.metadata[CTX_TARGET_FORMAT] == "openai_chat" - assert ctx.metadata["arbitrary_string_key"] == {"nested": "value"} - - -def test_proxy_context_is_constructible_with_no_args() -> None: - """Platform's bridge constructs ``ProxyContext()`` with no positional args - when building a side-pipeline. Required positional/kwargs is a breaking - change.""" - ctx = ProxyContext() - assert ctx is not None - assert hasattr(ctx, "metadata"), "ProxyContext must expose `.metadata` mapping" - - -@pytest.mark.parametrize( - "attr", - ["metadata"], -) -def test_proxy_context_exposes_required_attrs(attr: str) -> None: - """Attribute presence — exercised independently of constructor args.""" - ctx = ProxyContext() - assert hasattr(ctx, attr), f"ProxyContext.{attr} is required by Platform but is missing" diff --git a/tests/contract/test_request_response_types.py b/tests/contract/test_request_response_types.py deleted file mode 100644 index 3d6b7bb40..000000000 --- a/tests/contract/test_request_response_types.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Construction + dispatch contract for Rust-backed chat request aliases. - -The Python concrete request subclasses were collapsed into one Rust-backed -``ChatRequest`` type. The legacy provider-specific names remain importable as -aliases only; provider dispatch is keyed by ``request_type``. -""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import TypeAlias - -import pytest - -from switchyard.lib.chat_request.anthropic import AnthropicChatRequest -from switchyard.lib.chat_request.base import ChatRequest -from switchyard.lib.chat_request.openai_chat import OpenAIChatRequest -from switchyard.lib.chat_request.openai_responses import ResponsesChatRequest -from switchyard_rust.core import ChatRequestType, request_type_matches - -RequestAlias: TypeAlias = type[ChatRequest] -RequestFactory: TypeAlias = Callable[[dict], ChatRequest] - -OPENAI_CHAT_BODY = { - "model": "gpt-4", - "messages": [{"role": "user", "content": "ping"}], -} - -OPENAI_RESPONSES_BODY = { - "model": "gpt-4", - "input": "ping", -} - -ANTHROPIC_BODY = { - "model": "claude-3-haiku", - "messages": [{"role": "user", "content": "ping"}], - "max_tokens": 16, -} - - -REQUEST_CASES: list[tuple[str, RequestAlias, RequestFactory, object, dict]] = [ - ( - "OpenAIChatRequest", - OpenAIChatRequest, - ChatRequest.openai_chat, - ChatRequestType.OPENAI_CHAT, - OPENAI_CHAT_BODY, - ), - ( - "ResponsesChatRequest", - ResponsesChatRequest, - ChatRequest.openai_responses, - ChatRequestType.OPENAI_RESPONSES, - OPENAI_RESPONSES_BODY, - ), - ( - "AnthropicChatRequest", - AnthropicChatRequest, - ChatRequest.anthropic, - ChatRequestType.ANTHROPIC, - ANTHROPIC_BODY, - ), -] - - -@pytest.mark.parametrize( - ("alias_name", "alias", "_factory", "_request_type", "_body"), - REQUEST_CASES, -) -def test_request_legacy_names_are_aliases( - alias_name: str, - alias: RequestAlias, - _factory: RequestFactory, - _request_type: object, - _body: dict, -) -> None: - """Provider-specific request names remain aliases for the Rust type.""" - assert alias is ChatRequest, f"{alias_name} must alias ChatRequest" - - -@pytest.mark.parametrize( - ("_alias_name", "_alias", "factory", "_request_type", "body"), - REQUEST_CASES, -) -def test_request_constructible_with_factory( - _alias_name: str, - _alias: RequestAlias, - factory: RequestFactory, - _request_type: object, - body: dict, -) -> None: - """Each request factory must round-trip the raw body. - - Platform's bridge layer constructs requests from raw dict payloads coming - off the IGW wire. The Rust-backed type exposes provider factories instead - of the deleted ``cls(body=...)`` Python subclass constructors. - """ - req = factory(body) - assert req.body == body, "ChatRequest did not round-trip body" - - -@pytest.mark.parametrize( - ("alias_name", "alias", "factory", "request_type", "body"), - REQUEST_CASES, -) -def test_request_type_dispatch_works( - alias_name: str, - alias: RequestAlias, - factory: RequestFactory, - request_type: object, - body: dict, -) -> None: - """The alias-backed request must expose the Rust request type for dispatch.""" - req = factory(body) - - assert isinstance(req, alias), f"isinstance({alias_name}, ChatRequest alias) is false" - assert isinstance(req, ChatRequest), f"{alias_name} is not a ChatRequest alias" - assert request_type_matches(req, request_type), f"{alias_name} request_type is wrong" diff --git a/tests/e2e/_helpers.py b/tests/e2e/_helpers.py deleted file mode 100644 index 3495d1305..000000000 --- a/tests/e2e/_helpers.py +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Shared helpers for passthrough e2e tests. - -Hosts the three vendor-specific ``get_weather`` tool definitions and -the tool-capable-model resolver used by the Chat Completions, -Responses, and Anthropic Messages e2e suites. Pure Python — no -fixtures, so we keep this as a regular module rather than a -``conftest.py``. -""" - -from __future__ import annotations - -import os - - -def resolve_tool_capable_model(default_model: str) -> str: - """Pick a backend model that supports OpenAI-style tool calling. - - Some NVIDIA-hosted vLLM deployments (notably the - ``nvidia/qwen/qwen3.5-*`` family) aren't launched with - ``--enable-auto-tool-choice`` / ``--tool-call-parser`` and reject - ``tool_choice`` with HTTP 400 — the passthrough chain itself is - fine, the backend just won't parse tool calls. Tool-call tests - should therefore run against a known tool-capable model so we're - validating the passthrough's tool wiring rather than the backend's - vLLM flags. - - Resolution order: - - 1. ``OPENROUTER_TOOL_MODEL`` / ``NVIDIA_TOOL_MODEL`` env var — - explicit override for any backend or any model - 2. ``openai/openai/gpt-5.2`` when ``default_model`` starts with - ``nvidia/`` — that's the vLLM-hosted family described above, - and ``openai/openai/gpt-5.2`` is known to support tool calling - on ``inference-api.nvidia.com`` - 3. ``default_model`` otherwise — so non-NVIDIA backends keep - whatever the test suite was configured with - """ - override = os.environ.get("OPENROUTER_TOOL_MODEL") or os.environ.get("NVIDIA_TOOL_MODEL") - if override: - return override - if default_model.startswith("nvidia/"): - return "openai/openai/gpt-5.2" - return default_model - - -# --------------------------------------------------------------------------- -# Tool definitions — same semantic tool (``get_weather(city)``) rendered -# in each vendor's wire format, because each API has subtly different -# expectations: -# -# * Chat Completions nests the function metadata under ``function``. -# * Responses API flattens it out alongside ``type=function``. -# * Anthropic uses ``input_schema`` instead of ``parameters`` and has -# no ``type`` wrapper at all. -# --------------------------------------------------------------------------- - - -_WEATHER_SCHEMA: dict = { - "type": "object", - "properties": { - "city": { - "type": "string", - "description": "The city name, e.g. 'Tokyo'.", - }, - }, - "required": ["city"], -} - -_WEATHER_NAME = "get_weather" -_WEATHER_DESC = "Get the current weather for a city." - - -# Chat Completions format: ``{"type": "function", "function": {...}}``. -CHAT_COMPLETIONS_WEATHER_TOOL: dict = { - "type": "function", - "function": { - "name": _WEATHER_NAME, - "description": _WEATHER_DESC, - "parameters": _WEATHER_SCHEMA, - }, -} - - -# Responses API format: flat — ``type``, ``name``, ``description``, -# ``parameters`` all at the top level. -RESPONSES_WEATHER_TOOL: dict = { - "type": "function", - "name": _WEATHER_NAME, - "description": _WEATHER_DESC, - "parameters": _WEATHER_SCHEMA, -} - - -# Anthropic Messages format: no ``type``, ``input_schema`` instead of -# ``parameters``. -ANTHROPIC_WEATHER_TOOL: dict = { - "name": _WEATHER_NAME, - "description": _WEATHER_DESC, - "input_schema": _WEATHER_SCHEMA, -} diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py deleted file mode 100644 index 5ae9b4883..000000000 --- a/tests/e2e/conftest.py +++ /dev/null @@ -1,249 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Pytest fixtures for switchyard end-to-end production tests. - -These tests hit a real OpenAI-compatible backend (OpenRouter by default, -but any compatible URL works via env vars) through a subprocess-launched -``switchyard`` CLI. They're gated on an API key env var being set; without -one every test in this directory skips. - -Configuration (env vars, in resolution order): - -* ``OPENROUTER_API_KEY`` / ``NVIDIA_API_KEY`` — required; the test suite - skips without one -* ``OPENROUTER_BASE_URL`` / ``NVIDIA_BASE_URL`` — defaults to the selected - provider's OpenAI-compatible base URL -* ``OPENROUTER_MODEL`` / ``NVIDIA_MODEL`` — defaults to the selected provider's - GPT-5.2 model id - -Run with:: - - OPENROUTER_API_KEY=sk-or-... uv run pytest tests/e2e/ -v -""" - -from __future__ import annotations - -import logging -import os -import socket -import subprocess -import sys -import time -from collections.abc import Generator -from pathlib import Path - -import pytest - -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s [%(levelname)s] %(message)s", - datefmt="%H:%M:%S", - stream=sys.stdout, -) -logger = logging.getLogger("e2e") - -REPO_ROOT = Path(__file__).parent.parent.parent - -SERVER_STARTUP_TIMEOUT = 60.0 - - -def get_nvidia_config() -> dict: - """Resolve backend configuration from env vars. - - Returns a dict with ``api_key``, ``base_url``, and ``model``. - ``api_key`` is ``None`` when nothing's set — fixtures that depend - on it call ``pytest.skip``. - """ - openrouter_key = os.environ.get("OPENROUTER_API_KEY") - if openrouter_key: - return { - "provider": "openrouter", - "api_key": openrouter_key, - "base_url": ( - os.environ.get("OPENROUTER_BASE_URL") - or "https://openrouter.ai/api/v1" - ), - "model": os.environ.get("OPENROUTER_MODEL") or "openai/gpt-5.2", - } - - nvidia_key = os.environ.get("NVIDIA_API_KEY") - if nvidia_key: - return { - "provider": "nvidia", - "api_key": nvidia_key, - "base_url": ( - os.environ.get("NVIDIA_BASE_URL") - or "https://inference-api.nvidia.com/v1" - ), - "model": os.environ.get("NVIDIA_MODEL") or "openai/openai/gpt-5.2", - } - - return { - "provider": "openrouter", - "api_key": None, - "base_url": ( - os.environ.get("OPENROUTER_BASE_URL") - or "https://openrouter.ai/api/v1" - ), - "model": os.environ.get("OPENROUTER_MODEL") or "openai/gpt-5.2", - } - - -def find_free_port() -> int: - """Bind a socket to port 0 to claim a free port, then close it.""" - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.bind(("127.0.0.1", 0)) - s.listen(1) - return s.getsockname()[1] - - -def wait_for_server(port: int, timeout: float = 30.0, server_name: str = "server") -> bool: - """Poll ``127.0.0.1:port`` until it accepts TCP connections, or timeout.""" - start_time = time.time() - last_log_time = start_time - attempt = 0 - - while time.time() - start_time < timeout: - attempt += 1 - elapsed = time.time() - start_time - try: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.settimeout(1.0) - s.connect(("127.0.0.1", port)) - logger.info(f" {server_name} ready after {elapsed:.1f}s (attempt {attempt})") - return True - except (TimeoutError, ConnectionRefusedError, OSError): - if time.time() - last_log_time >= 5.0: - logger.info( - f" Still waiting for {server_name}... " - f"({elapsed:.1f}s elapsed, attempt {attempt})" - ) - last_log_time = time.time() - time.sleep(0.5) - - logger.warning(f" {server_name} failed to start after {timeout:.1f}s ({attempt} attempts)") - return False - - -def stop_server_subprocess(proc: subprocess.Popen, kill_timeout: float = 5.0) -> None: - """Gracefully terminate a subprocess, force-killing if it hangs.""" - if proc.poll() is not None: - return - proc.terminate() - try: - proc.wait(timeout=kill_timeout) - except subprocess.TimeoutExpired: - logger.warning("[Subprocess] Did not stop gracefully, killing...") - proc.kill() - proc.wait() - - -@pytest.fixture(scope="session") -def nvidia_config() -> dict: - """Backend configuration shared across the e2e suite. - - Skips the entire dependent test if no backend API key is set — - we don't want silent successes from a no-op backend. - """ - config = get_nvidia_config() - if not config["api_key"]: - pytest.skip("OPENROUTER_API_KEY or NVIDIA_API_KEY not set — required for e2e tests") - return config - - -def _start_passthrough_server( - port: int, - api_key: str, - base_url: str, -) -> subprocess.Popen: - """Launch ``switchyard passthrough`` as a subprocess. - - Invoked via ``python -m switchyard.cli.switchyard_cli`` (not - ``.venv/bin/switchyard``) so the tests are independent of whether - the editable-install script shim has been regenerated since the - last package install. - - Uses ``--inbound both`` so the one server exposes all three - inbound formats (``/v1/chat/completions``, ``/v1/responses``, - ``/v1/messages``) simultaneously — the Chat Completions, Responses, - and Anthropic Messages tests share this one process. - """ - cmd = [ - sys.executable, - "-m", "switchyard.cli.switchyard_cli", - "passthrough", - "--host", "127.0.0.1", - "--port", str(port), - "--inbound", "both", - "--api-key", api_key, - "--base-url", base_url, - ] - - env = os.environ.copy() - env.setdefault("OPENAI_API_KEY", api_key) - - return subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - cwd=str(REPO_ROOT), - env=env, - ) - - -@pytest.fixture(scope="session") -def passthrough_server(nvidia_config: dict) -> Generator[dict, None, None]: - """Start a real switchyard passthrough server on a free port. - - Session-scoped: every e2e file (Chat Completions, Responses, - Anthropic Messages) shares one running server, saving subprocess - startup overhead. - - Yields a dict with ``process``, ``port``, ``base_url``, ``model``. - """ - port = find_free_port() - - logger.info("") - logger.info(f"[Passthrough] {'=' * 60}") - logger.info("[Passthrough] Starting switchyard passthrough server") - logger.info(f"[Passthrough] Port: {port}") - logger.info(f"[Passthrough] Backend: {nvidia_config['base_url']}") - logger.info(f"[Passthrough] Model: {nvidia_config['model']}") - logger.info(f"[Passthrough] {'=' * 60}") - - proc = _start_passthrough_server( - port=port, - api_key=nvidia_config["api_key"], - base_url=nvidia_config["base_url"], - ) - - server_ready = wait_for_server( - port, - timeout=SERVER_STARTUP_TIMEOUT, - server_name="Passthrough", - ) - if not server_ready: - stop_server_subprocess(proc) - stdout = proc.stdout.read().decode() if proc.stdout else "" - stderr = proc.stderr.read().decode() if proc.stderr else "" - pytest.fail( - f"Passthrough server failed to start within " - f"{SERVER_STARTUP_TIMEOUT}s.\n" - f"stdout: {stdout[:2000]}\n" - f"stderr: {stderr[:2000]}" - ) - - base_url = f"http://127.0.0.1:{port}" - logger.info(f"[Passthrough] Server ready at {base_url}") - - yield { - "process": proc, - "port": port, - "base_url": base_url, - "model": nvidia_config["model"], - } - - logger.info("[Passthrough] Shutting down server...") - stop_server_subprocess(proc) - logger.info("[Passthrough] Server stopped") diff --git a/tests/e2e/test_passthrough_e2e.py b/tests/e2e/test_passthrough_e2e.py deleted file mode 100644 index 39cbf9c89..000000000 --- a/tests/e2e/test_passthrough_e2e.py +++ /dev/null @@ -1,552 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Production end-to-end smoke test for ``switchyard`` passthrough. - -Exercises the real production path end-to-end:: - - openai SDK → HTTP (127.0.0.1) → switchyard passthrough - → real backend → response - -Uses the official ``openai`` Python SDK as the client so we're -validating the exact request shape a real customer integration would -produce, and we get SDK-side validation of the response wire shape -for free. - -Unlike the in-process integration tests under ``tests/`` — which use -FastAPI's ``TestClient`` with a mocked backend — this test hits a real -OpenAI-compatible backend through the subprocess-launched -``switchyard`` CLI entry point. - -The subprocess / server fixture lives in this package's -``conftest.py`` so the Responses and Anthropic sibling suites can share -the same running server. - -Prerequisites: - - ``OPENROUTER_API_KEY`` or ``NVIDIA_API_KEY`` env var (skips otherwise) - -Run with:: - - OPENROUTER_API_KEY=sk-or-... pytest tests/e2e/test_passthrough_e2e.py -v -""" - -from __future__ import annotations - -import json -import logging -from collections.abc import Iterable -from typing import Any - -import openai -import pytest - -from ._helpers import ( - CHAT_COMPLETIONS_WEATHER_TOOL, - resolve_tool_capable_model, -) - -pytestmark = pytest.mark.integration - -logger = logging.getLogger("e2e.passthrough") - - -# --------------------------------------------------------------------------- -# Shared helpers -# --------------------------------------------------------------------------- - - -def _make_openai_client(passthrough_server: dict) -> openai.OpenAI: - """Build an OpenAI SDK client pointed at the passthrough server. - - The passthrough server injects its own upstream ``api_key`` into - calls it makes to the real backend, so the ``api_key`` we give the - SDK here is just a non-empty placeholder the SDK needs for its - ``Authorization`` header (the passthrough inbound layer doesn't - validate it). - """ - return openai.OpenAI( - base_url=f"{passthrough_server['base_url']}/v1", - api_key="not-used-passthrough-forwards-its-own", - timeout=60.0, - max_retries=0, - ) - - -def _reasoning_text(delta_or_message: Any) -> str | None: - """Return ``reasoning`` / ``reasoning_content`` from a pydantic field. - - Both names are vendor extensions produced by reasoning models - (different providers use different spellings), so they live in the - pydantic ``model_extra`` bag rather than as typed attributes on - ``ChatCompletionMessage`` / ``ChoiceDelta``. - """ - extra = getattr(delta_or_message, "model_extra", None) or {} - return extra.get("reasoning") or extra.get("reasoning_content") - - -def _collect_sdk_stream_deltas( - chunks: Iterable[Any], -) -> tuple[int, str, str, list[dict]]: - """Aggregate deltas from an OpenAI SDK ``ChatCompletionChunk`` stream. - - Reassembles fragmented fields across chunks using the OpenAI - streaming contract: - - * ``delta.content`` fragments → concatenated string - * ``delta.reasoning`` / ``delta.reasoning_content`` fragments (vendor - extensions for reasoning models) → concatenated string - * ``delta.tool_calls[*]`` fragments → rebuilt per-``index`` tool-call - dicts matching the non-streaming ``message.tool_calls`` shape - (``id``, ``type``, ``function.name``, ``function.arguments``) - - Returns ``(chunk_count, content, reasoning, tool_calls)``. - """ - chunk_count = 0 - content_parts: list[str] = [] - reasoning_parts: list[str] = [] - tool_calls_by_index: dict[int, dict] = {} - - for chunk in chunks: - chunk_count += 1 - if not chunk.choices: - continue - delta = chunk.choices[0].delta - - if delta.content: - content_parts.append(delta.content) - - reasoning = _reasoning_text(delta) - if reasoning: - reasoning_parts.append(str(reasoning)) - - for tc_delta in delta.tool_calls or []: - idx = tc_delta.index if tc_delta.index is not None else 0 - entry = tool_calls_by_index.setdefault( - idx, - { - "id": "", - "type": "function", - "function": {"name": "", "arguments": ""}, - }, - ) - if tc_delta.id: - entry["id"] = tc_delta.id - if tc_delta.function: - if tc_delta.function.name: - entry["function"]["name"] += tc_delta.function.name - if tc_delta.function.arguments: - entry["function"]["arguments"] += tc_delta.function.arguments - - tool_calls = [tool_calls_by_index[i] for i in sorted(tool_calls_by_index)] - return ( - chunk_count, - "".join(content_parts), - "".join(reasoning_parts), - tool_calls, - ) - - -# --------------------------------------------------------------------------- -# Tests -# --------------------------------------------------------------------------- - - -class TestPassthroughE2E: - """OpenAI SDK → Switchyard chain → real backend round-trip.""" - - # ------------------------------------------------------------------ - # Basic say-hi - # ------------------------------------------------------------------ - - def test_say_hi_get_response_back(self, passthrough_server: dict) -> None: - """``client.chat.completions.create("hi")`` round-trips successfully. - - Validates the minimum contract of the passthrough chain: - - * the SDK can reach the server and the request deserializes - cleanly into a ``ChatCompletion`` object - * the assistant role is returned - * non-empty assistant text comes back — in ``content`` for - non-reasoning models, or ``reasoning_content`` for reasoning - models that still had tokens left in their thinking phase - - ``max_tokens`` is deliberately generous to accommodate - *reasoning* models (e.g. Qwen 3.5 / GPT-5 series) that spend - the first several hundred tokens on internal reasoning before - emitting user-visible ``content``. - """ - client = _make_openai_client(passthrough_server) - response = client.chat.completions.create( - model=passthrough_server["model"], - messages=[{"role": "user", "content": "hi"}], - max_tokens=2048, - ) - - assert response.object == "chat.completion" - assert response.choices and len(response.choices) >= 1 - - choice = response.choices[0] - message = choice.message - assert message.role == "assistant" - - content = message.content - reasoning = _reasoning_text(message) - assert content or reasoning, ( - f"No assistant text in either 'content' or 'reasoning_content' " - f"(finish_reason={choice.finish_reason!r}): " - f"{response.model_dump()}" - ) - - logger.info(f" [passthrough] finish_reason: {choice.finish_reason!r}") - if content: - logger.info(f" [passthrough] content[:100]: {content[:100]!r}") - if reasoning: - logger.info(f" [passthrough] reasoning[:100]: {reasoning[:100]!r}") - - def test_say_hi_streaming(self, passthrough_server: dict) -> None: - """``stream=True`` yields ``ChatCompletionChunk`` objects. - - Validates the streaming contract end-to-end: the SDK can - iterate the SSE stream (so the passthrough's envelope — - ``Content-Type: text/event-stream``, ``data: ...`` frames, - ``[DONE]`` terminator — must be well-formed), chunks - deserialize as ``chat.completion.chunk``, and at least one - delta carries visible text (``delta.content`` or vendor - ``delta.reasoning``). - """ - client = _make_openai_client(passthrough_server) - stream = client.chat.completions.create( - model=passthrough_server["model"], - messages=[{"role": "user", "content": "hi"}], - max_tokens=2048, - stream=True, - ) - - chunk_count, content, reasoning, _ = _collect_sdk_stream_deltas(stream) - - assert chunk_count >= 1, "expected at least one streamed chunk" - assert content or reasoning, ( - f"no delta carried visible content or reasoning tokens " - f"across {chunk_count} chunks" - ) - - logger.info(f" [passthrough] streamed {chunk_count} chunk frames + [DONE]") - - # ------------------------------------------------------------------ - # System prompt — verifies a ``system`` role is forwarded through - # the chain to the backend and the response is well-formed. - # ------------------------------------------------------------------ - - def test_system_prompt_non_streaming(self, passthrough_server: dict) -> None: - """``system`` + ``user`` messages flow through and the backend honors them. - - Planted keyword ``pineapple`` is instructed via the system - prompt; when the model produces user-visible ``content`` we - assert the keyword is present, proving the system turn reached - the backend. For reasoning-only truncations (content empty, - all budget spent in ``reasoning_content``) we fall back to - asserting well-formedness — the chain still faithfully carried - the system turn even if the model didn't get to emit it. - """ - client = _make_openai_client(passthrough_server) - response = client.chat.completions.create( - model=passthrough_server["model"], - messages=[ - { - "role": "system", - "content": ( - "You are a helpful assistant. Always include " - "the exact word 'pineapple' in your response." - ), - }, - {"role": "user", "content": "Say hello."}, - ], - max_tokens=2048, - ) - - assert response.object == "chat.completion" - choice = response.choices[0] - message = choice.message - assert message.role == "assistant" - - content = message.content or "" - reasoning = _reasoning_text(message) or "" - assert content or reasoning, ( - f"no assistant text: {response.model_dump()}" - ) - - if content: - assert "pineapple" in content.lower(), ( - f"system prompt keyword missing from content — " - f"system turn may not have been forwarded. " - f"content={content[:200]!r}" - ) - - logger.info(f" [passthrough][system] finish_reason={choice.finish_reason!r}") - if content: - logger.info(f" [passthrough][system] content[:150]: {content[:150]!r}") - - def test_system_prompt_streaming(self, passthrough_server: dict) -> None: - """Same system-prompt round-trip, but over SSE via SDK streaming.""" - client = _make_openai_client(passthrough_server) - stream = client.chat.completions.create( - model=passthrough_server["model"], - messages=[ - { - "role": "system", - "content": ( - "You are a helpful assistant. Always include " - "the exact word 'pineapple' in your response." - ), - }, - {"role": "user", "content": "Say hello."}, - ], - max_tokens=2048, - stream=True, - ) - - chunk_count, content, reasoning, _ = _collect_sdk_stream_deltas(stream) - assert content or reasoning, ( - f"no streamed assistant text across {chunk_count} chunks" - ) - - if content: - assert "pineapple" in content.lower(), ( - f"system prompt keyword missing from streamed content — " - f"system turn may not have been forwarded. " - f"content={content[:200]!r}" - ) - - logger.info( - f" [passthrough][system-stream] {chunk_count} chunks, " - f"content[:100]={content[:100]!r}" - ) - - # ------------------------------------------------------------------ - # Multi-turn — verifies prior ``user`` + ``assistant`` turns are - # forwarded so the backend sees full conversation history. - # ------------------------------------------------------------------ - - def test_multi_turn_non_streaming(self, passthrough_server: dict) -> None: - """Prior conversation history flows through the chain. - - Plants the name ``Alice`` two turns back, then asks the model - to recall it. If the history reached the backend, the name - appears in the reply; if only the last user turn reached it, - the model has nothing to recall. Reasoning-only cut-offs are - forgiven (same convention as the system-prompt test above). - """ - client = _make_openai_client(passthrough_server) - response = client.chat.completions.create( - model=passthrough_server["model"], - messages=[ - { - "role": "user", - "content": "My name is Alice. Please remember this.", - }, - { - "role": "assistant", - "content": "Got it, Alice. I will remember your name.", - }, - { - "role": "user", - "content": ( - "What is my name? Reply with only the name " - "itself, nothing else." - ), - }, - ], - max_tokens=2048, - ) - - assert response.object == "chat.completion" - choice = response.choices[0] - message = choice.message - assert message.role == "assistant" - - content = message.content or "" - reasoning = _reasoning_text(message) or "" - assert content or reasoning, ( - f"no assistant text: {response.model_dump()}" - ) - - if content: - assert "alice" in content.lower(), ( - f"prior-turn context not recalled — multi-turn history " - f"may not have been forwarded. content={content[:200]!r}" - ) - - logger.info(f" [passthrough][multi-turn] finish_reason={choice.finish_reason!r}") - if content: - logger.info(f" [passthrough][multi-turn] content[:150]: {content[:150]!r}") - - def test_multi_turn_streaming(self, passthrough_server: dict) -> None: - """Same multi-turn round-trip, but over SSE via SDK streaming.""" - client = _make_openai_client(passthrough_server) - stream = client.chat.completions.create( - model=passthrough_server["model"], - messages=[ - { - "role": "user", - "content": "My name is Alice. Please remember this.", - }, - { - "role": "assistant", - "content": "Got it, Alice. I will remember your name.", - }, - { - "role": "user", - "content": ( - "What is my name? Reply with only the name " - "itself, nothing else." - ), - }, - ], - max_tokens=2048, - stream=True, - ) - - chunk_count, content, reasoning, _ = _collect_sdk_stream_deltas(stream) - assert content or reasoning, ( - f"no streamed assistant text across {chunk_count} chunks" - ) - - if content: - assert "alice" in content.lower(), ( - f"prior-turn context not recalled over SSE — multi-turn " - f"history may not have been forwarded. " - f"content={content[:200]!r}" - ) - - logger.info( - f" [passthrough][multi-turn-stream] {chunk_count} chunks, " - f"content[:100]={content[:100]!r}" - ) - - # ------------------------------------------------------------------ - # Tool calls — verifies ``tools`` definitions flow to the backend - # and an OpenAI-format tool call comes back through the chain. - # ------------------------------------------------------------------ - - def test_tool_call_non_streaming(self, passthrough_server: dict) -> None: - """``tools`` + ``tool_choice`` round-trip end-to-end. - - Validates that: - - * the ``tools`` definition is forwarded to the backend, - * the backend's ``tool_calls`` response survives the chain, - * the first tool call has the expected OpenAI shape - (``id``, ``type=function``, ``function.name``, - ``function.arguments``) and the arguments parse as JSON - containing the requested parameter. - - Uses :func:`resolve_tool_capable_model` to avoid vLLM-hosted - models that lack ``--enable-auto-tool-choice`` and would reject - ``tool_choice`` with HTTP 400 — we want to test the - passthrough's tool wiring, not the backend's flags. - """ - tool_model = resolve_tool_capable_model(passthrough_server["model"]) - logger.info(f" [passthrough][tools] using tool-capable model: {tool_model!r}") - - client = _make_openai_client(passthrough_server) - response = client.chat.completions.create( - model=tool_model, - messages=[{ - "role": "user", - "content": ( - "What is the weather in Tokyo? " - "You MUST use the get_weather tool." - ), - }], - tools=[CHAT_COMPLETIONS_WEATHER_TOOL], - tool_choice="auto", - max_tokens=2048, - ) - - assert response.object == "chat.completion" - message = response.choices[0].message - assert message.role == "assistant" - - tool_calls = message.tool_calls or [] - assert tool_calls, ( - f"expected at least one tool call from the backend, " - f"got message={message.model_dump()}" - ) - - first = tool_calls[0] - assert first.type == "function" - assert first.function.name == "get_weather", ( - f"unexpected function name: {first.function.name!r}" - ) - - raw_args = first.function.arguments or "{}" - args = json.loads(raw_args) if isinstance(raw_args, str) else raw_args - assert "city" in args, ( - f"expected 'city' in arguments, got {args!r}" - ) - - logger.info( - f" [passthrough][tools] tool_calls={len(tool_calls)}, " - f"fn={first.function.name!r}, args={args!r}" - ) - - def test_tool_call_streaming(self, passthrough_server: dict) -> None: - """Streaming tool-call deltas reassemble into a valid call. - - Each chunk's ``delta.tool_calls`` carries incremental fragments - (``id`` on the first chunk, ``function.name`` on one chunk, - ``function.arguments`` split across many). The helper - :func:`_collect_sdk_stream_deltas` rebuilds them using the - OpenAI streaming contract; this test then runs the same - structural assertions as the non-streaming variant. - - Uses :func:`resolve_tool_capable_model` for the same reason as - the non-streaming tool test. - """ - tool_model = resolve_tool_capable_model(passthrough_server["model"]) - logger.info(f" [passthrough][tools-stream] using tool-capable model: {tool_model!r}") - - client = _make_openai_client(passthrough_server) - stream = client.chat.completions.create( - model=tool_model, - messages=[{ - "role": "user", - "content": ( - "What is the weather in Tokyo? " - "You MUST use the get_weather tool." - ), - }], - tools=[CHAT_COMPLETIONS_WEATHER_TOOL], - tool_choice="auto", - max_tokens=2048, - stream=True, - ) - - chunk_count, content, reasoning, tool_calls = _collect_sdk_stream_deltas( - stream, - ) - assert tool_calls, ( - f"expected at least one streamed tool call, got " - f"{chunk_count} chunks, " - f"content[:100]={content[:100]!r}, " - f"reasoning[:100]={reasoning[:100]!r}" - ) - - first = tool_calls[0] - assert first.get("type") == "function" - fn = first.get("function") or {} - assert fn.get("name") == "get_weather", ( - f"unexpected function name: {fn.get('name')!r}" - ) - - raw_args = fn.get("arguments") or "{}" - args = json.loads(raw_args) if isinstance(raw_args, str) else raw_args - assert "city" in args, ( - f"expected 'city' in reassembled arguments, got {args!r}" - ) - - logger.info( - f" [passthrough][tools-stream] {chunk_count} chunks, " - f"tool_calls={len(tool_calls)}, fn={fn.get('name')!r}, " - f"args={args!r}" - ) diff --git a/tests/e2e/test_passthrough_responses_e2e.py b/tests/e2e/test_passthrough_responses_e2e.py deleted file mode 100644 index 481331db5..000000000 --- a/tests/e2e/test_passthrough_responses_e2e.py +++ /dev/null @@ -1,557 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Production end-to-end smoke test for ``switchyard`` passthrough on ``/v1/responses``. - -Exercises the cross-format production path end-to-end:: - - openai SDK (responses.create) → HTTP (127.0.0.1) - → switchyard passthrough - → TranslationEngine.request_to (Responses → Chat) - → real backend (Chat Completions) - → TranslationEngine.response_to (Chat → Responses) - → Responses-shaped response - -Unlike the sibling Chat Completions suite (which tests same-format -passthrough), this suite specifically validates the inbound and -outbound translators — a Responses-shaped request must survive -translation to Chat Completions, execute against the backend, and be -translated back to a Responses-shaped response. - -Covers both non-streaming (``responses.create``) and streaming -(``responses.create(stream=True)``) paths — the latter validates that -``TranslationEngine.stream_for_request`` drives the -OpenAI-chunk → Responses-SSE conversion end-to-end through -``iter_preframed_sse`` and the ``/v1/responses`` endpoint. - -Uses the official ``openai`` Python SDK as the client -(``client.responses.create(...)``) so we validate the exact request -shape a real customer integration would produce, and we get typed -response objects for free. - -Shares the ``passthrough_server`` fixture with the other e2e files -via ``conftest.py`` — one subprocess server per session. - -Run with:: - - OPENROUTER_API_KEY=sk-or-... pytest tests/e2e/test_passthrough_responses_e2e.py -v -""" - -from __future__ import annotations - -import json -import logging -from typing import Any - -import openai -import pytest - -from ._helpers import ( - RESPONSES_WEATHER_TOOL, - resolve_tool_capable_model, -) - -pytestmark = pytest.mark.integration - -logger = logging.getLogger("e2e.responses") - - -# --------------------------------------------------------------------------- -# Shared helpers -# --------------------------------------------------------------------------- - - -def _make_openai_client(passthrough_server: dict) -> openai.OpenAI: - """Build an OpenAI SDK client pointed at the passthrough server. - - Same convention as the Chat Completions sibling — the passthrough - injects its own upstream ``api_key`` when calling the real backend, - so the SDK-side ``api_key`` is just a placeholder for the - ``Authorization`` header. - """ - return openai.OpenAI( - base_url=f"{passthrough_server['base_url']}/v1", - api_key="not-used-passthrough-forwards-its-own", - timeout=60.0, - max_retries=0, - ) - - -def _extract_assistant_text(response: Any) -> str: - """Concatenate all ``output_text`` blocks across ``output`` items. - - The Responses API ``output`` array is heterogeneous — it can - contain ``message``, ``reasoning``, ``function_call``, and other - item types. This helper walks only ``type=message`` items and - pulls out their ``output_text`` blocks, returning the combined - assistant-visible text (empty string if none). - """ - parts: list[str] = [] - for item in response.output or []: - if getattr(item, "type", None) != "message": - continue - for block in getattr(item, "content", None) or []: - if getattr(block, "type", None) == "output_text": - text = getattr(block, "text", None) - if text: - parts.append(text) - return "".join(parts) - - -def _extract_function_calls(response: Any) -> list[Any]: - """Return every ``type=function_call`` item in ``response.output``. - - In the Responses API tool calls are top-level items in ``output`` - rather than nested inside a ``message`` (unlike Chat Completions - where they live under ``message.tool_calls``). This helper keeps - the extraction logic in one place for the tool test below. - """ - return [ - item - for item in (response.output or []) - if getattr(item, "type", None) == "function_call" - ] - - -def _collect_responses_stream_events( - stream: Any, -) -> tuple[list[str], str, dict[int, dict[str, Any]]]: - """Drain a Responses API stream and summarize events. - - Returns ``(event_types, text, function_calls)``: - - * ``event_types`` — the ``type`` field of every ``ResponseStreamEvent`` - received, in order. Used for lifecycle structural assertions - (``response.created`` first, ``response.completed`` last). - * ``text`` — all ``response.output_text.delta`` strings concatenated. - * ``function_calls`` — keyed by ``output_index``, each entry carries - ``{"name", "call_id", "arguments"}`` with ``arguments`` being the - concatenated ``response.function_call_arguments.delta`` fragments. - """ - event_types: list[str] = [] - text_parts: list[str] = [] - function_calls: dict[int, dict[str, Any]] = {} - - for event in stream: - ev_type = getattr(event, "type", None) - if ev_type: - event_types.append(ev_type) - - if ev_type == "response.output_item.added": - item = getattr(event, "item", None) - if item is not None and getattr(item, "type", None) == "function_call": - idx = getattr(event, "output_index", 0) - function_calls[idx] = { - "name": getattr(item, "name", ""), - "call_id": getattr(item, "call_id", ""), - "arguments": "", - } - elif ev_type == "response.output_text.delta": - text_parts.append(getattr(event, "delta", "") or "") - elif ev_type == "response.function_call_arguments.delta": - idx = getattr(event, "output_index", 0) - if idx in function_calls: - function_calls[idx]["arguments"] += ( - getattr(event, "delta", "") or "" - ) - - return event_types, "".join(text_parts), function_calls - - -# --------------------------------------------------------------------------- -# Tests -# --------------------------------------------------------------------------- - - -class TestResponsesPassthroughE2E: - """OpenAI Responses SDK → cross-format passthrough → backend → Responses out.""" - - # ------------------------------------------------------------------ - # Basic say-hi - # ------------------------------------------------------------------ - - def test_say_hi_get_response_back(self, passthrough_server: dict) -> None: - """``client.responses.create(input="hi")`` round-trips successfully. - - Validates the minimum contract of the Responses passthrough: - - * the Responses SDK can reach the server and the request - deserializes cleanly into a ``Response`` object - * ``output`` is non-empty and contains an ``assistant`` message - with at least one ``output_text`` block that has visible text - """ - client = _make_openai_client(passthrough_server) - response = client.responses.create( - model=passthrough_server["model"], - input="hi", - max_output_tokens=2048, - ) - - assert response.output, ( - f"expected non-empty output, got {response.model_dump()}" - ) - - text = _extract_assistant_text(response) - assert text, ( - f"no assistant output_text across {len(response.output)} items: " - f"{response.model_dump()}" - ) - - logger.info( - f" [Responses] output_items={len(response.output)}, " - f"text[:100]={text[:100]!r}" - ) - - # ------------------------------------------------------------------ - # System prompt — Responses API uses ``instructions`` as the - # top-level system-equivalent parameter (not a ``system`` role). - # ------------------------------------------------------------------ - - def test_system_prompt_non_streaming(self, passthrough_server: dict) -> None: - """``instructions`` parameter flows through the cross-format translator. - - Responses API models the system prompt as a top-level - ``instructions`` string, distinct from the message list. The - ``TranslationEngine.request_to`` must lift - ``instructions`` into a ``role=system`` message before calling - the backend. If that translation drops the field, the model - has no way to honor the planted ``pineapple`` keyword, and - this test catches it. - """ - client = _make_openai_client(passthrough_server) - response = client.responses.create( - model=passthrough_server["model"], - instructions=( - "You are a helpful assistant. Always include the exact " - "word 'pineapple' in your response." - ), - input="Say hello.", - max_output_tokens=2048, - ) - - assert response.output, ( - f"expected non-empty output: {response.model_dump()}" - ) - - # Keyword check — only when the model produced visible - # output_text. Reasoning models can burn their entire - # ``max_output_tokens`` budget on internal thought before the - # Responses translator surfaces any ``output_text`` block; - # that's a backend-scheduling artifact, not a passthrough - # failure, so we forgive it. The shape checks above still - # prove the ``instructions`` translation didn't blow up. - text = _extract_assistant_text(response) - if text: - assert "pineapple" in text.lower(), ( - f"instructions not honored — the ``instructions`` → " - f"system translation may have dropped the field. " - f"text={text[:200]!r}" - ) - - logger.info( - f" [Responses][system] output_items={len(response.output)}, " - f"text[:150]={text[:150]!r}" - ) - - # ------------------------------------------------------------------ - # Multi-turn — Responses API accepts a list of message items as - # ``input``, exercising the translator's handling of assistant-role - # history items. - # ------------------------------------------------------------------ - - def test_multi_turn_non_streaming(self, passthrough_server: dict) -> None: - """Multi-turn history via message-list ``input`` flows through. - - Sends a 3-item ``input`` (``user`` / ``assistant`` / ``user``) - and asks the model to recall a name planted two turns back. - If the translator drops the assistant turn, the model has no - context and can't recall the name. - """ - # Each input item is explicitly tagged ``type="message"``: - # ``TranslationEngine._convert_input_items_to_messages`` - # dispatches on ``item["type"]`` and silently drops items that - # don't carry one, which would leave ``messages=[]`` and 400 - # the backend with "list index out of range". - client = _make_openai_client(passthrough_server) - response = client.responses.create( - model=passthrough_server["model"], - input=[ - { - "type": "message", - "role": "user", - "content": "My name is Alice. Please remember this.", - }, - { - "type": "message", - "role": "assistant", - "content": "Got it, Alice. I will remember your name.", - }, - { - "type": "message", - "role": "user", - "content": ( - "What is my name? Reply with only the name " - "itself, nothing else." - ), - }, - ], - max_output_tokens=2048, - ) - - assert response.output, ( - f"expected non-empty output: {response.model_dump()}" - ) - - # Same reasoning-truncation forgiveness as the ``instructions`` - # test above. - text = _extract_assistant_text(response) - if text: - assert "alice" in text.lower(), ( - f"prior-turn context not recalled — the ``input`` list " - f"→ messages translation may have dropped history items. " - f"text={text[:200]!r}" - ) - - logger.info( - f" [Responses][multi-turn] output_items={len(response.output)}, " - f"text[:150]={text[:150]!r}" - ) - - # ------------------------------------------------------------------ - # Tool call — Responses API puts ``function_call`` items directly - # into ``output`` rather than nesting them under a message's - # ``tool_calls`` field. This test confirms that shape survives the - # round-trip through Chat Completions format. - # ------------------------------------------------------------------ - - def test_tool_call_non_streaming(self, passthrough_server: dict) -> None: - """``tools`` definitions flow through and a ``function_call`` item returns. - - Validates that: - - * the flat Responses-shape tool definition is forwarded through - the translator (Responses → Chat Completions nested shape) - * the backend's ``tool_calls`` on the Chat response are - translated back to top-level ``function_call`` items in - ``response.output`` - * the first function call has the expected name and its - ``arguments`` string parses as JSON containing ``city`` - - Uses :func:`resolve_tool_capable_model` to avoid vLLM-hosted - models that lack ``--enable-auto-tool-choice``. - """ - tool_model = resolve_tool_capable_model(passthrough_server["model"]) - logger.info( - f" [Responses][tools] using tool-capable model: {tool_model!r}" - ) - - client = _make_openai_client(passthrough_server) - response = client.responses.create( - model=tool_model, - input=( - "What is the weather in Tokyo? " - "You MUST use the get_weather tool." - ), - tools=[RESPONSES_WEATHER_TOOL], - tool_choice="auto", - max_output_tokens=2048, - ) - - function_calls = _extract_function_calls(response) - assert function_calls, ( - f"expected at least one function_call item in response.output, " - f"got {[getattr(i, 'type', None) for i in response.output or []]}" - ) - - first = function_calls[0] - assert first.name == "get_weather", ( - f"unexpected function name: {first.name!r}" - ) - - raw_args = first.arguments or "{}" - args = json.loads(raw_args) if isinstance(raw_args, str) else raw_args - assert "city" in args, ( - f"expected 'city' in arguments, got {args!r}" - ) - - logger.info( - f" [Responses][tools] function_calls={len(function_calls)}, " - f"name={first.name!r}, args={args!r}" - ) - - # ------------------------------------------------------------------ - # Streaming — validates the Chat Completions → Responses SSE - # translation drives through the chain end-to-end. - # - # Wire path: - # OpenAI SDK responses.create(stream=True) → /v1/responses - # → chain → OpenAiPassthroughBackend returns an OpenAI stream response - # → translate_stream dispatches on the Responses request format - # → stream_chat_to_responses_sse yields pre-framed SSE strings - # → iter_preframed_sse forwards them → client SDK parses them - # ------------------------------------------------------------------ - - def test_say_hi_streaming(self, passthrough_server: dict) -> None: - """``responses.create(stream=True)`` yields a valid Responses SSE stream. - - Validates the minimum streaming contract: - - * the SDK can iterate the SSE stream (so the passthrough's - envelope — ``Content-Type: text/event-stream``, ``event: - response.*\\ndata: {...}\\n\\n`` frames — must be well-formed) - * the stream emits ``response.created`` before any content and - ``response.completed`` at the end - * at least one ``response.output_text.delta`` carries visible - text (so the translator text-path works chunk-by-chunk) - """ - client = _make_openai_client(passthrough_server) - stream = client.responses.create( - model=passthrough_server["model"], - input="hi", - max_output_tokens=2048, - stream=True, - ) - event_types, text, _ = _collect_responses_stream_events(stream) - - assert event_types and event_types[0] == "response.created", ( - f"expected first event to be response.created, got {event_types[:3]!r}" - ) - assert "response.completed" in event_types, ( - f"missing response.completed, got types={event_types!r}" - ) - assert text, ( - f"no output_text.delta across {len(event_types)} events: " - f"types={event_types!r}" - ) - - logger.info( - f" [Responses-stream] events={len(event_types)}, " - f"text[:100]={text[:100]!r}" - ) - - def test_system_prompt_streaming(self, passthrough_server: dict) -> None: - """``instructions`` flows through the streaming cross-format translator.""" - client = _make_openai_client(passthrough_server) - stream = client.responses.create( - model=passthrough_server["model"], - instructions=( - "You are a helpful assistant. Always include the exact " - "word 'pineapple' in your response." - ), - input="Say hello.", - max_output_tokens=2048, - stream=True, - ) - event_types, text, _ = _collect_responses_stream_events(stream) - - assert "response.created" in event_types - assert "response.completed" in event_types - - if text: - assert "pineapple" in text.lower(), ( - f"instructions not honored over the stream path. " - f"text={text[:200]!r}" - ) - - logger.info( - f" [Responses-stream][system] events={len(event_types)}, " - f"text[:150]={text[:150]!r}" - ) - - def test_multi_turn_streaming(self, passthrough_server: dict) -> None: - """Multi-turn history via message-list ``input`` streams back.""" - client = _make_openai_client(passthrough_server) - stream = client.responses.create( - model=passthrough_server["model"], - input=[ - { - "type": "message", - "role": "user", - "content": "My name is Alice. Please remember this.", - }, - { - "type": "message", - "role": "assistant", - "content": "Got it, Alice. I will remember your name.", - }, - { - "type": "message", - "role": "user", - "content": ( - "What is my name? Reply with only the name " - "itself, nothing else." - ), - }, - ], - max_output_tokens=2048, - stream=True, - ) - event_types, text, _ = _collect_responses_stream_events(stream) - - assert "response.created" in event_types - assert "response.completed" in event_types - - if text: - assert "alice" in text.lower(), ( - f"prior-turn context not recalled over the stream path. " - f"text={text[:200]!r}" - ) - - logger.info( - f" [Responses-stream][multi-turn] events={len(event_types)}, " - f"text[:150]={text[:150]!r}" - ) - - def test_tool_call_streaming(self, passthrough_server: dict) -> None: - """Streaming tool-call deltas reassemble into a valid ``function_call``. - - Validates: - - * ``response.output_item.added`` carries a ``function_call`` - item with the expected ``name`` - * ``response.function_call_arguments.delta`` fragments - concatenate into a parseable JSON object containing ``city`` - * the stream terminates with ``response.completed`` - """ - tool_model = resolve_tool_capable_model(passthrough_server["model"]) - logger.info( - f" [Responses-stream][tools] using tool-capable model: {tool_model!r}" - ) - - client = _make_openai_client(passthrough_server) - stream = client.responses.create( - model=tool_model, - input=( - "What is the weather in Tokyo? " - "You MUST use the get_weather tool." - ), - tools=[RESPONSES_WEATHER_TOOL], - tool_choice="auto", - max_output_tokens=2048, - stream=True, - ) - event_types, _, function_calls = _collect_responses_stream_events(stream) - - assert "response.completed" in event_types - assert function_calls, ( - f"no streamed function_call item across {len(event_types)} events: " - f"types={event_types!r}" - ) - - first_idx = sorted(function_calls)[0] - first = function_calls[first_idx] - assert first["name"] == "get_weather", ( - f"unexpected function name: {first['name']!r}" - ) - - raw_args = first["arguments"] or "{}" - args = json.loads(raw_args) - assert "city" in args, ( - f"expected 'city' in reassembled arguments, got {args!r}" - ) - - logger.info( - f" [Responses-stream][tools] events={len(event_types)}, " - f"function_calls={len(function_calls)}, " - f"name={first['name']!r}, args={args!r}" - ) diff --git a/tests/e2e_multiturn_responses.py b/tests/e2e_multiturn_responses.py deleted file mode 100644 index 08cd79992..000000000 --- a/tests/e2e_multiturn_responses.py +++ /dev/null @@ -1,369 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""E2E test: multi-turn Responses API with GPT-OSS-120B via NVIDIA API. - -Simulates Codex-like multi-turn tool-call conversations through the proxy's -/v1/responses endpoint. After all turns complete, loads the saved traces -from --rl-log-dir and verifies that each turn produced a separate assistant -message (i.e. the turn-merge bug is fixed). - -Usage: - 1. Start the proxy in a separate terminal: - source .venv/bin/activate - switchyard passthrough \ - --port 4000 \ - --api-key "$OPENAI_API_KEY" \ - --base-url https://inference-api.nvidia.com/v1 \ - --enable-rl-logging --rl-log-dir ./e2e_traces - - 2. Run this script: - python tests/e2e_multiturn_responses.py --proxy-url http://localhost:4000 - - The script exits 0 on success, 1 on failure. -""" - -import argparse -import json -import sys -import time -import uuid -from pathlib import Path - -import httpx - -# --------------------------------------------------------------------------- -# Config -# --------------------------------------------------------------------------- - -DEFAULT_PROXY_URL = "http://localhost:4000" -DEFAULT_MODEL = "nvidia/openai/gpt-oss-120b" -DEFAULT_TRACE_DIR = Path(__file__).parent.parent / "e2e_traces" -SESSION_ID = str(uuid.uuid4()) - -TOOLS = [ - { - "type": "function", - "name": "exec_command", - "description": "Runs a shell command and returns stdout.", - "parameters": { - "type": "object", - "properties": { - "cmd": {"type": "string", "description": "Shell command to execute."}, - }, - "required": ["cmd"], - "additionalProperties": False, - }, - }, -] - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -def send_responses_request( - proxy_url: str, - model: str, - input_items: list, - session_id: str, - stream: bool = True, - timeout: float = 120.0, -) -> dict: - """Send a Responses API request and return the parsed response. - - For streaming, collects SSE events and returns the response.completed payload. - For non-streaming, returns the JSON response directly. - """ - body = { - "model": model, - "input": input_items, - "tools": TOOLS, - "tool_choice": "auto", - "stream": stream, - "temperature": 0.2, - } - - headers = { - "Content-Type": "application/json", - "proxy_x_session_id": session_id, - } - - url = f"{proxy_url}/v1/responses" - - if not stream: - with httpx.Client(timeout=timeout) as client: - resp = client.post(url, json=body, headers=headers) - resp.raise_for_status() - return resp.json() - - # Streaming: collect SSE events - completed_response = None - with httpx.Client(timeout=timeout) as client: - with client.stream("POST", url, json=body, headers=headers) as resp: - resp.raise_for_status() - buffer = "" - for chunk in resp.iter_text(): - buffer += chunk - while "\n\n" in buffer: - event_str, buffer = buffer.split("\n\n", 1) - lines = event_str.strip().split("\n") - event_type = None - event_data = None - for line in lines: - if line.startswith("event: "): - event_type = line[7:] - elif line.startswith("data: "): - event_data = line[6:] - - if event_type == "response.completed" and event_data: - completed_response = json.loads(event_data) - elif event_type == "error" and event_data: - err = json.loads(event_data) - print(f" SSE error: {err}", file=sys.stderr) - - if completed_response is None: - raise RuntimeError("Never received response.completed event") - - return completed_response - - -def extract_tool_calls(response: dict) -> list[dict]: - """Extract function_call items from a Responses API response.""" - output = response.get("response", response).get("output", []) - return [item for item in output if item.get("type") == "function_call"] - - -def extract_text(response: dict) -> str: - """Extract text content from a Responses API response.""" - output = response.get("response", response).get("output", []) - parts = [] - for item in output: - if item.get("type") == "message": - for content in item.get("content", []): - if content.get("type") == "output_text": - parts.append(content.get("text", "")) - return "\n".join(parts) - - -def simulate_tool_results(tool_calls: list[dict]) -> list[dict]: - """Generate fake tool results for the given tool calls.""" - results = [] - for tc in tool_calls: - args = json.loads(tc.get("arguments", "{}")) - cmd = args.get("cmd", "unknown") - # Fake output based on command - if "ls" in cmd: - output = "file1.py\nfile2.py\nREADME.md\n" - elif "echo" in cmd: - output = "hello\n" - elif "cat" in cmd: - output = "# File content\nprint('hello world')\n" - elif "pwd" in cmd: - output = "/workspace\n" - else: - output = f"Executed: {cmd}\n" - - results.append({ - "type": "function_call_output", - "call_id": tc.get("call_id", ""), - "output": output, - }) - return results - - -def build_history(turns: list[tuple[list[dict], list[dict]]]) -> list[dict]: - """Build Responses API input items from a list of (tool_calls, tool_results) turns. - - Each turn's function_calls come first (grouped), then function_call_outputs. - This matches how Codex sends multi-turn history. - """ - items = [] - for tool_calls, tool_results in turns: - for tc in tool_calls: - items.append({ - "type": "function_call", - "name": tc.get("name", ""), - "call_id": tc.get("call_id", ""), - "arguments": tc.get("arguments", "{}"), - }) - for tr in tool_results: - items.append(tr) - return items - - -# --------------------------------------------------------------------------- -# Load and verify traces -# --------------------------------------------------------------------------- - -def load_session_traces(trace_dir: Path, session_id: str) -> list[dict]: - """Load trace files for a specific session, sorted by timestamp.""" - if not trace_dir.exists(): - return [] - traces = [] - for f in sorted(trace_dir.glob("*.json")): - with open(f) as fh: - t = json.load(fh) - if t.get("sessionID") == session_id: - traces.append(t) - # Also check subdirectories (e.g. openai/) - for f in sorted(trace_dir.rglob("*.json")): - with open(f) as fh: - t = json.load(fh) - if t.get("sessionID") == session_id and t not in traces: - traces.append(t) - # Sort by timestamp - traces.sort(key=lambda t: t.get("timestamp", "")) - return traces - - -def verify_traces(traces: list[dict], expected_turns: int) -> bool: - """Verify that traces have correct multi-turn structure. - - Returns True if all traces show separate assistant messages per turn. - """ - ok = True - - for i, trace in enumerate(traces): - msgs = trace["request"]["messages"] - asst_msgs = [m for m in msgs if m["role"] == "assistant"] - tool_msgs = [m for m in msgs if m["role"] == "tool"] - - # Trace i should have exactly i assistant messages in its history - # (one per previous turn) - expected_asst = i # trace 0 has 0, trace 1 has 1, etc. - - if len(asst_msgs) != expected_asst: - print( - f" FAIL trace {i+1}: expected {expected_asst} assistant msg(s) " - f"in history, got {len(asst_msgs)}" - ) - if len(asst_msgs) > 0: - for j, am in enumerate(asst_msgs): - n_calls = len(am.get("tool_calls", [])) - print(f" assistant msg {j}: {n_calls} tool_calls") - ok = False - else: - # Verify each assistant message has the right tool_calls count - total_tool_calls = sum( - len(am.get("tool_calls", [])) for am in asst_msgs - ) - print( - f" OK trace {i+1}: {len(asst_msgs)} assistant msg(s), " - f"{total_tool_calls} total tool_calls, {len(tool_msgs)} tool results" - ) - - return ok - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - -def main(): - parser = argparse.ArgumentParser(description="E2E multi-turn Responses API test") - parser.add_argument("--proxy-url", default=DEFAULT_PROXY_URL) - parser.add_argument("--model", default=DEFAULT_MODEL) - parser.add_argument("--trace-dir", type=Path, default=DEFAULT_TRACE_DIR) - parser.add_argument("--no-stream", action="store_true", help="Use non-streaming mode") - parser.add_argument("--max-turns", type=int, default=3, help="Max conversation turns") - args = parser.parse_args() - - stream = not args.no_stream - print(f"Session ID: {SESSION_ID}") - print(f"Proxy: {args.proxy_url}") - print(f"Model: {args.model}") - print(f"Stream: {stream}") - print(f"Trace dir: {args.trace_dir}") - print() - - # Initial user message - user_prompt = ( - "You are a helpful coding assistant with access to exec_command tool. " - "Please: 1) List files in the current directory, 2) Show the current " - "working directory, 3) Echo 'hello world'. Do all three using exec_command." - ) - - # Build the conversation input, starting with just the user message - input_items: list[dict] = [ - {"type": "message", "role": "user", "content": user_prompt}, - ] - - turns: list[tuple[list[dict], list[dict]]] = [] - turn = 0 - - while turn < args.max_turns: - turn += 1 - print(f"--- Turn {turn} ---") - print(f" Sending {len(input_items)} input items...") - - try: - response = send_responses_request( - args.proxy_url, args.model, input_items, SESSION_ID, - stream=stream, timeout=120.0, - ) - except Exception as e: - print(f" ERROR: {e}", file=sys.stderr) - sys.exit(1) - - tool_calls = extract_tool_calls(response) - text = extract_text(response) - - if text: - print(f" Text: {text[:100]}{'...' if len(text) > 100 else ''}") - - if not tool_calls: - print(f" No tool calls — conversation complete after {turn} turn(s)") - break - - print(f" Got {len(tool_calls)} tool call(s):") - for tc in tool_calls: - args_str = tc.get("arguments", "{}") - cmd = json.loads(args_str).get("cmd", "?") - print(f" - {tc['name']}({cmd})") - - # Simulate tool execution - tool_results = simulate_tool_results(tool_calls) - turns.append((tool_calls, tool_results)) - - # Rebuild input with full history for next turn - history_items = build_history(turns) - input_items = [ - {"type": "message", "role": "user", "content": user_prompt}, - *history_items, - ] - - # Small delay to let trace writes flush - time.sleep(1) - - print() - - # Wait for background trace writer to flush - print("Waiting 3s for trace writer to flush...") - time.sleep(3) - - # Load and verify traces - print(f"\n--- Verifying traces in {args.trace_dir} ---") - traces = load_session_traces(args.trace_dir, SESSION_ID) - - if not traces: - print(f" WARNING: No traces found for session {SESSION_ID}") - print(f" Make sure --enable-rl-logging and --rl-log-dir={args.trace_dir} are set") - sys.exit(1) - - print(f" Found {len(traces)} trace(s) for session {SESSION_ID}") - - ok = verify_traces(traces, turn) - - if ok: - print("\nPASS: All traces show correct multi-turn structure!") - print("The turn-merge bug is fixed.") - sys.exit(0) - else: - print("\nFAIL: Traces show merged turns — the bug is NOT fixed.") - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/tests/getting_started/test_getting_started.py b/tests/getting_started/test_getting_started.py index d862ac9ad..d41a648c0 100644 --- a/tests/getting_started/test_getting_started.py +++ b/tests/getting_started/test_getting_started.py @@ -12,7 +12,7 @@ def test_getting_started_documents_current_paths() -> None: ).read_text() assert guide.index("## Launcher Path") < guide.index("## Server Path") - assert 'uv tool install --python 3.12 "nemo-switchyard[cli,server]"' in guide + assert 'uv tool install --python 3.12 "nemo-switchyard[cli]"' in guide assert "switchyard launch claude --model switchyard" in guide assert "cargo install --locked switchyard-server" in guide assert "switchyard-server --config routes.toml --dry-run" in guide diff --git a/tests/readme/test_readme.py b/tests/readme/test_readme.py index fc7cbcd96..0cdfbb528 100644 --- a/tests/readme/test_readme.py +++ b/tests/readme/test_readme.py @@ -10,7 +10,7 @@ def test_readme_documents_current_paths() -> None: readme = (Path(__file__).resolve().parents[2] / "README.md").read_text() assert readme.index("### Launcher Path") < readme.index("### Server Path") - assert 'uv tool install --python 3.12 "nemo-switchyard[cli,server]"' in readme + assert 'uv tool install --python 3.12 "nemo-switchyard[cli]"' in readme assert "switchyard launch claude --model switchyard" in readme assert "cargo install --locked switchyard-server" in readme assert "switchyard-server --config routes.toml --dry-run" in readme diff --git a/tests/test_anthropic_native_llm_backend.py b/tests/test_anthropic_native_llm_backend.py deleted file mode 100644 index 54ed14a9d..000000000 --- a/tests/test_anthropic_native_llm_backend.py +++ /dev/null @@ -1,74 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Python binding tests for the Rust-owned Anthropic native backend.""" - -from __future__ import annotations - -import pytest - -from switchyard.lib.backends.multi_llm_backend import build_native_backend -from switchyard_rust.components import AnthropicNativeBackend, BackendFormat, LlmTarget -from switchyard_rust.core import ChatRequestType - - -def _anthropic_target(**overrides: object) -> LlmTarget: - data: dict[str, object] = { - "id": "anthropic", - "model": "claude-sonnet-test", - "format": BackendFormat.ANTHROPIC, - "base_url": "https://api.anthropic.com", - "api_key": "sk-ant-test", - "timeout_secs": 12.5, - } - data.update(overrides) - return LlmTarget(**data) - - -def test_constructs_from_resolved_anthropic_target() -> None: - target = _anthropic_target() - - backend = AnthropicNativeBackend(target) - - assert backend.target == target - assert backend.target.endpoint.base_url == "https://api.anthropic.com" - assert backend.target.endpoint.api_key == "sk-ant-test" - assert backend.target.endpoint.timeout_secs == 12.5 - - -def test_supported_request_types_is_anthropic_only() -> None: - backend = AnthropicNativeBackend(_anthropic_target()) - - assert backend.supported_request_types == [ChatRequestType.ANTHROPIC] - - -@pytest.mark.parametrize( - "target_format", - [BackendFormat.OPENAI, BackendFormat.AUTO, "openai", "auto"], -) -def test_rejects_unresolved_or_non_anthropic_target_format(target_format: object) -> None: - with pytest.raises(RuntimeError, match="resolved Anthropic format"): - AnthropicNativeBackend(_anthropic_target(format=target_format)) - - -def test_build_native_backend_selects_anthropic_binding() -> None: - backend = build_native_backend(_anthropic_target()) - - assert isinstance(backend, AnthropicNativeBackend) - - -def test_target_is_immutable_after_binding_construction() -> None: - target = _anthropic_target() - backend = AnthropicNativeBackend(target) - - with pytest.raises(AttributeError): - backend.target.model = "mutated" - assert backend.target.model == "claude-sonnet-test" - - -def test_request_types_are_value_objects_not_strings() -> None: - request_type = AnthropicNativeBackend(_anthropic_target()).supported_request_types[0] - - assert request_type is ChatRequestType.ANTHROPIC - assert request_type.value == "anthropic" - assert not isinstance(request_type, str) diff --git a/tests/test_anthropic_openai_translation.py b/tests/test_anthropic_openai_translation.py deleted file mode 100644 index 471b544a3..000000000 --- a/tests/test_anthropic_openai_translation.py +++ /dev/null @@ -1,101 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for Anthropic Messages to OpenAI Chat request translation.""" - -from __future__ import annotations - -from collections.abc import Iterable, Mapping -from typing import Any - -from switchyard_rust.translation import TranslationEngine - -ENGINE = TranslationEngine() - - -def _translate_anthropic_request_to_openai( - *, - messages: Iterable[Mapping[str, Any]], - **kwargs: Any, -) -> dict[str, Any]: - return ENGINE.translate_request( - "anthropic_messages", - "openai_chat", - {"messages": list(messages), **kwargs}, - ) - - -def _system_messages(result: dict) -> list[dict]: - return [m for m in result["messages"] if m.get("role") == "system"] - - -def test_system_as_string(): - result = _translate_anthropic_request_to_openai( - messages=[{"role": "user", "content": "Hello"}], - system="You are helpful.", - model="claude-3-5-sonnet-20241022", - max_tokens=100, - ) - sys_msgs = _system_messages(result) - assert len(sys_msgs) == 1 - assert sys_msgs[0]["content"] == "You are helpful." - - -def test_system_as_list_of_blocks(): - result = _translate_anthropic_request_to_openai( - messages=[{"role": "user", "content": "Hello"}], - system=[{"type": "text", "text": "You are helpful."}], - model="claude-3-5-sonnet-20241022", - max_tokens=100, - ) - sys_msgs = _system_messages(result) - assert len(sys_msgs) == 1 - assert "You are helpful." in sys_msgs[0]["content"] - - -def test_system_as_tuple_of_blocks_not_silently_dropped(): - """Tuples are valid Iterable[TextBlockParam] but isinstance(..., list) is False. - - Before the fix this test fails — the system prompt is silently dropped. - """ - system = ({"type": "text", "text": "You are helpful."},) - result = _translate_anthropic_request_to_openai( - messages=[{"role": "user", "content": "Hello"}], - system=system, - model="claude-3-5-sonnet-20241022", - max_tokens=100, - ) - sys_msgs = _system_messages(result) - assert len(sys_msgs) == 1, ( - f"System prompt from tuple was silently dropped. messages={result['messages']}" - ) - assert "You are helpful." in sys_msgs[0]["content"] - - -def test_system_as_generator_of_blocks_not_silently_dropped(): - """Generators are valid Iterable[TextBlockParam] but isinstance(..., list) is False.""" - - def block_gen(): - yield {"type": "text", "text": "You are a generator."} - - result = _translate_anthropic_request_to_openai( - messages=[{"role": "user", "content": "Hello"}], - system=block_gen(), - model="claude-3-5-sonnet-20241022", - max_tokens=100, - ) - sys_msgs = _system_messages(result) - assert len(sys_msgs) == 1, ( - f"System prompt from generator was silently dropped. messages={result['messages']}" - ) - assert "You are a generator." in sys_msgs[0]["content"] - - -def test_system_none_produces_no_system_message(): - result = _translate_anthropic_request_to_openai( - messages=[{"role": "user", "content": "Hello"}], - system=None, - model="claude-3-5-sonnet-20241022", - max_tokens=100, - ) - assert _system_messages(result) == [] diff --git a/tests/test_anthropic_output_config_strip.py b/tests/test_anthropic_output_config_strip.py deleted file mode 100644 index e426926c9..000000000 --- a/tests/test_anthropic_output_config_strip.py +++ /dev/null @@ -1,34 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Inbound Anthropic bodies must drop output_config.format (a Claude Code -structured-output schema upstream Anthropic model groups reject) while keeping -output_config.effort.""" - -from switchyard.lib.endpoints.anthropic_messages_endpoint import ( - _strip_unsupported_output_config, -) - - -def test_strips_format_keeps_effort(): - body = {"model": "x", "output_config": {"effort": "high", "format": {"schema": {}}}} - _strip_unsupported_output_config(body) - assert body["output_config"] == {"effort": "high"} - - -def test_drops_output_config_when_only_format(): - body = {"model": "x", "output_config": {"format": {"schema": {}}}} - _strip_unsupported_output_config(body) - assert "output_config" not in body - - -def test_noop_without_output_config(): - body = {"model": "x", "messages": []} - _strip_unsupported_output_config(body) - assert body == {"model": "x", "messages": []} - - -def test_noop_when_no_format_key(): - body = {"model": "x", "output_config": {"effort": "high"}} - _strip_unsupported_output_config(body) - assert body["output_config"] == {"effort": "high"} diff --git a/tests/test_anthropic_probe.py b/tests/test_anthropic_probe.py deleted file mode 100644 index fc323d511..000000000 --- a/tests/test_anthropic_probe.py +++ /dev/null @@ -1,156 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for :func:`probe_anthropic_messages_support`. - -The probe is a single HTTP POST with empty body + real auth; responses -partition cleanly into "endpoint wired" vs "not wired / broken". Tests -mock httpx at the ``AsyncClient`` level. -""" - -from __future__ import annotations - -import logging -from unittest.mock import AsyncMock, MagicMock, patch - -import httpx -import pytest - -from switchyard.lib.backends.backend_format_resolver import ( - probe_anthropic_messages_support, - strip_v1_suffix, -) - - -def _fake_response(status_code: int) -> MagicMock: - resp = MagicMock() - resp.status_code = status_code - return resp - - -def _patch_httpx(response: MagicMock | Exception): - """Return a context manager patching AsyncClient.post. - - If ``response`` is an Exception, it's raised from the POST; otherwise - it's returned. - """ - async_client = MagicMock() - if isinstance(response, Exception): - async_client.post = AsyncMock(side_effect=response) - else: - async_client.post = AsyncMock(return_value=response) - - # AsyncClient is used as an async context manager - async_cm = MagicMock() - async_cm.__aenter__ = AsyncMock(return_value=async_client) - async_cm.__aexit__ = AsyncMock(return_value=False) - - return patch.object(httpx, "AsyncClient", return_value=async_cm) - - -class TestProbeAnthropicMessagesSupport: - async def test_404_returns_false(self): - with _patch_httpx(_fake_response(404)): - assert await probe_anthropic_messages_support( - base_url="https://x.example/v1", api_key="sk-test", - ) is False - - async def test_400_returns_true(self): - # Endpoint exists, validator ran and rejected our empty body. - with _patch_httpx(_fake_response(400)): - assert await probe_anthropic_messages_support( - base_url="https://x.example/v1", api_key="sk-test", - ) is True - - async def test_422_returns_true(self): - with _patch_httpx(_fake_response(422)): - assert await probe_anthropic_messages_support( - base_url="https://x.example/v1", api_key="sk-test", - ) is True - - async def test_200_returns_true(self): - with _patch_httpx(_fake_response(200)): - assert await probe_anthropic_messages_support( - base_url="https://x.example/v1", api_key="sk-test", - ) is True - - async def test_401_returns_false(self): - with _patch_httpx(_fake_response(401)): - assert await probe_anthropic_messages_support( - base_url="https://x.example/v1", api_key="sk-test", - ) is False - - async def test_5xx_returns_false(self): - with _patch_httpx(_fake_response(500)): - assert await probe_anthropic_messages_support( - base_url="https://x.example/v1", api_key="sk-test", - ) is False - - async def test_timeout_returns_false(self): - with _patch_httpx(httpx.TimeoutException("slow")): - assert await probe_anthropic_messages_support( - base_url="https://x.example/v1", api_key="sk-test", - ) is False - - async def test_timeout_does_not_log_warning(self, caplog): - caplog.set_level(logging.WARNING) - with _patch_httpx(httpx.TimeoutException("slow")): - assert await probe_anthropic_messages_support( - base_url="https://x.example/v1", api_key="sk-test", - ) is False - - assert "Anthropic /v1/messages probe" not in caplog.text - - async def test_connection_error_returns_false(self): - with _patch_httpx(httpx.ConnectError("dns fail")): - assert await probe_anthropic_messages_support( - base_url="https://x.example/v1", api_key="sk-test", - ) is False - - async def test_probe_url_is_root_plus_v1_messages(self): - """``--base-url`` follows OpenAI convention (ends in ``/v1``) but - the probe's target is ``{root}/v1/messages``. Make sure we don't - produce ``/v1/v1/messages`` or strip essential path components. - """ - captured: dict = {} - - async def fake_post(url, headers, json): - captured["url"] = url - return _fake_response(400) - - async_client = MagicMock() - async_client.post = AsyncMock(side_effect=fake_post) - async_cm = MagicMock() - async_cm.__aenter__ = AsyncMock(return_value=async_client) - async_cm.__aexit__ = AsyncMock(return_value=False) - - with patch.object(httpx, "AsyncClient", return_value=async_cm): - await probe_anthropic_messages_support( - base_url="https://inference-api.nvidia.com/v1", api_key="sk-test", - ) - assert captured["url"] == "https://inference-api.nvidia.com/v1/messages" - - @pytest.mark.parametrize("status", [402, 403, 405, 429]) - async def test_other_4xx_treated_as_endpoint_exists(self, status): - with _patch_httpx(_fake_response(status)): - assert await probe_anthropic_messages_support( - base_url="https://x.example/v1", api_key="sk-test", - ) is True - - -class TestStripV1Suffix: - def test_strips_trailing_v1(self): - assert strip_v1_suffix("https://host/v1") == "https://host" - - def test_strips_trailing_v1_with_slash(self): - assert strip_v1_suffix("https://host/v1/") == "https://host" - - def test_no_v1_suffix_untouched(self): - assert strip_v1_suffix("https://host") == "https://host" - - def test_trailing_slash_stripped_even_when_no_v1(self): - assert strip_v1_suffix("https://host/") == "https://host" - - def test_v1_in_middle_of_path_untouched(self): - # Defensive: /v1 embedded in a longer path must not be stripped. - assert strip_v1_suffix("https://host/v1/sub") == "https://host/v1/sub" diff --git a/tests/test_backend_format_resolver.py b/tests/test_backend_format_resolver.py deleted file mode 100644 index 4d1b1f714..000000000 --- a/tests/test_backend_format_resolver.py +++ /dev/null @@ -1,399 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import httpx -import pytest - -from switchyard.lib import startup_timing -from switchyard.lib.backends import ( - backend_format_resolver as resolver_mod, -) -from switchyard.lib.backends.llm_target import ( - BackendFormat, - LlmTarget, -) - - -class _RecordingProbe: - def __init__(self, result: bool) -> None: - self.result = result - self.calls: list[dict[str, object]] = [] - - def __call__(self, **kwargs: object) -> bool: - self.calls.append(dict(kwargs)) - return self.result - - -def _no_probe(name: str): - def fail(**_: object) -> bool: - pytest.fail(f"explicit backend formats must not probe ({name})") - return fail - - -# --------------------------------------------------------------------------- -# Explicit format — no probing at all -# --------------------------------------------------------------------------- - - -def test_explicit_format_does_not_probe(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(resolver_mod, "probe_openai_chat_completions_support_sync", - _no_probe("chat-completions")) - monkeypatch.setattr(resolver_mod, "probe_anthropic_messages_support_sync", - _no_probe("anthropic")) - monkeypatch.setattr(resolver_mod, "probe_openai_responses_support_sync", - _no_probe("responses")) - - resolution = resolver_mod.BackendFormatResolver.resolve( - LlmTarget(model="m", format=BackendFormat.ANTHROPIC), - ) - - assert resolution.format is BackendFormat.ANTHROPIC - assert resolution.reason == "backend format is explicitly configured" - - -# --------------------------------------------------------------------------- -# Model-prefix fast-path — skips all probes -# --------------------------------------------------------------------------- - - -@pytest.mark.parametrize("model", [ - "anthropic/claude-3-5-sonnet", - "anthropic/claude-haiku", - "claude-3-opus-20240229", - "claude-sonnet-4-6", -]) -def test_auto_model_prefix_anthropic_skips_probes( - monkeypatch: pytest.MonkeyPatch, model: str -) -> None: - """anthropic/ and claude prefixes → ANTHROPIC without probing.""" - monkeypatch.setattr(resolver_mod, "probe_openai_chat_completions_support_sync", - _no_probe("chat-completions")) - monkeypatch.setattr(resolver_mod, "probe_anthropic_messages_support_sync", - _no_probe("anthropic")) - monkeypatch.setattr(resolver_mod, "probe_openai_responses_support_sync", - _no_probe("responses")) - - resolution = resolver_mod.BackendFormatResolver.resolve( - LlmTarget( - model=model, - format=BackendFormat.AUTO, - base_url="https://provider.test/v1", - api_key="sk-test", # pragma: allowlist secret - ), - ) - - assert resolution.format is BackendFormat.ANTHROPIC - assert "prefix" in resolution.reason - - -@pytest.mark.parametrize("model", [ - "openrouter/anthropic/claude-3-5-sonnet", - "aws/anthropic/bedrock-claude-opus-4-7", -]) -def test_auto_gateway_anthropic_model_probes_chat_completions_first( - monkeypatch: pytest.MonkeyPatch, model: str -) -> None: - """Gateway-namespaced models are NOT fast-pathed; Chat Completions probe runs first.""" - chat_probe = _RecordingProbe(result=True) - monkeypatch.setattr(resolver_mod, "probe_openai_chat_completions_support_sync", chat_probe) - monkeypatch.setattr(resolver_mod, "probe_anthropic_messages_support_sync", - _no_probe("anthropic")) - monkeypatch.setattr(resolver_mod, "probe_openai_responses_support_sync", - _no_probe("responses")) - - resolution = resolver_mod.BackendFormatResolver.resolve( - LlmTarget( - model=model, - format=BackendFormat.AUTO, - base_url="https://openrouter.ai/api/v1", - api_key="sk-test", # pragma: allowlist secret - ), - ) - - assert resolution.format is BackendFormat.OPENAI - assert len(chat_probe.calls) == 1 - - -# --------------------------------------------------------------------------- -# AUTO probe order: Chat Completions → Anthropic → Responses -# --------------------------------------------------------------------------- - - -def test_auto_chat_completions_wins_when_supported( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Chat Completions probe succeeds → OPENAI; remaining probes are never called.""" - monkeypatch.setattr(resolver_mod, "probe_openai_chat_completions_support_sync", - _RecordingProbe(result=True)) - monkeypatch.setattr(resolver_mod, "probe_anthropic_messages_support_sync", - _no_probe("anthropic")) - monkeypatch.setattr(resolver_mod, "probe_openai_responses_support_sync", - _no_probe("responses")) - - resolution = resolver_mod.BackendFormatResolver.resolve( - LlmTarget( - model="nvidia/nvidia/nemotron-nano-9b-v2", - format=BackendFormat.AUTO, - base_url="https://inference-api.nvidia.com/v1", - api_key="sk-test", # pragma: allowlist secret - ), - ) - - assert resolution.format is BackendFormat.OPENAI - - -def test_auto_resolves_to_anthropic_when_chat_completions_unavailable( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Chat Completions probe fails, Anthropic probe succeeds → ANTHROPIC.""" - probe = _RecordingProbe(result=True) - monkeypatch.setattr(resolver_mod, "probe_openai_chat_completions_support_sync", - _RecordingProbe(result=False)) - monkeypatch.setattr(resolver_mod, "probe_anthropic_messages_support_sync", probe) - monkeypatch.setattr(resolver_mod, "probe_openai_responses_support_sync", - _no_probe("responses")) - - resolution = resolver_mod.BackendFormatResolver.resolve( - LlmTarget( - model="some-model", - format=BackendFormat.AUTO, - base_url="https://api.anthropic.com/v1", - api_key="sk-test", # pragma: allowlist secret - ), - ) - - assert resolution.format is BackendFormat.ANTHROPIC - assert probe.calls == [{ - "base_url": "https://api.anthropic.com/v1", - "api_key": "sk-test", - "model": "some-model", - "timeout_s": 3.0, - }] - - -def test_auto_resolves_to_responses_when_only_responses_probe_succeeds( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Chat Completions and Anthropic probes fail, Responses probe succeeds → RESPONSES.""" - monkeypatch.setattr(resolver_mod, "probe_openai_chat_completions_support_sync", - _RecordingProbe(result=False)) - monkeypatch.setattr(resolver_mod, "probe_anthropic_messages_support_sync", - _RecordingProbe(result=False)) - monkeypatch.setattr(resolver_mod, "probe_openai_responses_support_sync", - lambda **_: True) - - resolution = resolver_mod.BackendFormatResolver.resolve( - LlmTarget( - model="some-model", - format=BackendFormat.AUTO, - base_url="https://provider.test/v1", - api_key="sk-test", # pragma: allowlist secret - ), - ) - - assert resolution.format is BackendFormat.RESPONSES - - -def test_auto_falls_back_to_openai_when_all_probes_fail( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """All three probes fail → OPENAI (Chat Completions assumed universal).""" - monkeypatch.setattr(resolver_mod, "probe_openai_chat_completions_support_sync", - _RecordingProbe(result=False)) - monkeypatch.setattr(resolver_mod, "probe_anthropic_messages_support_sync", - _RecordingProbe(result=False)) - monkeypatch.setattr(resolver_mod, "probe_openai_responses_support_sync", - lambda **_: False) - - resolution = resolver_mod.BackendFormatResolver.resolve( - LlmTarget( - model="some-model", - format=BackendFormat.AUTO, - base_url="https://provider.test/v1", - api_key="sk-test", # pragma: allowlist secret - ), - ) - - assert resolution.format is BackendFormat.OPENAI - - -def test_auto_chat_completions_timeout_assumes_openai( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """A Chat Completions probe that times out (raises, not a fast 404) resolves - to OPENAI without stacking the slower /v1/messages and /v1/responses probes. - A 404 returns False and still falls through — that path is covered by - test_auto_resolves_to_anthropic_when_chat_completions_unavailable.""" - def timed_out(**_: object) -> bool: - raise httpx.ReadTimeout("probe timed out") - - monkeypatch.setattr(resolver_mod, "probe_openai_chat_completions_support_sync", - timed_out) - monkeypatch.setattr(resolver_mod, "probe_anthropic_messages_support_sync", - _no_probe("anthropic")) - monkeypatch.setattr(resolver_mod, "probe_openai_responses_support_sync", - _no_probe("responses")) - - resolution = resolver_mod.BackendFormatResolver.resolve( - LlmTarget( - model="slow-endpoint-model", - format=BackendFormat.AUTO, - base_url="https://slow.test/v1", - api_key="sk-test", # pragma: allowlist secret - timeout_secs=0.05, - ), - ) - - assert resolution.format is BackendFormat.OPENAI - assert "timed out" in resolution.reason - - -def test_auto_records_each_probe_in_startup_timing( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """With timing on, each probe reached leaves its own mark so - `launch --startup-timing` can show a per-route breakdown.""" - monkeypatch.setattr(startup_timing, "enabled", True) - startup_timing._marks.clear() - - monkeypatch.setattr(resolver_mod, "probe_openai_chat_completions_support_sync", - _RecordingProbe(result=False)) - monkeypatch.setattr(resolver_mod, "probe_anthropic_messages_support_sync", - _RecordingProbe(result=True)) - monkeypatch.setattr(resolver_mod, "probe_openai_responses_support_sync", - _no_probe("responses")) - - resolution = resolver_mod.BackendFormatResolver.resolve( - LlmTarget( - model="some-model", - format=BackendFormat.AUTO, - base_url="https://api.anthropic.com/v1", - api_key="sk-test", # pragma: allowlist secret - ), - ) - - assert resolution.format is BackendFormat.ANTHROPIC - labels = [label for label, _ in startup_timing._marks] - assert labels == ["chain init", "probe: /v1/chat/completions", "probe: /v1/messages"] - startup_timing._marks.clear() - - -# --------------------------------------------------------------------------- -# Missing inputs -# --------------------------------------------------------------------------- - - -def test_auto_format_requires_base_url() -> None: - with pytest.raises(ValueError, match="requires base_url"): - resolver_mod.BackendFormatResolver.resolve( - LlmTarget( - model="m", - format=BackendFormat.AUTO, - api_key="sk-test", # pragma: allowlist secret - ), - ) - - -def test_auto_format_requires_api_key() -> None: - with pytest.raises(ValueError, match="requires api_key"): - resolver_mod.BackendFormatResolver.resolve( - LlmTarget( - model="m", - format=BackendFormat.AUTO, - base_url="https://provider.test/v1", - ), - ) - - -# --------------------------------------------------------------------------- -# Model + timeout forwarding -# --------------------------------------------------------------------------- - - -def test_auto_forwards_endpoint_timeout_to_all_probes( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """tier.endpoint.timeout_secs is forwarded as timeout_s to all three probes.""" - chat_probe = _RecordingProbe(result=False) - anthropic_probe = _RecordingProbe(result=False) - responses_probe = _RecordingProbe(result=False) - monkeypatch.setattr(resolver_mod, "probe_openai_chat_completions_support_sync", chat_probe) - monkeypatch.setattr(resolver_mod, "probe_anthropic_messages_support_sync", anthropic_probe) - monkeypatch.setattr(resolver_mod, "probe_openai_responses_support_sync", responses_probe) - - resolver_mod.BackendFormatResolver.resolve( - LlmTarget( - model="nvidia/nvidia/nemotron-nano", - format=BackendFormat.AUTO, - base_url="https://integrate.api.nvidia.com/v1", - api_key="sk-test", # pragma: allowlist secret - timeout_secs=30.0, - ), - ) - - assert chat_probe.calls[0]["timeout_s"] == 30.0 - assert anthropic_probe.calls[0]["timeout_s"] == 30.0 - assert responses_probe.calls[0]["timeout_s"] == 30.0 - - -def test_auto_passes_model_to_all_probes( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """model is forwarded to all three probes so mixed providers are correctly scoped.""" - chat_probe = _RecordingProbe(result=False) - anthropic_probe = _RecordingProbe(result=False) - responses_probe = _RecordingProbe(result=False) - monkeypatch.setattr(resolver_mod, "probe_openai_chat_completions_support_sync", chat_probe) - monkeypatch.setattr(resolver_mod, "probe_anthropic_messages_support_sync", anthropic_probe) - monkeypatch.setattr(resolver_mod, "probe_openai_responses_support_sync", responses_probe) - - resolver_mod.BackendFormatResolver.resolve( - LlmTarget( - model="openrouter/non-claude-model", - format=BackendFormat.AUTO, - base_url="https://openrouter.ai/api/v1", - api_key="sk-test", # pragma: allowlist secret - ), - ) - - assert chat_probe.calls[0]["model"] == "openrouter/non-claude-model" - assert anthropic_probe.calls[0]["model"] == "openrouter/non-claude-model" - assert responses_probe.calls[0]["model"] == "openrouter/non-claude-model" - - -# --------------------------------------------------------------------------- -# _interpret_status — model-error body detection -# --------------------------------------------------------------------------- - - -def test_interpret_status_400_with_model_not_found_body_returns_false() -> None: - """400 whose body names the model as not found is a probe failure.""" - import json as _json - - body = _json.dumps({ - "error": {"type": "invalid_request_error", "message": "model: gpt-4o not found"}, - }).encode() - assert resolver_mod._interpret_status(400, body) is False - - -def test_interpret_status_400_with_not_found_error_type_returns_false() -> None: - """400/404-style 'not_found_error' type is treated as a probe failure.""" - import json as _json - - body = _json.dumps({"error": {"type": "not_found_error", "message": "model not found"}}).encode() - assert resolver_mod._interpret_status(400, body) is False - - -def test_interpret_status_400_with_field_error_returns_true() -> None: - """400 about missing fields (not the model) means route exists — True.""" - import json as _json - - body = _json.dumps({ - "error": {"type": "invalid_request_error", "message": "messages: field required"}, - }).encode() - assert resolver_mod._interpret_status(400, body) is True - - -def test_interpret_status_400_without_body_returns_true() -> None: - """400 with no body (legacy call-site) preserves old behaviour — True.""" - assert resolver_mod._interpret_status(400) is True diff --git a/tests/test_build_and_serve.py b/tests/test_build_and_serve.py deleted file mode 100644 index 547f8ddd7..000000000 --- a/tests/test_build_and_serve.py +++ /dev/null @@ -1,384 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for ``switchyard.server.server_util.build_and_serve``. - -``build_and_serve`` is the second app-construction code path (the first is -``build_switchyard_app`` directly). PR #28 fixed a regression where this -function lazy-imported three modules that no longer exist after the -open-source cleanup, raising ``ModuleNotFoundError`` for every CLI subcommand -that called it (e.g., ``random-routing``). - -These tests exercise the function offline by patching out ``uvicorn.run``, -capturing the FastAPI app it would have served, and driving real HTTP -requests through that app via ``httpx.ASGITransport``. A regression in any -of build_and_serve's imports, app construction, or extra-endpoint wiring -fails one of these tests. -""" - -from __future__ import annotations - -import argparse -from collections.abc import AsyncIterator -from typing import Any - -import httpx -import pytest -from fastapi import APIRouter, FastAPI -from openai.types.chat import ChatCompletion -from openai.types.chat.chat_completion import Choice -from openai.types.chat.chat_completion_message import ChatCompletionMessage -from openai.types.completion_usage import CompletionUsage - -from switchyard.lib.endpoints.base import Endpoint -from switchyard.lib.proxy_context import ProxyContext -from switchyard.lib.roles import LLMBackend -from switchyard.lib.switchyard import Switchyard -from switchyard.server import server_util -from switchyard_rust.core import ChatRequest, ChatRequestType, ChatResponse -from switchyard_rust.translation import TranslationEngine - -_ALL_REQUEST_TYPES = [ - ChatRequestType.OPENAI_CHAT, - ChatRequestType.OPENAI_RESPONSES, - ChatRequestType.ANTHROPIC, -] - -# --------------------------------------------------------------------------- -# Fakes -# --------------------------------------------------------------------------- - - -class _StubBackend(LLMBackend): - def supported_request_types(self) -> list[ChatRequestType]: - return list(_ALL_REQUEST_TYPES) - - async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: - completion = ChatCompletion( - id="chatcmpl-stub", - object="chat.completion", - created=1700000000, - model="stub", - choices=[ - Choice( - index=0, - message=ChatCompletionMessage(role="assistant", content="ok"), - finish_reason="stop", - ) - ], - usage=CompletionUsage(prompt_tokens=1, completion_tokens=1, total_tokens=2), - ) - return ChatResponse.openai_completion(completion) - - -class _CountingStubBackend(_StubBackend): - """Variant of _StubBackend that records each call so tests can assert short-circuit.""" - - def __init__(self) -> None: - self.call_count = 0 - - async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: - self.call_count += 1 - return await super().call(ctx, request) - - -class _SentinelEndpoint(Endpoint): - """Adds ``GET /sentinel`` so tests can verify ``extra_endpoints`` are wired.""" - - def register(self, app: FastAPI) -> None: - router = APIRouter() - - @router.get("/sentinel") - async def _sentinel() -> dict[str, str]: - return {"sentinel": "ok"} - - app.include_router(router) - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _ns(**overrides: Any) -> argparse.Namespace: - """Build the argparse namespace ``build_and_serve`` expects.""" - defaults = {"host": "127.0.0.1", "port": 4000, "reload": False, "workers": 1} - defaults.update(overrides) - return argparse.Namespace(**defaults) - - -def _capture_uvicorn(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: - """Patch ``uvicorn.run`` to capture its kwargs without starting a server.""" - captured: dict[str, Any] = {} - - def _fake_run(app: FastAPI, **kwargs: Any) -> None: - captured["app"] = app - captured["kwargs"] = kwargs - - import uvicorn - - monkeypatch.setattr(uvicorn, "run", _fake_run) - return captured - - -def _switchyard() -> Switchyard: - return Switchyard(backend=_StubBackend(), translator=TranslationEngine()) - - -# --------------------------------------------------------------------------- -# Tests -# --------------------------------------------------------------------------- - - -def test_build_and_serve_imports_succeed(monkeypatch: pytest.MonkeyPatch) -> None: - """Direct guard for PR #28: ``build_and_serve`` must not raise on import. - - The previous regression was a lazy import of deleted modules - (``endpoint_sets``, ``server``, ``server_config``); the function ran - cleanly until called, then crashed at the lazy-import line. Calling - it with a stub uvicorn proves all of its imports resolve. - """ - captured = _capture_uvicorn(monkeypatch) - server_util.build_and_serve(_ns(), _switchyard()) - assert "app" in captured, "build_and_serve never reached uvicorn.run" - - -def test_build_and_serve_passes_host_and_port_through(monkeypatch: pytest.MonkeyPatch) -> None: - captured = _capture_uvicorn(monkeypatch) - server_util.build_and_serve(_ns(host="0.0.0.0", port=5555), _switchyard()) - assert captured["kwargs"]["host"] == "0.0.0.0" - assert captured["kwargs"]["port"] == 5555 - - -def test_build_and_serve_defaults_port_to_4000_when_none( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Documented default in --port help; regression-guard the fallback path.""" - captured = _capture_uvicorn(monkeypatch) - server_util.build_and_serve(_ns(port=None), _switchyard()) - assert captured["kwargs"]["port"] == 4000 - - -def test_build_and_serve_workers_default_to_one(monkeypatch: pytest.MonkeyPatch) -> None: - """Namespace without ``workers`` must not raise — ``getattr`` fallback applies.""" - captured = _capture_uvicorn(monkeypatch) - args = argparse.Namespace(host="127.0.0.1", port=4000, reload=False) - server_util.build_and_serve(args, _switchyard()) - assert captured["kwargs"]["workers"] == 1 - - -def test_build_and_serve_registers_extra_endpoints(monkeypatch: pytest.MonkeyPatch) -> None: - """``extra_endpoints`` must be registered onto the app before serving.""" - captured = _capture_uvicorn(monkeypatch) - server_util.build_and_serve( - _ns(), - _switchyard(), - extra_endpoints=[_SentinelEndpoint()], - ) - routes = {getattr(r, "path", None) for r in captured["app"].routes} - assert "/sentinel" in routes - - -@pytest.fixture -async def served_client( - monkeypatch: pytest.MonkeyPatch, -) -> AsyncIterator[httpx.AsyncClient]: - """Drive the app ``build_and_serve`` would have served through ASGI. - - The full app boot path runs (build_switchyard_app → endpoint - registration → app.state wiring) but uvicorn never starts. - """ - captured = _capture_uvicorn(monkeypatch) - server_util.build_and_serve(_ns(), _switchyard(), extra_endpoints=[_SentinelEndpoint()]) - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=captured["app"]), - base_url="http://test", - ) as client: - yield client - - -class TestBuiltAppRoundTrips: - """Round-trip through the app build_and_serve assembled. - - Catches the kind of failure PR #28 fixed: server starts cleanly, but - every request hits ``AttributeError`` because of an internal wiring - mismatch. A unit test on ``build_and_serve`` alone wouldn't have - caught the ``app.state.switchyard`` vs ``app.state.switchyard`` - bug — only a real request through the assembled app does. - """ - - async def test_health(self, served_client: httpx.AsyncClient) -> None: - resp = await served_client.get("/health") - assert resp.status_code == 200 - assert resp.json() == {"status": "ok"} - - async def test_openai_chat_completions(self, served_client: httpx.AsyncClient) -> None: - resp = await served_client.post( - "/v1/chat/completions", - json={ - "model": "any", - "messages": [{"role": "user", "content": "hi"}], - }, - ) - assert resp.status_code == 200, resp.text - assert resp.json()["choices"][0]["message"]["content"] == "ok" - - async def test_anthropic_messages(self, served_client: httpx.AsyncClient) -> None: - resp = await served_client.post( - "/v1/messages", - json={ - "model": "any", - "max_tokens": 16, - "messages": [{"role": "user", "content": "hi"}], - }, - ) - assert resp.status_code == 200, resp.text - assert resp.json()["content"][0]["text"] == "ok" - - async def test_extra_endpoint_reachable(self, served_client: httpx.AsyncClient) -> None: - resp = await served_client.get("/sentinel") - assert resp.status_code == 200 - assert resp.json() == {"sentinel": "ok"} - - -class TestInvalidRequestBody: - """Malformed or non-object JSON bodies must return 400 with a structured error envelope. - - REQ-AG3: agents must receive structured errors with retry semantics so they can - distinguish client-side parse bugs (no retry) from transient server failures (retry). - These tests confirm the fix for the bare-500 regression reported in bug 6267258. - """ - - _ENDPOINTS = [ - "/v1/chat/completions", - "/v1/messages", - "/v1/responses", - ] - - @pytest.mark.parametrize("path", _ENDPOINTS) - async def test_malformed_json_returns_400( - self, served_client: httpx.AsyncClient, path: str - ) -> None: - resp = await served_client.post( - path, - content="{invalid json,,", - headers={"Content-Type": "application/json"}, - ) - assert resp.status_code == 400 - assert resp.headers["content-type"].startswith("application/json") - body = resp.json() - assert body["error"]["type"] == "invalid_request_error" - assert body["error"]["code"] == "invalid_body" - - @pytest.mark.parametrize("path", _ENDPOINTS) - async def test_json_array_body_returns_400( - self, served_client: httpx.AsyncClient, path: str - ) -> None: - resp = await served_client.post( - path, - content='["not", "an", "object"]', - headers={"Content-Type": "application/json"}, - ) - assert resp.status_code == 400 - assert resp.headers["content-type"].startswith("application/json") - body = resp.json() - assert body["error"]["type"] == "invalid_request_error" - assert body["error"]["code"] == "invalid_body" - - async def test_server_stays_healthy_after_bad_request( - self, served_client: httpx.AsyncClient - ) -> None: - await served_client.post( - "/v1/chat/completions", - content="{bad json", - headers={"Content-Type": "application/json"}, - ) - resp = await served_client.get("/health") - assert resp.status_code == 200 - - -@pytest.fixture -async def counting_client( - monkeypatch: pytest.MonkeyPatch, -) -> AsyncIterator[tuple[httpx.AsyncClient, _CountingStubBackend]]: - """Like served_client but exposes a call-counting backend for short-circuit checks.""" - captured: dict[str, Any] = {} - - def _fake_run(app: FastAPI, **kwargs: Any) -> None: - captured["app"] = app - captured["kwargs"] = kwargs - - import uvicorn - - monkeypatch.setattr(uvicorn, "run", _fake_run) - backend = _CountingStubBackend() - sw = Switchyard(backend=backend, translator=TranslationEngine()) - server_util.build_and_serve(_ns(), sw) - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=captured["app"]), - base_url="http://test", - ) as client: - yield client, backend - - -class TestEmptyMessages: - """Empty messages array must short-circuit with a structured 400 before reaching the backend. - - REQ-AG3: agents must distinguish client-side errors (no retry) from server failures - (retry). An empty messages array is a client bug — reporting it as 500 breaks - agent retry logic. This class pins the fix and confirms dispatch is skipped. - """ - - async def test_openai_chat_empty_messages_returns_400( - self, - counting_client: tuple[httpx.AsyncClient, _CountingStubBackend], - ) -> None: - client, backend = counting_client - resp = await client.post( - "/v1/chat/completions", - json={"model": "any", "messages": []}, - ) - assert resp.status_code == 400 - assert resp.headers["content-type"].startswith("application/json") - body = resp.json() - assert body["error"]["type"] == "invalid_request_error" - assert body["error"]["code"] == "empty_messages" - assert "messages" in body["error"]["message"] - assert backend.call_count == 0, "backend must not be invoked for empty messages" - - async def test_anthropic_messages_empty_messages_returns_400( - self, - counting_client: tuple[httpx.AsyncClient, _CountingStubBackend], - ) -> None: - client, backend = counting_client - resp = await client.post( - "/v1/messages", - json={"model": "any", "max_tokens": 16, "messages": []}, - ) - assert resp.status_code == 400 - assert resp.headers["content-type"].startswith("application/json") - body = resp.json() - assert body["error"]["type"] == "invalid_request_error" - assert body["error"]["code"] == "empty_messages" - assert backend.call_count == 0, "backend must not be invoked for empty messages" - - async def test_non_empty_messages_still_succeed( - self, - counting_client: tuple[httpx.AsyncClient, _CountingStubBackend], - ) -> None: - client, backend = counting_client - resp = await client.post( - "/v1/chat/completions", - json={"model": "any", "messages": [{"role": "user", "content": "hi"}]}, - ) - assert resp.status_code == 200, resp.text - assert backend.call_count == 1 - - async def test_server_stays_healthy_after_empty_messages( - self, - counting_client: tuple[httpx.AsyncClient, _CountingStubBackend], - ) -> None: - client, _ = counting_client - await client.post("/v1/chat/completions", json={"model": "any", "messages": []}) - resp = await client.get("/health") - assert resp.status_code == 200 diff --git a/tests/test_chat_request.py b/tests/test_chat_request.py deleted file mode 100644 index b315e94f9..000000000 --- a/tests/test_chat_request.py +++ /dev/null @@ -1,213 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for Rust-backed ChatRequest values.""" - -import pytest - -from switchyard_rust.core import ( - ChatRequest, - ChatRequestType, - request_type_enum, - request_type_matches, -) - - -class TestChatRequestType: - def test_enum_values(self): - assert ChatRequestType.OPENAI_CHAT.value == "openai_chat" - assert ChatRequestType.OPENAI_RESPONSES.value == "openai_responses" - assert ChatRequestType.ANTHROPIC.value == "anthropic" - - def test_enum_members(self): - assert [ - request_type_enum("openai_chat").value, - request_type_enum("openai_responses").value, - request_type_enum("anthropic").value, - ] == ["openai_chat", "openai_responses", "anthropic"] - - def test_unknown_request_type_is_rejected(self): - with pytest.raises(ValueError): - request_type_enum("not_a_real_format") - - -class TestChatRequestConstructor: - def test_cannot_instantiate(self): - with pytest.raises(TypeError): - ChatRequest() # type: ignore[abstract] - - -class TestOpenAIChatBinding: - @pytest.fixture() - def body(self): - return { - "model": "gpt-4o", - "messages": [{"role": "user", "content": "Hello"}], - } - - def test_request_type(self, body): - req = ChatRequest.openai_chat(body) - assert request_type_matches(req, ChatRequestType.OPENAI_CHAT) - - def test_body_access(self, body): - req = ChatRequest.openai_chat(body) - assert req.body == body - assert req.body is not body - - def test_body_field_access(self, body): - req = ChatRequest.openai_chat(body) - assert req.body["model"] == "gpt-4o" - assert req.body["messages"][0]["role"] == "user" - - def test_isinstance(self, body): - req = ChatRequest.openai_chat(body) - assert isinstance(req, ChatRequest) - assert request_type_matches(req, ChatRequestType.OPENAI_CHAT) - - def test_body_unpack(self, body): - """Verify **request.body works for SDK create() calls.""" - req = ChatRequest.openai_chat(body) - unpacked = {**req.body} - assert unpacked == body - - def test_body_mutation(self, body): - req = ChatRequest.openai_chat(body) - exported = req.body - exported["model"] = "mutating-exported-copy-does-not-write-through" - assert req.body["model"] == "gpt-4o" - req.set_model("gpt-4o-mini") - assert req.body["model"] == "gpt-4o-mini" - - -class TestResponsesChatBinding: - @pytest.fixture() - def body(self): - return { - "model": "gpt-4o", - "input": "Hello, world!", - "instructions": "Be helpful.", - } - - def test_request_type(self, body): - req = ChatRequest.openai_responses(body) - assert request_type_matches(req, ChatRequestType.OPENAI_RESPONSES) - - def test_body_access(self, body): - req = ChatRequest.openai_responses(body) - assert req.body == body - assert req.body is not body - - def test_body_field_access(self, body): - req = ChatRequest.openai_responses(body) - assert req.body["model"] == "gpt-4o" - assert req.body["input"] == "Hello, world!" - assert req.body["instructions"] == "Be helpful." - - def test_isinstance(self, body): - req = ChatRequest.openai_responses(body) - assert isinstance(req, ChatRequest) - assert request_type_matches(req, ChatRequestType.OPENAI_RESPONSES) - - def test_responses_specific_fields(self): - body = { - "model": "gpt-4o", - "input": "Continue", - "previous_response_id": "resp_abc123", - "truncation": "auto", - } - req = ChatRequest.openai_responses(body) - assert req.body["previous_response_id"] == "resp_abc123" - assert req.body["truncation"] == "auto" - - -class TestAnthropicChatBinding: - @pytest.fixture() - def body(self): - return { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "Hello"}], - "max_tokens": 1024, - } - - def test_request_type(self, body): - req = ChatRequest.anthropic(body) - assert request_type_matches(req, ChatRequestType.ANTHROPIC) - - def test_body_access(self, body): - req = ChatRequest.anthropic(body) - assert req.body == body - assert req.body is not body - - def test_body_field_access(self, body): - req = ChatRequest.anthropic(body) - assert req.body["model"] == "claude-sonnet-4-20250514" - assert req.body["max_tokens"] == 1024 - - def test_isinstance(self, body): - req = ChatRequest.anthropic(body) - assert isinstance(req, ChatRequest) - assert request_type_matches(req, ChatRequestType.ANTHROPIC) - - def test_anthropic_specific_fields(self): - body = { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "Hi"}], - "max_tokens": 2048, - "system": "You are a helpful assistant.", - "stream": True, - } - req = ChatRequest.anthropic(body) - assert req.body["system"] == "You are a helpful assistant." - assert req.body["stream"] is True - - -class TestRequestTypeDispatch: - """Verify request-format dispatch uses the Rust request tag.""" - - def _dispatch(self, request: ChatRequest) -> str: - if request_type_matches(request, ChatRequestType.OPENAI_CHAT): - return "openai_chat" - if request_type_matches(request, ChatRequestType.OPENAI_RESPONSES): - return "responses" - if request_type_matches(request, ChatRequestType.ANTHROPIC): - return "anthropic" - return "unknown" - - def test_dispatch_openai(self): - req = ChatRequest.openai_chat({"model": "gpt-4o", "messages": []}) - assert self._dispatch(req) == "openai_chat" - - def test_dispatch_responses(self): - req = ChatRequest.openai_responses({"model": "gpt-4o", "input": "hi"}) - assert self._dispatch(req) == "responses" - - def test_dispatch_anthropic(self): - req = ChatRequest.anthropic({"model": "claude-sonnet-4-20250514", "messages": [], "max_tokens": 1024}) - assert self._dispatch(req) == "anthropic" - - -class TestMatchDispatch: - """Verify match dispatch uses the request-format enum.""" - - def _dispatch(self, request: ChatRequest) -> str: - match request_type_enum(request.request_type): - case ChatRequestType.OPENAI_CHAT: - return "openai_chat" - case ChatRequestType.OPENAI_RESPONSES: - return "responses" - case ChatRequestType.ANTHROPIC: - return "anthropic" - case _: - return "unknown" - - def test_match_openai(self): - req = ChatRequest.openai_chat({"model": "gpt-4o", "messages": []}) - assert self._dispatch(req) == "openai_chat" - - def test_match_responses(self): - req = ChatRequest.openai_responses({"model": "gpt-4o", "input": "hi"}) - assert self._dispatch(req) == "responses" - - def test_match_anthropic(self): - req = ChatRequest.anthropic({"model": "claude-sonnet-4-20250514", "messages": [], "max_tokens": 1024}) - assert self._dispatch(req) == "anthropic" diff --git a/tests/test_chat_response.py b/tests/test_chat_response.py deleted file mode 100644 index 496f3d1c9..000000000 --- a/tests/test_chat_response.py +++ /dev/null @@ -1,770 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for the ChatResponse type hierarchy and all stream wrappers.""" - -from collections.abc import AsyncIterator - -import pytest -from anthropic.types import Message as AnthropicMessage -from anthropic.types import RawContentBlockDeltaEvent -from anthropic.types import Usage as AnthropicUsage -from anthropic.types.text_delta import TextDelta -from openai.types.chat import ChatCompletion, ChatCompletionChunk -from openai.types.chat.chat_completion import Choice -from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice -from openai.types.chat.chat_completion_chunk import ChoiceDelta -from openai.types.chat.chat_completion_message import ChatCompletionMessage -from openai.types.completion_usage import CompletionUsage -from openai.types.responses import Response as OpenAIResponse -from openai.types.responses import ResponseTextDeltaEvent - -from switchyard.lib.chat_response import AnthropicResponseStream, ResponsesApiStream, ResponseStream -from switchyard_rust.core import ( - ChatResponse, - ChatResponseType, - response_type_matches, -) - -# --------------------------------------------------------------------------- -# Fixtures -# --------------------------------------------------------------------------- - - -def make_completion(*, model: str = "gpt-4o", content: str = "hello") -> ChatCompletion: - return ChatCompletion( - id="chatcmpl-test", - object="chat.completion", - created=1700000000, - model=model, - choices=[ - Choice( - index=0, - message=ChatCompletionMessage(role="assistant", content=content), - finish_reason="stop", - ) - ], - usage=CompletionUsage(prompt_tokens=10, completion_tokens=5, total_tokens=15), - ) - - -def make_chunk(*, index: int = 0, content: str = "hi") -> ChatCompletionChunk: - return ChatCompletionChunk( - id="chatcmpl-test", - object="chat.completion.chunk", - created=1700000000, - model="gpt-4o", - choices=[ - ChunkChoice( - index=index, - delta=ChoiceDelta(content=content), - finish_reason=None, - ) - ], - ) - - -async def fake_stream(chunks: list[ChatCompletionChunk]) -> AsyncIterator[ChatCompletionChunk]: - for chunk in chunks: - yield chunk - - -# --- Anthropic fixtures --- - - -def make_anthropic_message( - *, model: str = "claude-sonnet-4-20250514", text: str = "hello" -) -> AnthropicMessage: - return AnthropicMessage( - id="msg_test", - type="message", - role="assistant", - model=model, - content=[{"type": "text", "text": text}], - stop_reason="end_turn", - usage=AnthropicUsage(input_tokens=10, output_tokens=5), - ) - - -def make_anthropic_content_delta(*, text: str = "hi") -> RawContentBlockDeltaEvent: - return RawContentBlockDeltaEvent( - type="content_block_delta", - index=0, - delta=TextDelta(type="text_delta", text=text), - ) - - -async def fake_anthropic_stream( - events: list[RawContentBlockDeltaEvent], -) -> AsyncIterator[RawContentBlockDeltaEvent]: - for event in events: - yield event - - -# --- OpenAI Responses API fixtures --- - - -def make_responses_api_response( - *, model: str = "gpt-4o", text: str = "hello" -) -> OpenAIResponse: - return OpenAIResponse( - id="resp_test", - created_at=1700000000, - model=model, - object="response", - output=[ - { - "type": "message", - "id": "msg_test", - "status": "completed", - "role": "assistant", - "content": [{"type": "output_text", "text": text, "annotations": []}], - } - ], - tool_choice="auto", - tools=[], - status="completed", - parallel_tool_calls=True, - text={"format": {"type": "text"}}, - ) - - -def make_responses_text_delta(*, delta: str = "hi") -> ResponseTextDeltaEvent: - return ResponseTextDeltaEvent( - type="response.output_text.delta", - item_id="item_test", - output_index=0, - content_index=0, - delta=delta, - logprobs=[], - sequence_number=0, - ) - - -async def fake_responses_stream( - events: list[ResponseTextDeltaEvent], -) -> AsyncIterator[ResponseTextDeltaEvent]: - for event in events: - yield event - - -# --------------------------------------------------------------------------- -# ChatResponseType -# --------------------------------------------------------------------------- - - -class TestChatResponseType: - def test_enum_values(self): - assert ChatResponseType.OPENAI_COMPLETION.value == "openai_completion" - assert ChatResponseType.OPENAI_STREAM.value == "openai_stream" - assert ( - ChatResponseType.OPENAI_RESPONSES_COMPLETION.value - == "openai_responses_completion" - ) - assert ChatResponseType.OPENAI_RESPONSES_STREAM.value == "openai_responses_stream" - assert ChatResponseType.ANTHROPIC_COMPLETION.value == "anthropic_completion" - assert ChatResponseType.ANTHROPIC_STREAM.value == "anthropic_stream" - - def test_enum_members(self): - assert [ - item.value - for item in ( - ChatResponseType.OPENAI_COMPLETION, - ChatResponseType.OPENAI_STREAM, - ChatResponseType.OPENAI_RESPONSES_COMPLETION, - ChatResponseType.OPENAI_RESPONSES_STREAM, - ChatResponseType.ANTHROPIC_COMPLETION, - ChatResponseType.ANTHROPIC_STREAM, - ) - ] == [ - "openai_completion", - "openai_stream", - "openai_responses_completion", - "openai_responses_stream", - "anthropic_completion", - "anthropic_stream", - ] - - -# --------------------------------------------------------------------------- -# ChatResponse ABC -# --------------------------------------------------------------------------- - - -class TestChatResponseABC: - def test_cannot_instantiate(self): - with pytest.raises(TypeError): - ChatResponse() # type: ignore[abstract] - - -# --------------------------------------------------------------------------- -# OpenAI completion ChatResponse -# --------------------------------------------------------------------------- - - -class TestOpenAICompletionResponse: - def test_response_type(self): - resp = ChatResponse.openai_completion(make_completion()) - assert resp.response_type == ChatResponseType.OPENAI_COMPLETION - - def test_body_access(self): - completion = make_completion() - resp = ChatResponse.openai_completion(completion) - assert resp.body == completion.model_dump(mode="json", exclude_none=True) - assert resp.body is not completion - - def test_body_sdk_fields(self): - resp = ChatResponse.openai_completion(make_completion(model="gpt-4o", content="world")) - assert resp.body["model"] == "gpt-4o" - assert resp.body["choices"][0]["message"]["content"] == "world" - assert resp.body["usage"]["total_tokens"] == 15 - - def test_response_type_match(self): - resp = ChatResponse.openai_completion(make_completion()) - assert isinstance(resp, ChatResponse) - assert response_type_matches(resp, ChatResponseType.OPENAI_COMPLETION) - assert not response_type_matches(resp, ChatResponseType.OPENAI_STREAM) - - -# --------------------------------------------------------------------------- -# OpenAI stream ChatResponse -# --------------------------------------------------------------------------- - - -class TestOpenAIStreamChatResponse: - def test_response_type(self): - stream = ResponseStream(fake_stream([])) - resp = ChatResponse.openai_stream(stream) - assert resp.response_type == ChatResponseType.OPENAI_STREAM - - async def test_stream_access(self): - chunks = [make_chunk(content="streamed")] - stream = ResponseStream(fake_stream(chunks)) - resp = ChatResponse.openai_stream(stream) - assert [chunk async for chunk in resp.stream] == chunks - - def test_response_type_match(self): - stream = ResponseStream(fake_stream([])) - resp = ChatResponse.openai_stream(stream) - assert isinstance(resp, ChatResponse) - assert response_type_matches(resp, ChatResponseType.OPENAI_STREAM) - assert not response_type_matches(resp, ChatResponseType.OPENAI_COMPLETION) - - -# --------------------------------------------------------------------------- -# response type dispatch -# --------------------------------------------------------------------------- - - -class TestIsinstanceDispatch: - def _dispatch(self, response: ChatResponse) -> str: - if response_type_matches(response, ChatResponseType.OPENAI_COMPLETION): - return "openai_completion" - if response_type_matches(response, ChatResponseType.OPENAI_STREAM): - return "openai_stream" - if response_type_matches(response, ChatResponseType.OPENAI_RESPONSES_COMPLETION): - return "responses_completion" - if response_type_matches(response, ChatResponseType.OPENAI_RESPONSES_STREAM): - return "responses_stream" - if response_type_matches(response, ChatResponseType.ANTHROPIC_COMPLETION): - return "anthropic_completion" - if response_type_matches(response, ChatResponseType.ANTHROPIC_STREAM): - return "anthropic_stream" - return "unknown" - - def test_dispatch_completion(self): - assert self._dispatch(ChatResponse.openai_completion(make_completion())) == "openai_completion" - - def test_dispatch_stream(self): - resp = ChatResponse.openai_stream(ResponseStream(fake_stream([]))) - assert self._dispatch(resp) == "openai_stream" - - def test_dispatch_responses_completion(self): - resp = ChatResponse.openai_responses_completion(make_responses_api_response()) - assert self._dispatch(resp) == "responses_completion" - - def test_dispatch_responses_stream(self): - stream = ResponsesApiStream(fake_responses_stream([])) - resp = ChatResponse.openai_responses_stream(stream) - assert self._dispatch(resp) == "responses_stream" - - def test_dispatch_anthropic_completion(self): - assert self._dispatch(ChatResponse.anthropic_completion(make_anthropic_message())) == "anthropic_completion" - - def test_dispatch_anthropic_stream(self): - stream = AnthropicResponseStream(fake_anthropic_stream([])) - resp = ChatResponse.anthropic_stream(stream) - assert self._dispatch(resp) == "anthropic_stream" - - -# --------------------------------------------------------------------------- -# ResponseStream -# --------------------------------------------------------------------------- - - -class TestResponseStream: - async def test_basic_iteration(self): - chunks = [make_chunk(content="a"), make_chunk(content="b")] - stream = ResponseStream(fake_stream(chunks)) - result = [chunk async for chunk in stream] - assert len(result) == 2 - assert result[0].choices[0].delta.content == "a" - assert result[1].choices[0].delta.content == "b" - - async def test_empty_stream(self): - stream = ResponseStream(fake_stream([])) - result = [chunk async for chunk in stream] - assert result == [] - - async def test_single_consumption(self): - stream = ResponseStream(fake_stream([make_chunk()])) - _ = [chunk async for chunk in stream] - with pytest.raises(RuntimeError, match="already been consumed"): - _ = [chunk async for chunk in stream] - - async def test_tap_observes_all_chunks(self): - observed: list[str] = [] - - async def log_tap(chunk: ChatCompletionChunk) -> None: - observed.append(chunk.choices[0].delta.content or "") - - chunks = [make_chunk(content="x"), make_chunk(content="y")] - stream = ResponseStream(fake_stream(chunks)) - stream.tap(log_tap) - - _ = [chunk async for chunk in stream] - assert observed == ["x", "y"] - - async def test_map_transforms_chunks(self): - async def upper_map(chunk: ChatCompletionChunk) -> ChatCompletionChunk: - chunk.choices[0].delta.content = (chunk.choices[0].delta.content or "").upper() - return chunk - - stream = ResponseStream(fake_stream([make_chunk(content="hello")])) - stream.map(upper_map) - - result = [chunk async for chunk in stream] - assert result[0].choices[0].delta.content == "HELLO" - - async def test_tap_sees_original_before_map(self): - tap_saw: list[str] = [] - - async def observe(chunk: ChatCompletionChunk) -> None: - tap_saw.append(chunk.choices[0].delta.content or "") - - async def transform(chunk: ChatCompletionChunk) -> ChatCompletionChunk: - chunk.choices[0].delta.content = "TRANSFORMED" - return chunk - - stream = ResponseStream(fake_stream([make_chunk(content="original")])) - stream.tap(observe) - stream.map(transform) - - result = [chunk async for chunk in stream] - assert tap_saw == ["original"] - assert result[0].choices[0].delta.content == "TRANSFORMED" - - async def test_multiple_maps_compose(self): - async def add_exclaim(chunk: ChatCompletionChunk) -> ChatCompletionChunk: - chunk.choices[0].delta.content = (chunk.choices[0].delta.content or "") + "!" - return chunk - - async def add_question(chunk: ChatCompletionChunk) -> ChatCompletionChunk: - chunk.choices[0].delta.content = (chunk.choices[0].delta.content or "") + "?" - return chunk - - stream = ResponseStream(fake_stream([make_chunk(content="hi")])) - stream.map(add_exclaim).map(add_question) - - result = [chunk async for chunk in stream] - assert result[0].choices[0].delta.content == "hi!?" - - async def test_tap_exception_does_not_break_stream(self): - async def bad_tap(chunk: ChatCompletionChunk) -> None: - raise ValueError("tap exploded") - - stream = ResponseStream(fake_stream([make_chunk(content="safe")])) - stream.tap(bad_tap) - - result = [chunk async for chunk in stream] - assert len(result) == 1 - assert result[0].choices[0].delta.content == "safe" - - async def test_failing_tap_is_quarantined(self): - call_count = 0 - - async def flaky_tap(chunk: ChatCompletionChunk) -> None: - nonlocal call_count - call_count += 1 - raise ValueError("always fails") - - chunks = [make_chunk(content="a"), make_chunk(content="b"), make_chunk(content="c")] - stream = ResponseStream(fake_stream(chunks)) - stream.tap(flaky_tap) - - result = [chunk async for chunk in stream] - assert len(result) == 3 - assert call_count == 1 # called once, then quarantined - - async def test_tap_returns_self_for_chaining(self): - async def noop(chunk: ChatCompletionChunk) -> None: - pass - - stream = ResponseStream(fake_stream([])) - assert stream.tap(noop) is stream - - async def test_map_returns_self_for_chaining(self): - async def identity(chunk: ChatCompletionChunk) -> ChatCompletionChunk: - return chunk - - stream = ResponseStream(fake_stream([])) - assert stream.map(identity) is stream - - async def test_multiple_taps(self): - seen_a: list[str] = [] - seen_b: list[str] = [] - - async def tap_a(chunk: ChatCompletionChunk) -> None: - seen_a.append(chunk.choices[0].delta.content or "") - - async def tap_b(chunk: ChatCompletionChunk) -> None: - seen_b.append(chunk.choices[0].delta.content or "") - - stream = ResponseStream(fake_stream([make_chunk(content="x")])) - stream.tap(tap_a).tap(tap_b) - - _ = [chunk async for chunk in stream] - assert seen_a == ["x"] - assert seen_b == ["x"] - - -# --------------------------------------------------------------------------- -# OpenAI Responses completion ChatResponse -# --------------------------------------------------------------------------- - - -class TestOpenAIResponsesCompletionResponse: - def test_response_type(self): - resp = ChatResponse.openai_responses_completion(make_responses_api_response()) - assert resp.response_type == ChatResponseType.OPENAI_RESPONSES_COMPLETION - - def test_body_access(self): - body = make_responses_api_response() - resp = ChatResponse.openai_responses_completion(body) - assert resp.body == body.model_dump(mode="json", exclude_none=True) - assert resp.body is not body - - def test_body_sdk_fields(self): - resp = ChatResponse.openai_responses_completion( - make_responses_api_response(model="gpt-4o", text="world") - ) - assert resp.body["model"] == "gpt-4o" - assert resp.body["output"][0]["content"][0]["text"] == "world" - assert resp.body["status"] == "completed" - - def test_response_type_match(self): - resp = ChatResponse.openai_responses_completion(make_responses_api_response()) - assert isinstance(resp, ChatResponse) - assert response_type_matches(resp, ChatResponseType.OPENAI_RESPONSES_COMPLETION) - assert not response_type_matches(resp, ChatResponseType.OPENAI_COMPLETION) - assert not response_type_matches(resp, ChatResponseType.OPENAI_STREAM) - assert not response_type_matches(resp, ChatResponseType.ANTHROPIC_COMPLETION) - - -# --------------------------------------------------------------------------- -# OpenAI Responses stream ChatResponse -# --------------------------------------------------------------------------- - - -class TestOpenAIResponsesStreamChatResponse: - def test_response_type(self): - stream = ResponsesApiStream(fake_responses_stream([])) - resp = ChatResponse.openai_responses_stream(stream) - assert resp.response_type == ChatResponseType.OPENAI_RESPONSES_STREAM - - async def test_stream_access(self): - events = [make_responses_text_delta(delta="streamed")] - stream = ResponsesApiStream(fake_responses_stream(events)) - resp = ChatResponse.openai_responses_stream(stream) - assert [event async for event in resp.stream] == events - - def test_response_type_match(self): - stream = ResponsesApiStream(fake_responses_stream([])) - resp = ChatResponse.openai_responses_stream(stream) - assert isinstance(resp, ChatResponse) - assert response_type_matches(resp, ChatResponseType.OPENAI_RESPONSES_STREAM) - assert not response_type_matches(resp, ChatResponseType.OPENAI_COMPLETION) - assert not response_type_matches(resp, ChatResponseType.OPENAI_RESPONSES_COMPLETION) - - -# --------------------------------------------------------------------------- -# ResponsesApiStream -# --------------------------------------------------------------------------- - - -class TestResponsesApiStream: - async def test_basic_iteration(self): - events = [make_responses_text_delta(delta="a"), make_responses_text_delta(delta="b")] - stream = ResponsesApiStream(fake_responses_stream(events)) - result = [event async for event in stream] - assert len(result) == 2 - assert result[0].delta == "a" - assert result[1].delta == "b" - - async def test_empty_stream(self): - stream = ResponsesApiStream(fake_responses_stream([])) - result = [event async for event in stream] - assert result == [] - - async def test_single_consumption(self): - stream = ResponsesApiStream(fake_responses_stream([make_responses_text_delta()])) - _ = [event async for event in stream] - with pytest.raises(RuntimeError, match="already been consumed"): - _ = [event async for event in stream] - - async def test_tap_observes_all_events(self): - observed: list[str] = [] - - async def log_tap(event: ResponseTextDeltaEvent) -> None: - observed.append(event.delta) - - events = [make_responses_text_delta(delta="x"), make_responses_text_delta(delta="y")] - stream = ResponsesApiStream(fake_responses_stream(events)) - stream.tap(log_tap) - - _ = [event async for event in stream] - assert observed == ["x", "y"] - - async def test_tap_exception_does_not_break_stream(self): - async def bad_tap(event: ResponseTextDeltaEvent) -> None: - raise ValueError("tap exploded") - - stream = ResponsesApiStream( - fake_responses_stream([make_responses_text_delta(delta="safe")]) - ) - stream.tap(bad_tap) - - result = [event async for event in stream] - assert len(result) == 1 - assert result[0].delta == "safe" - - async def test_failing_tap_is_quarantined(self): - call_count = 0 - - async def flaky_tap(event: ResponseTextDeltaEvent) -> None: - nonlocal call_count - call_count += 1 - raise ValueError("always fails") - - events = [ - make_responses_text_delta(delta="a"), - make_responses_text_delta(delta="b"), - make_responses_text_delta(delta="c"), - ] - stream = ResponsesApiStream(fake_responses_stream(events)) - stream.tap(flaky_tap) - - result = [event async for event in stream] - assert len(result) == 3 - assert call_count == 1 - - async def test_tap_returns_self_for_chaining(self): - async def noop(event: ResponseTextDeltaEvent) -> None: - pass - - stream = ResponsesApiStream(fake_responses_stream([])) - assert stream.tap(noop) is stream - - async def test_map_returns_self_for_chaining(self): - async def identity(event: ResponseTextDeltaEvent) -> ResponseTextDeltaEvent: - return event - - stream = ResponsesApiStream(fake_responses_stream([])) - assert stream.map(identity) is stream - - -# --------------------------------------------------------------------------- -# Anthropic completion ChatResponse -# --------------------------------------------------------------------------- - - -class TestAnthropicCompletionResponse: - def test_response_type(self): - resp = ChatResponse.anthropic_completion(make_anthropic_message()) - assert resp.response_type == ChatResponseType.ANTHROPIC_COMPLETION - - def test_body_access(self): - msg = make_anthropic_message() - resp = ChatResponse.anthropic_completion(msg) - assert resp.body == msg.model_dump(mode="json", exclude_none=True) - assert resp.body is not msg - - def test_body_sdk_fields(self): - resp = ChatResponse.anthropic_completion( - make_anthropic_message(model="claude-sonnet-4-20250514", text="world") - ) - assert resp.body["model"] == "claude-sonnet-4-20250514" - assert resp.body["content"][0]["text"] == "world" - assert resp.body["usage"]["input_tokens"] == 10 - assert resp.body["usage"]["output_tokens"] == 5 - assert resp.body["stop_reason"] == "end_turn" - - def test_response_type_match(self): - resp = ChatResponse.anthropic_completion(make_anthropic_message()) - assert isinstance(resp, ChatResponse) - assert response_type_matches(resp, ChatResponseType.ANTHROPIC_COMPLETION) - assert not response_type_matches(resp, ChatResponseType.OPENAI_COMPLETION) - assert not response_type_matches(resp, ChatResponseType.OPENAI_STREAM) - assert not response_type_matches(resp, ChatResponseType.ANTHROPIC_STREAM) - - -# --------------------------------------------------------------------------- -# Anthropic stream ChatResponse -# --------------------------------------------------------------------------- - - -class TestAnthropicStreamChatResponse: - def test_response_type(self): - stream = AnthropicResponseStream(fake_anthropic_stream([])) - resp = ChatResponse.anthropic_stream(stream) - assert resp.response_type == ChatResponseType.ANTHROPIC_STREAM - - async def test_stream_access(self): - events = [make_anthropic_content_delta(text="streamed")] - stream = AnthropicResponseStream(fake_anthropic_stream(events)) - resp = ChatResponse.anthropic_stream(stream) - assert [event async for event in resp.stream] == events - - def test_response_type_match(self): - stream = AnthropicResponseStream(fake_anthropic_stream([])) - resp = ChatResponse.anthropic_stream(stream) - assert isinstance(resp, ChatResponse) - assert response_type_matches(resp, ChatResponseType.ANTHROPIC_STREAM) - assert not response_type_matches(resp, ChatResponseType.OPENAI_COMPLETION) - assert not response_type_matches(resp, ChatResponseType.OPENAI_STREAM) - assert not response_type_matches(resp, ChatResponseType.ANTHROPIC_COMPLETION) - - -# --------------------------------------------------------------------------- -# AnthropicResponseStream -# --------------------------------------------------------------------------- - - -class TestAnthropicResponseStream: - async def test_basic_iteration(self): - events = [make_anthropic_content_delta(text="a"), make_anthropic_content_delta(text="b")] - stream = AnthropicResponseStream(fake_anthropic_stream(events)) - result = [event async for event in stream] - assert len(result) == 2 - assert result[0].delta.text == "a" - assert result[1].delta.text == "b" - - async def test_empty_stream(self): - stream = AnthropicResponseStream(fake_anthropic_stream([])) - result = [event async for event in stream] - assert result == [] - - async def test_single_consumption(self): - stream = AnthropicResponseStream(fake_anthropic_stream([make_anthropic_content_delta()])) - _ = [event async for event in stream] - with pytest.raises(RuntimeError, match="already been consumed"): - _ = [event async for event in stream] - - async def test_tap_observes_all_events(self): - observed: list[str] = [] - - async def log_tap(event: RawContentBlockDeltaEvent) -> None: - observed.append(event.delta.text) - - events = [make_anthropic_content_delta(text="x"), make_anthropic_content_delta(text="y")] - stream = AnthropicResponseStream(fake_anthropic_stream(events)) - stream.tap(log_tap) - - _ = [event async for event in stream] - assert observed == ["x", "y"] - - async def test_map_transforms_events(self): - async def upper_map(event: RawContentBlockDeltaEvent) -> RawContentBlockDeltaEvent: - return RawContentBlockDeltaEvent( - type="content_block_delta", - index=event.index, - delta=TextDelta(type="text_delta", text=event.delta.text.upper()), - ) - - stream = AnthropicResponseStream( - fake_anthropic_stream([make_anthropic_content_delta(text="hello")]) - ) - stream.map(upper_map) - - result = [event async for event in stream] - assert result[0].delta.text == "HELLO" - - async def test_tap_sees_original_before_map(self): - tap_saw: list[str] = [] - - async def observe(event: RawContentBlockDeltaEvent) -> None: - tap_saw.append(event.delta.text) - - async def transform(event: RawContentBlockDeltaEvent) -> RawContentBlockDeltaEvent: - return RawContentBlockDeltaEvent( - type="content_block_delta", - index=event.index, - delta=TextDelta(type="text_delta", text="TRANSFORMED"), - ) - - stream = AnthropicResponseStream( - fake_anthropic_stream([make_anthropic_content_delta(text="original")]) - ) - stream.tap(observe) - stream.map(transform) - - result = [event async for event in stream] - assert tap_saw == ["original"] - assert result[0].delta.text == "TRANSFORMED" - - async def test_tap_exception_does_not_break_stream(self): - async def bad_tap(event: RawContentBlockDeltaEvent) -> None: - raise ValueError("tap exploded") - - stream = AnthropicResponseStream( - fake_anthropic_stream([make_anthropic_content_delta(text="safe")]) - ) - stream.tap(bad_tap) - - result = [event async for event in stream] - assert len(result) == 1 - assert result[0].delta.text == "safe" - - async def test_failing_tap_is_quarantined(self): - call_count = 0 - - async def flaky_tap(event: RawContentBlockDeltaEvent) -> None: - nonlocal call_count - call_count += 1 - raise ValueError("always fails") - - events = [ - make_anthropic_content_delta(text="a"), - make_anthropic_content_delta(text="b"), - make_anthropic_content_delta(text="c"), - ] - stream = AnthropicResponseStream(fake_anthropic_stream(events)) - stream.tap(flaky_tap) - - result = [event async for event in stream] - assert len(result) == 3 - assert call_count == 1 - - async def test_tap_returns_self_for_chaining(self): - async def noop(event: RawContentBlockDeltaEvent) -> None: - pass - - stream = AnthropicResponseStream(fake_anthropic_stream([])) - assert stream.tap(noop) is stream - - async def test_map_returns_self_for_chaining(self): - async def identity(event: RawContentBlockDeltaEvent) -> RawContentBlockDeltaEvent: - return event - - stream = AnthropicResponseStream(fake_anthropic_stream([])) - assert stream.map(identity) is stream diff --git a/tests/test_cli_reference_docs.py b/tests/test_cli_reference_docs.py index 682ade769..857e3eb99 100644 --- a/tests/test_cli_reference_docs.py +++ b/tests/test_cli_reference_docs.py @@ -38,7 +38,6 @@ def test_reference_documents_launcher_before_server() -> None: related = text.index("## Related Documentation") assert launcher < server < removed < related - assert "switchyard serve" not in text def test_reference_marks_removed_setup_commands() -> None: @@ -48,10 +47,13 @@ def test_reference_marks_removed_setup_commands() -> None: removed = text[removed_start:related_start] commands = _subparsers(_build_parser()) - assert "`switchyard configure` and `switchyard verify` are not available" in removed + assert ( + "`switchyard configure`, `switchyard serve`, and `switchyard verify` are not" + in removed + ) assert "api_key_env" in removed - assert "The CLI does not save provider credentials or deployment paths" in removed - assert {"configure", "verify"}.isdisjoint(commands) + assert "The CLI does not save provider credentials, deployment paths" in removed + assert {"configure", "serve", "verify"}.isdisjoint(commands) def test_reference_lists_launcher_contract() -> None: diff --git a/tests/test_codex_multiturn_traces.py b/tests/test_codex_multiturn_traces.py deleted file mode 100644 index d238e33b6..000000000 --- a/tests/test_codex_multiturn_traces.py +++ /dev/null @@ -1,510 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Diagnostic tests for Codex multi-turn tool call history merging. - -Uses actual trace data from tmp/output/logs to verify whether the observed -"missing intermediate assistant-tool turns" is: - (a) Codex CLI behavior (client sends merged history) - (b) A bug in our Responses API → Chat Completions translation layer - (c) User misinterpretation of the logs - -CONCLUSION: It's (b) — a bug in Responses input-item translation. The -translator didn't flush the pending tool block at turn boundaries -(function_call_output → function_call transition), so tool calls across turns -were merged into one assistant message. -""" - -import json -from pathlib import Path - -import pytest - -from switchyard_rust.translation import TranslationEngine - -TRACE_DIR = Path(__file__).parent.parent / "tmp" / "output" / "logs" / "openai" -ENGINE = TranslationEngine() - - -def _responses_items_to_messages(items: list) -> list: - body = ENGINE.translate_request("openai_responses", "openai_chat", {"input": items}) - return list(body.get("messages", [])) - - -# --------------------------------------------------------------------------- -# Helper: load trace files sorted by timestamp -# --------------------------------------------------------------------------- - -def _load_traces(): - """Load all trace JSON files from the output directory, sorted by timestamp.""" - if not TRACE_DIR.exists(): - pytest.skip(f"Trace directory not found: {TRACE_DIR}") - files = sorted(TRACE_DIR.glob("*.json")) - if not files: - pytest.skip("No trace files found") - traces = [] - for f in files: - with open(f) as fh: - traces.append(json.load(fh)) - return traces - - -# --------------------------------------------------------------------------- -# Test 1: Verify what the traces actually show (structural analysis) -# --------------------------------------------------------------------------- - -class TestTraceStructuralAnalysis: - """Analyze the actual trace files to understand the conversation structure.""" - - def test_trace_count(self): - """There should be 4 traces for this Codex session.""" - traces = _load_traces() - assert len(traces) == 4, f"Expected 4 traces, got {len(traces)}" - - def test_trace1_is_initial_request(self): - """Trace 1: Initial request with only user messages, no history.""" - traces = _load_traces() - t1 = traces[0] - msgs = t1["request"]["messages"] - - # Should have: system, developer, user(AGENTS.md), user(env), user(task) - roles = [m["role"] for m in msgs] - assert roles == ["system", "developer", "user", "user", "user"] - - # Response should have 2 tool_calls (update_plan + apply_patch) - response_tool_calls = t1["response"]["choices"][0]["message"]["tool_calls"] - assert len(response_tool_calls) == 2 - names = [tc["function"]["name"] for tc in response_tool_calls] - assert names == ["update_plan", "apply_patch"] - - def test_trace2_has_turn1_history(self): - """Trace 2: Should include turn 1's assistant + tool results as history.""" - traces = _load_traces() - t2 = traces[1] - msgs = t2["request"]["messages"] - - roles = [m["role"] for m in msgs] - # Expected: system, developer, user, user, user, assistant, tool, tool - assert roles == ["system", "developer", "user", "user", "user", - "assistant", "tool", "tool"] - - # The assistant message should have exactly 2 tool_calls (from turn 1) - assistant_msg = msgs[5] - assert len(assistant_msg["tool_calls"]) == 2 - names = [tc["function"]["name"] for tc in assistant_msg["tool_calls"]] - assert names == ["update_plan", "apply_patch"] - - def test_trace3_shows_merged_turns_pre_fix(self): - """Trace 3: Captured before fix — turns 1+2 merged into single assistant msg. - - These traces were recorded with the old buggy translation layer. - After the fix in _responses_items_to_messages(), new traces - would show separate assistant messages per turn. - """ - traces = _load_traces() - t3 = traces[2] - msgs = t3["request"]["messages"] - - [m["role"] for m in msgs] - - # Count assistant messages in the history - assistant_msgs = [m for m in msgs if m["role"] == "assistant"] - tool_msgs = [m for m in msgs if m["role"] == "tool"] - - # BUG: There's only 1 assistant message when there should be 2 - assert len(assistant_msgs) == 1, ( - f"BUG CONFIRMED: Only {len(assistant_msgs)} assistant message(s) — " - f"turns 1+2 are merged into a single assistant message" - ) - - # The single assistant message has 9 tool_calls (2 from turn 1 + 7 from turn 2) - merged_tool_calls = assistant_msgs[0]["tool_calls"] - assert len(merged_tool_calls) == 9, ( - f"Expected 9 merged tool_calls (2+7), got {len(merged_tool_calls)}" - ) - - # Verify the first 2 are from turn 1 - assert merged_tool_calls[0]["function"]["name"] == "update_plan" - assert merged_tool_calls[1]["function"]["name"] == "apply_patch" - # The remaining 7 are from turn 2 - turn2_names = [tc["function"]["name"] for tc in merged_tool_calls[2:]] - assert all(n == "exec_command" for n in turn2_names) - - # There should be 9 tool messages too - assert len(tool_msgs) == 9 - - def test_trace4_shows_all_turns_merged_pre_fix(self): - """Trace 4: Captured before fix — all turns merged into single assistant msg.""" - traces = _load_traces() - t4 = traces[3] - msgs = t4["request"]["messages"] - - assistant_msgs = [m for m in msgs if m["role"] == "assistant"] - [m for m in msgs if m["role"] == "tool"] - - # BUG: Still only 1 assistant message - assert len(assistant_msgs) == 1, ( - f"BUG CONFIRMED: Only {len(assistant_msgs)} assistant message(s) — " - f"all turns merged" - ) - - # Should have even more tool_calls (2 + 7 + 3 = 12 from turn 3 additions) - total_tool_calls = len(assistant_msgs[0]["tool_calls"]) - assert total_tool_calls > 9, ( - f"Expected >9 tool_calls in merged message, got {total_tool_calls}" - ) - - -# --------------------------------------------------------------------------- -# Test 2: Reproduce the bug with reconstructed Responses API input -# --------------------------------------------------------------------------- - -class TestTranslationLayerMultiTurnBug: - """Reproduce the turn-merging bug using _responses_items_to_messages() - with input items reconstructed from the trace data.""" - - @staticmethod - def _make_two_turn_input(): - """Reconstruct the Responses API input Codex would send for trace 3. - - Turn 1: 2 parallel tool calls (update_plan + apply_patch) + results - Turn 2: 7 parallel tool calls (5× ls + 2× echo) + results - - This is the standard Responses API format: all calls from a turn - grouped together, then all outputs, then next turn's calls, etc. - """ - return [ - # --- User message --- - {"type": "message", "role": "user", "content": "Create a Python script"}, - - # --- Turn 1: 2 tool calls --- - { - "type": "function_call", - "name": "update_plan", - "call_id": "call_turn1_a", - "arguments": '{"plan": [{"step": "Create script", "status": "completed"}]}', - }, - { - "type": "function_call", - "name": "apply_patch", - "call_id": "call_turn1_b", - "arguments": '{"command": "*** Begin Patch..."}', - }, - # --- Turn 1: results --- - { - "type": "function_call_output", - "call_id": "call_turn1_a", - "output": "Plan updated", - }, - { - "type": "function_call_output", - "call_id": "call_turn1_b", - "output": "unsupported call: apply_patch", - }, - - # --- Turn 2: 7 tool calls --- - { - "type": "function_call", - "name": "exec_command", - "call_id": "call_turn2_a", - "arguments": '{"cmd": "ls -R ."}', - }, - { - "type": "function_call", - "name": "exec_command", - "call_id": "call_turn2_b", - "arguments": '{"cmd": "ls -R ."}', - }, - { - "type": "function_call", - "name": "exec_command", - "call_id": "call_turn2_c", - "arguments": '{"cmd": "ls -R ."}', - }, - { - "type": "function_call", - "name": "exec_command", - "call_id": "call_turn2_d", - "arguments": '{"cmd": "ls -R ."}', - }, - { - "type": "function_call", - "name": "exec_command", - "call_id": "call_turn2_e", - "arguments": '{"cmd": "ls -R .", "max_output_tokens": 2000}', - }, - { - "type": "function_call", - "name": "exec_command", - "call_id": "call_turn2_f", - "arguments": '{"cmd": "echo hello", "max_output_tokens": 20}', - }, - { - "type": "function_call", - "name": "exec_command", - "call_id": "call_turn2_g", - "arguments": '{"cmd": "echo hello", "shell": "/bin/bash"}', - }, - # --- Turn 2: results --- - { - "type": "function_call_output", - "call_id": "call_turn2_a", - "output": ".:\ninput.json\nlogs\n", - }, - { - "type": "function_call_output", - "call_id": "call_turn2_b", - "output": ".:\ninput.json\nlogs\n", - }, - { - "type": "function_call_output", - "call_id": "call_turn2_c", - "output": ".:\ninput.json\nlogs\n", - }, - { - "type": "function_call_output", - "call_id": "call_turn2_d", - "output": ".:\ninput.json\nlogs\n", - }, - { - "type": "function_call_output", - "call_id": "call_turn2_e", - "output": ".:\ninput.json\nlogs\n", - }, - { - "type": "function_call_output", - "call_id": "call_turn2_f", - "output": "hello\n", - }, - { - "type": "function_call_output", - "call_id": "call_turn2_g", - "output": "hello\n", - }, - ] - - def test_two_turns_produce_separate_assistant_messages(self): - """Two turns of tool calls produce separate assistant messages. - - The proxy detects the function_call_output → function_call turn - boundary and flushes, producing: - user → assistant(2 calls) → 2 tools → assistant(7 calls) → 7 tools - """ - items = self._make_two_turn_input() - messages = _responses_items_to_messages(items) - - user_msgs = [m for m in messages if m["role"] == "user"] - asst_msgs = [m for m in messages if m["role"] == "assistant"] - tool_msgs = [m for m in messages if m["role"] == "tool"] - - assert len(user_msgs) == 1 - assert len(asst_msgs) == 2, "Should have 2 assistant messages (one per turn)" - assert len(asst_msgs[0]["tool_calls"]) == 2, "Turn 1: 2 tool calls" - assert len(asst_msgs[1]["tool_calls"]) == 7, "Turn 2: 7 tool calls" - assert len(tool_msgs) == 9, "Total 9 tool results" - - # Verify message ordering - expected_roles = [ - "user", # original request - "assistant", # turn 1: 2 tool calls - "tool", # turn 1 result 1 - "tool", # turn 1 result 2 - "assistant", # turn 2: 7 tool calls - "tool", # turn 2 results... - "tool", - "tool", - "tool", - "tool", - "tool", - "tool", - ] - actual_roles = [m["role"] for m in messages] - assert actual_roles == expected_roles - - def test_single_turn_still_works(self): - """Single-turn tool calls should still be merged into one assistant message.""" - items = [ - {"type": "message", "role": "user", "content": "Do two things"}, - { - "type": "function_call", - "name": "tool_a", - "call_id": "call_a", - "arguments": "{}", - }, - { - "type": "function_call", - "name": "tool_b", - "call_id": "call_b", - "arguments": "{}", - }, - { - "type": "function_call_output", - "call_id": "call_a", - "output": "result_a", - }, - { - "type": "function_call_output", - "call_id": "call_b", - "output": "result_b", - }, - ] - messages = _responses_items_to_messages(items) - - asst_msgs = [m for m in messages if m["role"] == "assistant"] - assert len(asst_msgs) == 1 - assert len(asst_msgs[0]["tool_calls"]) == 2 - - def test_three_turns_produce_three_assistant_messages(self): - """Three turns of tool calls produce 3 separate assistant messages. - - Reconstructs what Codex sends for trace 4 (turn 1 + turn 2 + turn 3). - """ - items = [ - {"type": "message", "role": "user", "content": "Create a script"}, - - # Turn 1: 2 calls - {"type": "function_call", "name": "update_plan", "call_id": "t1_a", "arguments": "{}"}, - {"type": "function_call", "name": "apply_patch", "call_id": "t1_b", "arguments": "{}"}, - {"type": "function_call_output", "call_id": "t1_a", "output": "Plan updated"}, - {"type": "function_call_output", "call_id": "t1_b", "output": "unsupported"}, - - # Turn 2: 3 calls - {"type": "function_call", "name": "exec_command", "call_id": "t2_a", "arguments": '{"cmd":"ls"}'}, - {"type": "function_call", "name": "exec_command", "call_id": "t2_b", "arguments": '{"cmd":"echo hi"}'}, - {"type": "function_call", "name": "exec_command", "call_id": "t2_c", "arguments": '{"cmd":"pwd"}'}, - {"type": "function_call_output", "call_id": "t2_a", "output": "file1 file2"}, - {"type": "function_call_output", "call_id": "t2_b", "output": "hi"}, - {"type": "function_call_output", "call_id": "t2_c", "output": "/workspace"}, - - # Turn 3: 2 calls - {"type": "function_call", "name": "exec_command", "call_id": "t3_a", "arguments": '{"cmd":"cat file1"}'}, - {"type": "function_call", "name": "exec_command", "call_id": "t3_b", "arguments": '{"cmd":"cat file2"}'}, - {"type": "function_call_output", "call_id": "t3_a", "output": "content1"}, - {"type": "function_call_output", "call_id": "t3_b", "output": "content2"}, - ] - messages = _responses_items_to_messages(items) - - asst_msgs = [m for m in messages if m["role"] == "assistant"] - tool_msgs = [m for m in messages if m["role"] == "tool"] - - assert len(asst_msgs) == 3, "Should have 3 assistant messages" - assert len(asst_msgs[0]["tool_calls"]) == 2, "Turn 1: 2 calls" - assert len(asst_msgs[1]["tool_calls"]) == 3, "Turn 2: 3 calls" - assert len(asst_msgs[2]["tool_calls"]) == 2, "Turn 3: 2 calls" - assert len(tool_msgs) == 7, "Total 7 tool results" - - -# --------------------------------------------------------------------------- -# Test 3: Verify the specific bug location -# --------------------------------------------------------------------------- - -class TestBugLocation: - """Pinpoint the exact code path causing the merge.""" - - def test_turn_boundary_detected_at_output_to_call_transition(self): - """The converter flushes at function_call_output → function_call transitions. - - This transition marks a turn boundary: the previous turn's outputs are - done and a new LLM response is starting. - """ - # Minimal case: two turns with no message item between them - items = [ - # Turn 1 - {"type": "function_call", "name": "tool_a", "call_id": "a", "arguments": "{}"}, - {"type": "function_call_output", "call_id": "a", "output": "done_a"}, - # Turn 2 (no message item separating from turn 1) - {"type": "function_call", "name": "tool_b", "call_id": "b", "arguments": "{}"}, - {"type": "function_call_output", "call_id": "b", "output": "done_b"}, - ] - messages = _responses_items_to_messages(items) - - asst_msgs = [m for m in messages if m["role"] == "assistant"] - - # Correctly produces 2 separate assistant messages - assert len(asst_msgs) == 2 - assert len(asst_msgs[0]["tool_calls"]) == 1 - assert asst_msgs[0]["tool_calls"][0]["id"] == "a" - assert len(asst_msgs[1]["tool_calls"]) == 1 - assert asst_msgs[1]["tool_calls"][0]["id"] == "b" - - def test_message_item_correctly_triggers_flush(self): - """When a message item separates tool blocks, flush works correctly.""" - items = [ - # Turn 1 - {"type": "function_call", "name": "tool_a", "call_id": "a", "arguments": "{}"}, - {"type": "function_call_output", "call_id": "a", "output": "done_a"}, - # Explicit message item triggers flush - {"type": "message", "role": "assistant", "content": "Intermediate text"}, - # Turn 2 - {"type": "function_call", "name": "tool_b", "call_id": "b", "arguments": "{}"}, - {"type": "function_call_output", "call_id": "b", "output": "done_b"}, - ] - messages = _responses_items_to_messages(items) - - asst_msgs = [m for m in messages if m["role"] == "assistant"] - # This works correctly: 3 assistant messages (tool_a, text, tool_b) - assert len(asst_msgs) == 3 - assert len(asst_msgs[0]["tool_calls"]) == 1 # tool_a - assert asst_msgs[1]["content"] == "Intermediate text" - assert len(asst_msgs[2]["tool_calls"]) == 1 # tool_b - - -# --------------------------------------------------------------------------- -# Test 4: Verify this matches the trace data from tmp/output -# --------------------------------------------------------------------------- - -class TestTraceDataConsistency: - """Cross-reference trace data with the translation function output.""" - - def test_trace2_tool_call_ids_match(self): - """The tool_call IDs in trace 2's messages should match trace 1's response.""" - traces = _load_traces() - t1_response_calls = traces[0]["response"]["choices"][0]["message"]["tool_calls"] - t2_history_assistant = next( - m for m in traces[1]["request"]["messages"] if m["role"] == "assistant" - ) - - t1_ids = [tc["id"] for tc in t1_response_calls] - t2_ids = [tc["id"] for tc in t2_history_assistant["tool_calls"]] - - assert t1_ids == t2_ids, ( - "Turn 1 response tool_call IDs should appear in trace 2's history" - ) - - def test_trace3_contains_turn1_and_turn2_calls_merged(self): - """Trace 3's single assistant message has turn 1 + turn 2 tool_calls merged.""" - traces = _load_traces() - t3_assistant = next( - m for m in traces[2]["request"]["messages"] if m["role"] == "assistant" - ) - - t1_ids = [tc["id"] for tc in traces[0]["response"]["choices"][0]["message"]["tool_calls"]] - t2_ids = [tc["id"] for tc in traces[1]["response"]["choices"][0]["message"]["tool_calls"]] - - merged_ids = [tc["id"] for tc in t3_assistant["tool_calls"]] - - # The merged assistant message contains ALL IDs from both turns - assert merged_ids[:len(t1_ids)] == t1_ids, "First tool_calls should be from turn 1" - assert merged_ids[len(t1_ids):] == t2_ids, "Remaining tool_calls should be from turn 2" - - def test_all_traces_same_session(self): - """All traces belong to the same Codex session.""" - traces = _load_traces() - session_ids = {t["sessionID"] for t in traces} - assert len(session_ids) == 1, f"Expected 1 session, got {session_ids}" - - def test_tool_results_in_trace2_match_tool_call_ids(self): - """The tool_results attached to trace 1 should be consistent with trace 2's history.""" - traces = _load_traces() - t1 = traces[0] - - # tool_results from the deferred logging - if "tool_results" not in t1: - pytest.skip("Trace 1 doesn't have tool_results (may be first-turn pattern)") - - t1_result_ids = {tr["tool_call_id"] for tr in t1["tool_results"]} - t1_call_ids = {tc["id"] for tc in t1["response"]["choices"][0]["message"]["tool_calls"]} - - assert t1_result_ids == t1_call_ids, ( - "tool_results should correspond to the response's tool_calls" - ) diff --git a/tests/test_context_error_translation.py b/tests/test_context_error_translation.py deleted file mode 100644 index b2e0e8038..000000000 --- a/tests/test_context_error_translation.py +++ /dev/null @@ -1,50 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Inbound-format JSON shapes for :func:`context_exhausted_response`.""" - -from __future__ import annotations - -import json - -from switchyard.lib.endpoints.upstream_error import context_exhausted_response - - -def _body(exc_message: str, inbound: str) -> dict: - exc = RuntimeError(exc_message) - response = context_exhausted_response(exc, inbound=inbound) # type: ignore[arg-type] - assert response.status_code == 400 - return json.loads(response.body) - - -def test_anthropic_inbound_shape() -> None: - body = _body("context pool exhausted", "anthropic") - assert body == { - "error": { - "message": "context pool exhausted", - "type": "invalid_request_error", - "code": "context_length_exceeded", - }, - } - - -def test_openai_inbound_shape() -> None: - body = _body("context pool exhausted", "openai") - assert body == { - "error": { - "message": "context pool exhausted", - "type": "invalid_request_error", - "code": "context_length_exceeded", - }, - } - - -def test_openai_responses_inbound_shape() -> None: - body = _body("context pool exhausted", "openai-responses") - assert body == { - "error": { - "message": "context pool exhausted", - "type": "invalid_request_error", - "code": "context_length_exceeded", - }, - } diff --git a/tests/test_context_window_exceeded_endpoint.py b/tests/test_context_window_exceeded_endpoint.py deleted file mode 100644 index 5cbe71bbd..000000000 --- a/tests/test_context_window_exceeded_endpoint.py +++ /dev/null @@ -1,73 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Endpoint-level tests: SwitchyardContextWindowExceededError on a single-target -route must return HTTP 400 context_length_exceeded, not HTTP 500.""" - -from __future__ import annotations - -from collections.abc import AsyncIterator -from unittest.mock import AsyncMock - -import httpx -import pytest -from fastapi import FastAPI - -from switchyard.lib.endpoints.anthropic_messages_endpoint import AnthropicMessagesEndpoint -from switchyard.lib.endpoints.openai_chat_endpoint import OpenAIChatEndpoint -from switchyard.lib.endpoints.responses_endpoint import ResponsesEndpoint -from switchyard_rust.core import SwitchyardContextWindowExceededError - -_CHAT_BODY = {"model": "m", "messages": [{"role": "user", "content": "hi"}]} -_ANTHROPIC_BODY = {"model": "m", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 100} -_RESPONSES_BODY = {"model": "m", "input": "hi"} - - -def _app_raising(exc: Exception) -> FastAPI: - """Build a minimal FastAPI app whose chain always raises *exc*.""" - app = FastAPI() - mock_sw = AsyncMock() - mock_sw.call = AsyncMock(side_effect=exc) - app.state.switchyard = mock_sw - OpenAIChatEndpoint().register(app) - AnthropicMessagesEndpoint().register(app) - ResponsesEndpoint().register(app) - return app - - -@pytest.fixture -async def window_client() -> AsyncIterator[httpx.AsyncClient]: - """Async client wired to an app whose chain raises SwitchyardContextWindowExceededError.""" - exc = SwitchyardContextWindowExceededError("context window exceeded on single target") - app = _app_raising(exc) - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=app), base_url="http://test" - ) as c: - yield c - - -async def test_openai_chat_window_exceeded_returns_400(window_client: httpx.AsyncClient) -> None: - """POST /v1/chat/completions returns 400 context_length_exceeded, not 500.""" - resp = await window_client.post("/v1/chat/completions", json=_CHAT_BODY) - assert resp.status_code == 400 - body = resp.json() - assert body["error"]["code"] == "context_length_exceeded" - assert body["error"]["type"] == "invalid_request_error" - - -async def test_anthropic_messages_window_exceeded_returns_400(window_client: httpx.AsyncClient) -> None: - """POST /v1/messages returns 400 invalid_request_error, not 500.""" - resp = await window_client.post("/v1/messages", json=_ANTHROPIC_BODY) - assert resp.status_code == 400 - body = resp.json() - assert body["error"]["code"] == "context_length_exceeded" - assert body["error"]["type"] == "invalid_request_error" - - -async def test_responses_window_exceeded_returns_400(window_client: httpx.AsyncClient) -> None: - """POST /v1/responses returns 400 context_length_exceeded, not 500.""" - resp = await window_client.post("/v1/responses", json=_RESPONSES_BODY) - assert resp.status_code == 400 - body = resp.json() - assert body["error"]["code"] == "context_length_exceeded" - assert body["error"]["type"] == "invalid_request_error" diff --git a/tests/test_cost_estimator_gemini.py b/tests/test_cost_estimator_gemini.py index 3977102e7..4407b8535 100644 --- a/tests/test_cost_estimator_gemini.py +++ b/tests/test_cost_estimator_gemini.py @@ -10,7 +10,7 @@ from __future__ import annotations -from switchyard.lib.cost_estimator import MODEL_PRICING, estimate_cost +from switchyard.cli.launchers.cost_estimator import MODEL_PRICING, estimate_cost _GEMINI_KEYS = ( "gcp/google/gemini-3.5-flash", diff --git a/tests/test_endpoint_state_contract.py b/tests/test_endpoint_state_contract.py deleted file mode 100644 index 7df4cbe28..000000000 --- a/tests/test_endpoint_state_contract.py +++ /dev/null @@ -1,119 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Contract tests for the ``app.state`` wiring between -``build_switchyard_app`` and the three inbound endpoint handlers. - -The bug fixed in PR #28 was that the factory wrote ``app.state.switchyard`` -while every endpoint read ``app.state.switchyard`` — the server started -cleanly and unit tests stayed green, but every request hit -``AttributeError`` at runtime. - -These tests pin both sides of the contract against ``Switchyard.state_key`` -so a future rename either updates everything together or fails CI. -""" - -from __future__ import annotations - -import inspect - -import pytest -from fastapi import FastAPI -from openai.types.chat import ChatCompletion -from openai.types.chat.chat_completion import Choice -from openai.types.chat.chat_completion_message import ChatCompletionMessage -from openai.types.completion_usage import CompletionUsage - -from switchyard.lib.endpoints.anthropic_messages_endpoint import ( - AnthropicMessagesEndpoint, -) -from switchyard.lib.endpoints.openai_chat_endpoint import ( - OpenAIChatEndpoint, -) -from switchyard.lib.endpoints.responses_endpoint import ( - ResponsesEndpoint, -) -from switchyard.lib.proxy_context import ProxyContext -from switchyard.lib.roles import LLMBackend -from switchyard.lib.switchyard import Switchyard -from switchyard.server.switchyard_app import build_switchyard_app -from switchyard_rust.core import ChatRequest, ChatRequestType, ChatResponse -from switchyard_rust.translation import TranslationEngine - -_ENDPOINT_CLASSES = ( - OpenAIChatEndpoint, - AnthropicMessagesEndpoint, - ResponsesEndpoint, -) -_ALL_REQUEST_TYPES = [ - ChatRequestType.OPENAI_CHAT, - ChatRequestType.OPENAI_RESPONSES, - ChatRequestType.ANTHROPIC, -] - - -class _StubBackend(LLMBackend): - def supported_request_types(self) -> list[ChatRequestType]: - return list(_ALL_REQUEST_TYPES) - - async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: - completion = ChatCompletion( - id="chatcmpl-stub", - object="chat.completion", - created=1700000000, - model="stub", - choices=[ - Choice( - index=0, - message=ChatCompletionMessage(role="assistant", content="ok"), - finish_reason="stop", - ) - ], - usage=CompletionUsage(prompt_tokens=1, completion_tokens=1, total_tokens=2), - ) - return ChatResponse.openai_completion(completion) - - -def _build_app() -> tuple[FastAPI, Switchyard]: - switchyard = Switchyard(backend=_StubBackend(), translator=TranslationEngine()) - app = build_switchyard_app(switchyard) - return app, switchyard - - -def test_state_key_constant_is_switchyard() -> None: - """Pin the expected key. If this changes, every reader must change too.""" - assert Switchyard.state_key == "switchyard" - - -def test_factory_writes_switchyard_to_state_key_constant() -> None: - """``build_switchyard_app`` must store the instance under ``Switchyard.state_key``. - - Direct regression guard for PR #28: the factory previously wrote to - ``app.state.switchyard`` (wrong) instead of ``app.state.switchyard``. - """ - app, switchyard = _build_app() - stored = getattr(app.state, Switchyard.state_key) - assert stored is switchyard - - -@pytest.mark.parametrize( - "endpoint_cls", - _ENDPOINT_CLASSES, - ids=lambda c: c.__name__, -) -def test_endpoint_module_reads_state_key_constant(endpoint_cls: type) -> None: - """Each endpoint module must read from the same key the factory writes. - - Source-level check rather than a runtime assertion because the contract - is "the literal attribute name agrees" — a runtime test still passes if - both sides are wrong-and-matching, but the source check fails the moment - either side drifts from ``Switchyard.state_key``. - """ - module = inspect.getmodule(endpoint_cls) - assert module is not None - expected = f"request.app.state.{Switchyard.state_key}" - source = inspect.getsource(module) - assert expected in source, ( - f"{endpoint_cls.__name__} does not read from {expected!r}; " - f"the factory and the endpoint disagree on app.state." - ) diff --git a/tests/test_error_source_annotation.py b/tests/test_error_source_annotation.py deleted file mode 100644 index 57710da9f..000000000 --- a/tests/test_error_source_annotation.py +++ /dev/null @@ -1,156 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""failure-source annotation on error responses and the event log. - -Every client-facing error carries ``x-switchyard-error-source`` naming the -layer that originated it (``switchyard`` | ``provider``), plus -``x-switchyard-upstream-model`` when a routing selection had happened. These -tests cover the endpoint layer that renders those annotations from the ctx -stamps a backend sets. -""" - -from __future__ import annotations - -import json -import logging - -import pytest - -from switchyard.lib.endpoints.dispatch import model_not_found_response -from switchyard.lib.endpoints.error_envelope import ( - ERROR_SOURCE_HEADER, - UPSTREAM_MODEL_HEADER, - error_response, - upstream_error_response, -) -from switchyard.lib.endpoints.upstream_error import ( - context_exhausted_response, - handle_chain_exception, - upstream_response_from_ctx, -) -from switchyard.lib.endpoints.upstream_error_log import log_upstream_attempt_failure -from switchyard.lib.proxy_context import ( - CTX_ERROR_SOURCE, - CTX_UPSTREAM_HTTP_BODY, - CTX_UPSTREAM_HTTP_STATUS, - CTX_UPSTREAM_MODEL, - ProxyContext, -) - -_UPSTREAM_401 = { - "error": {"message": "bad key", "type": "auth_error", "code": "invalid_api_key"} -} - - -# --- envelope builders ------------------------------------------------------- - - -def test_synthesized_envelope_defaults_to_switchyard_source() -> None: - resp = error_response(400, "bad", error_type="invalid_request_error", code="invalid_body") - assert resp.headers[ERROR_SOURCE_HEADER] == "switchyard" - assert UPSTREAM_MODEL_HEADER not in resp.headers - - -def test_upstream_envelope_labels_provider_and_upstream_model() -> None: - resp = upstream_error_response(429, _UPSTREAM_401, upstream_model="gpt-5") - assert resp.headers[ERROR_SOURCE_HEADER] == "provider" - assert resp.headers[UPSTREAM_MODEL_HEADER] == "gpt-5" - - -def test_model_not_found_labels_switchyard() -> None: - resp = model_not_found_response("nope") - assert resp.status_code == 404 - assert resp.headers[ERROR_SOURCE_HEADER] == "switchyard" - - -def test_context_exhausted_labels_switchyard() -> None: - resp = context_exhausted_response(RuntimeError("pool exhausted"), "openai") - assert resp.status_code == 400 - assert resp.headers[ERROR_SOURCE_HEADER] == "switchyard" - - -# --- ctx-driven recovery paths ---------------------------------------------- - - -def _ctx_with_upstream_stash(**extra: object) -> ProxyContext: - ctx = ProxyContext() - ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] = 401 - ctx.metadata[CTX_UPSTREAM_HTTP_BODY] = _UPSTREAM_401 - for key, value in extra.items(): - ctx.metadata[key] = value - return ctx - - -def test_stashed_status_defaults_to_provider() -> None: - """A backend that stashes an upstream status without a source label gets - the passthrough default — the stash channel exists for provider errors.""" - resp = upstream_response_from_ctx(_ctx_with_upstream_stash()) - assert resp is not None - assert resp.status_code == 401 - assert resp.headers[ERROR_SOURCE_HEADER] == "provider" - assert UPSTREAM_MODEL_HEADER not in resp.headers - - -def test_stashed_switchyard_source_overrides_provider_default() -> None: - """caller_required-style rejections ride the upstream channel but must - surface as switchyard-originated.""" - ctx = _ctx_with_upstream_stash(**{CTX_ERROR_SOURCE: "switchyard"}) - resp = upstream_response_from_ctx(ctx) - assert resp is not None - assert resp.headers[ERROR_SOURCE_HEADER] == "switchyard" - - -def test_stashed_upstream_model_reaches_header() -> None: - ctx = _ctx_with_upstream_stash(**{CTX_UPSTREAM_MODEL: "gpt-5"}) - resp = upstream_response_from_ctx(ctx) - assert resp is not None - assert resp.headers[UPSTREAM_MODEL_HEADER] == "gpt-5" - - -def test_unexpected_internal_500_labels_switchyard() -> None: - resp = handle_chain_exception( - RuntimeError("boom"), ProxyContext(), inbound="openai", log_msg="test failure" - ) - assert resp.status_code == 500 - assert resp.headers[ERROR_SOURCE_HEADER] == "switchyard" - assert UPSTREAM_MODEL_HEADER not in resp.headers - - -def test_network_failure_500_labels_provider() -> None: - """A status-less upstream fault (network error after retries) renders as - the internal 500 envelope but is labeled provider via the ctx stamp.""" - ctx = ProxyContext() - ctx.metadata[CTX_ERROR_SOURCE] = "provider" - ctx.metadata[CTX_UPSTREAM_MODEL] = "gpt-5" - - resp = handle_chain_exception( - RuntimeError("connection reset"), ctx, inbound="openai", log_msg="test failure" - ) - - assert resp.status_code == 500 - assert json.loads(bytes(resp.body))["error"]["code"] == "internal_chain_error" - assert resp.headers[ERROR_SOURCE_HEADER] == "provider" - assert resp.headers[UPSTREAM_MODEL_HEADER] == "gpt-5" - - -# --- structured event log ---------------------------------------------------- - - -def test_attempt_failure_log_carries_upstream_model_and_source( - caplog: pytest.LogCaptureFixture, -) -> None: - with caplog.at_level(logging.WARNING, logger="switchyard.upstream_errors"): - log_upstream_attempt_failure( - model="route-id", - attempt=1, - status_code=429, - error=RuntimeError("rate limited"), - upstream_model="gpt-5", - ) - - record = json.loads(caplog.records[-1].message) - assert record["model"] == "route-id" - assert record["upstream_model"] == "gpt-5" - assert record["error_source"] == "provider" - assert record["code"] == "429" diff --git a/tests/test_format_translate_processor.py b/tests/test_format_translate_processor.py deleted file mode 100644 index 1686723e1..000000000 --- a/tests/test_format_translate_processor.py +++ /dev/null @@ -1,593 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -import json -from collections.abc import AsyncIterator -from typing import Any - -import pytest -from openai.types.chat import ChatCompletionChunk - -from switchyard.lib.chat_response.anthropic import AnthropicResponseStream -from switchyard.lib.chat_response.openai_chat import ResponseStream -from switchyard.lib.chat_response.openai_responses import ResponsesApiStream -from switchyard.lib.processors.format_translate import ( - FormatTranslateResponseProcessor, - ModelFormatLookupProcessor, - StampOriginalFormatProcessor, - TranslateConfig, -) -from switchyard.lib.proxy_context import ( - CTX_ORIGINAL_FORMAT, - CTX_ORIGINAL_REQUEST, - CTX_PROXY_ACTUAL_MODEL, - CTX_TARGET_FORMAT, - ProxyContext, -) -from switchyard_rust.core import ( - ChatRequest, - ChatRequestType, - ChatResponse, - ChatResponseType, - request_type_value, - response_type_matches, -) - - -async def _aiter(items: list[Any]) -> AsyncIterator[Any]: - for item in items: - yield item - - -def _chat_chunk( - *, - content: str | None = None, - finish_reason: str | None = None, -) -> ChatCompletionChunk: - delta: dict[str, Any] = {} - if content is not None: - delta["content"] = content - return ChatCompletionChunk.model_validate( - { - "id": "chatcmpl-test", - "object": "chat.completion.chunk", - "created": 1700000000, - "model": "backend-model", - "choices": [ - { - "index": 0, - "delta": delta, - "finish_reason": finish_reason, - } - ], - } - ) - - -def _chat_tool_chunk( - *, - name: str | None = None, - arguments: str | None = None, - finish_reason: str | None = None, -) -> ChatCompletionChunk: - function: dict[str, str] = {} - if name is not None: - function["name"] = name - if arguments is not None: - function["arguments"] = arguments - return ChatCompletionChunk.model_validate( - { - "id": "chatcmpl-test", - "object": "chat.completion.chunk", - "created": 1700000000, - "model": "backend-model", - "choices": [ - { - "index": 0, - "delta": { - "tool_calls": [ - { - "index": 0, - "id": "call_search", - "type": "function", - "function": function, - } - ] - }, - "finish_reason": finish_reason, - } - ], - } - ) - - -def _anthropic_events() -> list[dict[str, Any]]: - return [ - { - "type": "message_start", - "message": { - "id": "msg_test", - "type": "message", - "role": "assistant", - "content": [], - "model": "claude-upstream", - "stop_reason": None, - "stop_sequence": None, - "usage": {"input_tokens": 3, "output_tokens": 0}, - }, - }, - { - "type": "content_block_start", - "index": 0, - "content_block": {"type": "text", "text": ""}, - }, - { - "type": "content_block_delta", - "index": 0, - "delta": {"type": "text_delta", "text": "hello"}, - }, - {"type": "content_block_stop", "index": 0}, - { - "type": "message_delta", - "delta": {"stop_reason": "end_turn", "stop_sequence": None}, - "usage": {"output_tokens": 1}, - }, - {"type": "message_stop"}, - ] - - -def _anthropic_tool_events() -> list[dict[str, Any]]: - return [ - { - "type": "message_start", - "message": { - "id": "msg_tool", - "type": "message", - "role": "assistant", - "content": [], - "model": "claude-upstream", - "stop_reason": None, - "stop_sequence": None, - "usage": {"input_tokens": 9, "output_tokens": 0}, - }, - }, - { - "type": "content_block_start", - "index": 0, - "content_block": { - "type": "tool_use", - "id": "toolu_weather", - "name": "get_weather", - "input": {}, - }, - }, - { - "type": "content_block_delta", - "index": 0, - "delta": {"type": "input_json_delta", "partial_json": '{"city":"SF"}'}, - }, - {"type": "content_block_stop", "index": 0}, - { - "type": "message_delta", - "delta": {"stop_reason": "tool_use", "stop_sequence": None}, - "usage": {"output_tokens": 4}, - }, - {"type": "message_stop"}, - ] - - -def _responses_events() -> list[dict[str, Any]]: - return [ - { - "type": "response.created", - "response": { - "id": "resp_test", - "object": "response", - "created_at": 1700000000, - "status": "in_progress", - "model": "responses-upstream", - "output": [], - }, - }, - { - "type": "response.output_item.added", - "output_index": 0, - "item": { - "type": "message", - "id": "msg_test", - "role": "assistant", - "status": "in_progress", - "content": [], - }, - }, - { - "type": "response.output_text.delta", - "output_index": 0, - "content_index": 0, - "delta": "hello", - }, - { - "type": "response.completed", - "response": { - "id": "resp_test", - "object": "response", - "created_at": 1700000000, - "status": "completed", - "model": "responses-upstream", - "output": [ - { - "type": "message", - "id": "msg_test", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": "hello"}], - } - ], - "usage": {"input_tokens": 5, "output_tokens": 2, "total_tokens": 7}, - }, - }, - ] - - -async def _collect(stream: AsyncIterator[Any]) -> list[Any]: - return [item async for item in stream] - - -def _frame_data(frame: str) -> dict[str, Any]: - data_line = next(line for line in frame.splitlines() if line.startswith("data: ")) - return json.loads(data_line.removeprefix("data: ")) - - -async def test_model_format_lookup_prefers_rust_selected_model() -> None: - processor = ModelFormatLookupProcessor( - TranslateConfig(models=[ - {"model": "rust-selected", "backend_format": "anthropic"}, - {"model": "legacy-selected", "backend_format": "openai"}, - {"model": "responses-selected", "backend_format": "responses"}, - ]) - ) - ctx = ProxyContext(metadata={CTX_PROXY_ACTUAL_MODEL: "legacy-selected"}) - ctx.selected_model = "rust-selected" - - await processor.process( - ctx, - ChatRequest.openai_chat({"model": "client-model", "messages": []}), - ) - - assert ctx.metadata[CTX_TARGET_FORMAT] == ChatRequestType.ANTHROPIC - - -async def test_model_format_lookup_preserves_responses_target() -> None: - processor = ModelFormatLookupProcessor( - TranslateConfig(models=[{"model": "responses-selected", "backend_format": "responses"}]) - ) - ctx = ProxyContext() - ctx.selected_model = "responses-selected" - - await processor.process( - ctx, - ChatRequest.openai_responses({"model": "client-model", "input": "hi"}), - ) - - assert ctx.metadata[CTX_TARGET_FORMAT] == ChatRequestType.OPENAI_RESPONSES - - -async def test_model_format_lookup_falls_back_to_legacy_metadata() -> None: - processor = ModelFormatLookupProcessor( - TranslateConfig(models=[{"model": "legacy-selected", "backend_format": "openai"}]) - ) - ctx = ProxyContext(metadata={CTX_PROXY_ACTUAL_MODEL: "legacy-selected"}) - - await processor.process( - ctx, - ChatRequest.anthropic({"model": "client-model", "messages": []}), - ) - - assert ctx.metadata[CTX_TARGET_FORMAT] == ChatRequestType.OPENAI_CHAT - - -@pytest.mark.asyncio -async def test_stamp_original_format_preserves_original_body_snapshot() -> None: - body = {"model": "gpt-client", "messages": [{"role": "user", "content": "hi"}]} - ctx = ProxyContext() - - await StampOriginalFormatProcessor().process(ctx, ChatRequest.openai_chat(body)) - body["messages"][0]["content"] = "mutated" - - assert request_type_value(ctx.metadata[CTX_ORIGINAL_FORMAT]) == "openai_chat" - assert ctx.metadata[CTX_ORIGINAL_REQUEST] == { - "model": "gpt-client", - "messages": [{"role": "user", "content": "hi"}], - } - - -@pytest.mark.asyncio -async def test_streaming_openai_response_translates_back_to_anthropic() -> None: - ctx = ProxyContext( - metadata={ - CTX_ORIGINAL_FORMAT: ChatRequestType.ANTHROPIC, - CTX_ORIGINAL_REQUEST: {"model": "claude-client"}, - } - ) - ctx.selected_model = "served/model" - response = ChatResponse.openai_stream( - ResponseStream( - _aiter( - [ - _chat_chunk(content="hel"), - _chat_chunk(content="lo", finish_reason="stop"), - ] - ) - ) - ) - - result = await FormatTranslateResponseProcessor().process(ctx, response) - - assert response_type_matches(result, ChatResponseType.ANTHROPIC_STREAM) - events = await _collect(result.stream) - text = "".join( - event["delta"]["text"] - for event in events - if event["type"] == "content_block_delta" and event["delta"]["type"] == "text_delta" - ) - assert events[0]["message"]["model"] == "served/model" - assert text == "hello" - assert [event["type"] for event in events].count("content_block_start") == 1 - - -@pytest.mark.asyncio -async def test_streaming_openai_length_finish_translates_to_anthropic_max_tokens() -> None: - ctx = ProxyContext( - metadata={ - CTX_ORIGINAL_FORMAT: ChatRequestType.ANTHROPIC, - CTX_ORIGINAL_REQUEST: {"model": "claude-client"}, - } - ) - response = ChatResponse.openai_stream( - ResponseStream(_aiter([_chat_chunk(content="truncated", finish_reason="length")])) - ) - - result = await FormatTranslateResponseProcessor().process(ctx, response) - - assert response_type_matches(result, ChatResponseType.ANTHROPIC_STREAM) - events = await _collect(result.stream) - message_delta = next(event for event in events if event["type"] == "message_delta") - assert message_delta["delta"]["stop_reason"] == "max_tokens" - - -@pytest.mark.asyncio -async def test_streaming_openai_response_translates_back_to_responses() -> None: - ctx = ProxyContext( - metadata={ - CTX_ORIGINAL_FORMAT: ChatRequestType.OPENAI_RESPONSES, - CTX_ORIGINAL_REQUEST: {"model": "responses-client", "input": "hi"}, - } - ) - ctx.selected_model = "served/model" - response = ChatResponse.openai_stream( - ResponseStream( - _aiter( - [ - _chat_chunk(content="hel"), - _chat_chunk(content="lo", finish_reason="stop"), - ] - ) - ) - ) - - result = await FormatTranslateResponseProcessor().process(ctx, response) - - assert response_type_matches(result, ChatResponseType.OPENAI_RESPONSES_STREAM) - frames = await _collect(result.stream) - payloads = [_frame_data(frame) for frame in frames] - assert payloads[0]["response"]["model"] == "served/model" - assert ( - "".join( - payload["delta"] - for payload in payloads - if payload["type"] == "response.output_text.delta" - ) - == "hello" - ) - assert payloads[-1]["type"] == "response.completed" - - -@pytest.mark.asyncio -async def test_streaming_openai_to_responses_keeps_unique_output_indexes() -> None: - ctx = ProxyContext( - metadata={ - CTX_ORIGINAL_FORMAT: ChatRequestType.OPENAI_RESPONSES, - CTX_ORIGINAL_REQUEST: {"model": "responses-client", "input": "search"}, - } - ) - response = ChatResponse.openai_stream( - ResponseStream( - _aiter( - [ - _chat_tool_chunk(name="search", arguments='{"q":"x"}'), - _chat_chunk(content="Checking"), - _chat_chunk(finish_reason="tool_calls"), - ] - ) - ) - ) - - result = await FormatTranslateResponseProcessor().process(ctx, response) - - assert response_type_matches(result, ChatResponseType.OPENAI_RESPONSES_STREAM) - payloads = [_frame_data(frame) for frame in await _collect(result.stream)] - added = [ - (payload["output_index"], payload["item"]["type"]) - for payload in payloads - if payload["type"] == "response.output_item.added" - ] - assert added == [(0, "function_call"), (1, "message")] - - function_index = added[0][0] - args_done = next( - payload - for payload in payloads - if payload["type"] == "response.function_call_arguments.done" - ) - assert args_done["output_index"] == function_index - - completed = payloads[-1]["response"] - assert [item["type"] for item in completed["output"]] == [ - "function_call", - "message", - ] - - -@pytest.mark.asyncio -async def test_streaming_anthropic_response_translates_back_to_responses() -> None: - ctx = ProxyContext( - metadata={ - CTX_ORIGINAL_FORMAT: ChatRequestType.OPENAI_RESPONSES, - CTX_ORIGINAL_REQUEST: {"model": "responses-client", "input": "hi"}, - } - ) - response = ChatResponse.anthropic_stream( - AnthropicResponseStream(_aiter(_anthropic_events())), - ) - - result = await FormatTranslateResponseProcessor().process(ctx, response) - - assert response_type_matches(result, ChatResponseType.OPENAI_RESPONSES_STREAM) - payloads = [_frame_data(frame) for frame in await _collect(result.stream)] - assert ( - "".join( - payload["delta"] - for payload in payloads - if payload["type"] == "response.output_text.delta" - ) - == "hello" - ) - assert payloads[-1]["type"] == "response.completed" - - -@pytest.mark.asyncio -async def test_streaming_anthropic_response_translates_back_to_openai() -> None: - ctx = ProxyContext( - metadata={ - CTX_ORIGINAL_FORMAT: ChatRequestType.OPENAI_CHAT, - CTX_ORIGINAL_REQUEST: {"model": "gpt-client"}, - } - ) - response = ChatResponse.anthropic_stream( - AnthropicResponseStream(_aiter(_anthropic_events())), - ) - - result = await FormatTranslateResponseProcessor().process(ctx, response) - - assert response_type_matches(result, ChatResponseType.OPENAI_STREAM) - chunks = [chunk.model_dump(exclude_none=True) for chunk in await _collect(result.stream)] - assert chunks[0]["choices"][0]["delta"] == {"role": "assistant"} - assert chunks[1]["model"] == "claude-upstream" - assert chunks[1]["choices"][0]["delta"]["content"] == "hello" - assert chunks[-1]["choices"][0]["finish_reason"] == "stop" - assert chunks[-1]["usage"]["prompt_tokens"] == 3 - assert chunks[-1]["usage"]["completion_tokens"] == 1 - - -@pytest.mark.asyncio -async def test_streaming_anthropic_cache_usage_translates_back_to_openai() -> None: - ctx = ProxyContext( - metadata={ - CTX_ORIGINAL_FORMAT: ChatRequestType.OPENAI_CHAT, - CTX_ORIGINAL_REQUEST: {"model": "gpt-client"}, - } - ) - events = _anthropic_events() - events[0]["message"]["usage"] = { - "input_tokens": 3, - "cache_creation_input_tokens": 4, - "cache_read_input_tokens": 2, - "output_tokens": 0, - } - response = ChatResponse.anthropic_stream(AnthropicResponseStream(_aiter(events))) - - result = await FormatTranslateResponseProcessor().process(ctx, response) - - assert response_type_matches(result, ChatResponseType.OPENAI_STREAM) - chunks = [chunk.model_dump(exclude_none=True) for chunk in await _collect(result.stream)] - usage = chunks[-1]["usage"] - assert usage["prompt_tokens"] == 9 - assert usage["completion_tokens"] == 1 - assert usage["total_tokens"] == 10 - assert usage["prompt_tokens_details"]["cached_tokens"] == 2 - assert usage["prompt_tokens_details"]["cache_creation_tokens"] == 4 - - -@pytest.mark.asyncio -async def test_streaming_responses_response_translates_back_to_openai() -> None: - ctx = ProxyContext( - metadata={ - CTX_ORIGINAL_FORMAT: ChatRequestType.OPENAI_CHAT, - CTX_ORIGINAL_REQUEST: {"model": "gpt-client"}, - } - ) - response = ChatResponse.openai_responses_stream( - ResponsesApiStream(_aiter(_responses_events())), - ) - - result = await FormatTranslateResponseProcessor().process(ctx, response) - - assert response_type_matches(result, ChatResponseType.OPENAI_STREAM) - chunks = [chunk.model_dump(exclude_none=True) for chunk in await _collect(result.stream)] - assert chunks[0]["choices"][0]["delta"] == {"role": "assistant"} - assert chunks[1]["choices"][0]["delta"]["content"] == "hello" - assert chunks[-1]["choices"][0]["finish_reason"] == "stop" - assert chunks[-1]["usage"]["prompt_tokens"] == 5 - assert chunks[-1]["usage"]["completion_tokens"] == 2 - - -@pytest.mark.asyncio -async def test_streaming_responses_response_translates_back_to_anthropic() -> None: - ctx = ProxyContext( - metadata={ - CTX_ORIGINAL_FORMAT: ChatRequestType.ANTHROPIC, - CTX_ORIGINAL_REQUEST: {"model": "claude-client"}, - } - ) - ctx.selected_model = "served/model" - response = ChatResponse.openai_responses_stream( - ResponsesApiStream(_aiter(_responses_events())), - ) - - result = await FormatTranslateResponseProcessor().process(ctx, response) - - assert response_type_matches(result, ChatResponseType.ANTHROPIC_STREAM) - events = await _collect(result.stream) - text = "".join( - event["delta"]["text"] - for event in events - if event["type"] == "content_block_delta" and event["delta"]["type"] == "text_delta" - ) - assert events[0]["message"]["model"] == "served/model" - assert text == "hello" - - -@pytest.mark.asyncio -async def test_streaming_anthropic_tool_use_translates_back_to_openai_tool_call() -> None: - ctx = ProxyContext( - metadata={ - CTX_ORIGINAL_FORMAT: ChatRequestType.OPENAI_CHAT, - CTX_ORIGINAL_REQUEST: {"model": "gpt-client"}, - } - ) - response = ChatResponse.anthropic_stream( - AnthropicResponseStream(_aiter(_anthropic_tool_events())), - ) - - result = await FormatTranslateResponseProcessor().process(ctx, response) - - assert response_type_matches(result, ChatResponseType.OPENAI_STREAM) - chunks = [chunk.model_dump(exclude_none=True) for chunk in await _collect(result.stream)] - first_tool_delta = chunks[1]["choices"][0]["delta"]["tool_calls"][0] - args_delta = chunks[2]["choices"][0]["delta"]["tool_calls"][0] - assert first_tool_delta["id"] == "toolu_weather" - assert first_tool_delta["function"]["name"] == "get_weather" - assert args_delta["function"]["arguments"] == '{"city":"SF"}' - assert chunks[-1]["choices"][0]["finish_reason"] == "tool_calls" diff --git a/tests/test_inference_e2e.py b/tests/test_inference_e2e.py deleted file mode 100644 index f8420b46d..000000000 --- a/tests/test_inference_e2e.py +++ /dev/null @@ -1,599 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""End-to-end inference tests using a mock LLM backend. - -Exercises the full HTTP stack — FastAPI endpoints → Switchyard chain → -mock backend → response translation — without touching any live LLM provider. - -All tests run offline: no API keys, no network access, no running model. -""" - -from __future__ import annotations - -import json -from collections.abc import AsyncIterator -from unittest.mock import AsyncMock, patch - -import httpx -import pytest -from fastapi import FastAPI -from fastapi.testclient import TestClient -from openai.types.chat import ChatCompletion, ChatCompletionChunk -from openai.types.chat.chat_completion import Choice -from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice -from openai.types.chat.chat_completion_chunk import ChoiceDelta -from openai.types.chat.chat_completion_message import ChatCompletionMessage -from openai.types.chat.chat_completion_message_function_tool_call import ( - ChatCompletionMessageFunctionToolCall, - Function, -) -from openai.types.completion_usage import CompletionUsage - -from switchyard.lib.chat_response.openai_chat import ResponseStream -from switchyard.lib.proxy_context import ProxyContext -from switchyard.lib.roles import LLMBackend -from switchyard.lib.switchyard import Switchyard -from switchyard.server.switchyard_app import build_switchyard_app -from switchyard_rust.core import ChatRequest, ChatRequestType, ChatResponse -from switchyard_rust.translation import TranslationEngine - -# --------------------------------------------------------------------------- -# Mock LLM backends -# --------------------------------------------------------------------------- - -_REPLY = "hello back" -_ALL_REQUEST_TYPES = [ - ChatRequestType.OPENAI_CHAT, - ChatRequestType.OPENAI_RESPONSES, - ChatRequestType.ANTHROPIC, -] - - -class _MockLLMBackend(LLMBackend): - """Returns a canned OpenAI completion response for every call.""" - - def __init__(self, completion: ChatCompletion) -> None: - self._completion = completion - - def supported_request_types(self) -> list[ChatRequestType]: - return list(_ALL_REQUEST_TYPES) - - async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: - return ChatResponse.openai_completion(self._completion) - - -class _StreamingMockLLMBackend(LLMBackend): - """Returns a fixed OpenAI stream response for every call.""" - - def __init__(self, chunks: list[ChatCompletionChunk]) -> None: - self._chunks = list(chunks) - - def supported_request_types(self) -> list[ChatRequestType]: - return list(_ALL_REQUEST_TYPES) - - async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: - chunks = self._chunks - - async def _iter() -> AsyncIterator[ChatCompletionChunk]: - for chunk in chunks: - yield chunk - - return ChatResponse.openai_stream(ResponseStream(_iter())) - - -class _RaisingMockLLMBackend(LLMBackend): - """Raises ``exc`` for every call — exercises the error path through the chain.""" - - def __init__(self, exc: BaseException) -> None: - self._exc = exc - - def supported_request_types(self) -> list[ChatRequestType]: - return list(_ALL_REQUEST_TYPES) - - async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: - raise self._exc - - -class _TranslatingMockLLMBackend(LLMBackend): - """Chat-only backend that translates the inbound request to Chat at the top of - ``call`` — mirroring any real chat-completions backend (native / passthrough). - - An unsupported inbound field (e.g. a bad message role) is rejected by the - translation engine here, before any upstream call, exactly as it would be for - a production chat backend. - """ - - def supported_request_types(self) -> list[ChatRequestType]: - return [ChatRequestType.OPENAI_CHAT] - - async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: - request = TranslationEngine().request_to_any_of(request, [ChatRequestType.OPENAI_CHAT]) - return ChatResponse.openai_completion(_make_completion()) - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _make_completion(*, model: str = "mock-model", content: str = _REPLY) -> ChatCompletion: - return ChatCompletion( - id="chatcmpl-test", - object="chat.completion", - created=1700000000, - model=model, - choices=[ - Choice( - index=0, - message=ChatCompletionMessage(role="assistant", content=content), - finish_reason="stop", - ) - ], - usage=CompletionUsage(prompt_tokens=5, completion_tokens=3, total_tokens=8), - ) - - -def _make_tool_call_completion( - *, - model: str = "mock-model", - tool_name: str = "get_weather", - tool_args: str = '{"city": "Paris"}', - tool_call_id: str = "call_test_123", -) -> ChatCompletion: - """Completion whose assistant message contains a single tool call.""" - return ChatCompletion( - id="chatcmpl-test-tool", - object="chat.completion", - created=1700000000, - model=model, - choices=[ - Choice( - index=0, - message=ChatCompletionMessage( - role="assistant", - content=None, - tool_calls=[ - ChatCompletionMessageFunctionToolCall( - id=tool_call_id, - type="function", - function=Function(name=tool_name, arguments=tool_args), - ) - ], - ), - finish_reason="tool_calls", - ) - ], - usage=CompletionUsage(prompt_tokens=20, completion_tokens=10, total_tokens=30), - ) - - -def _make_chunks(*, model: str = "mock-model", content: str = _REPLY) -> list[ChatCompletionChunk]: - return [ - ChatCompletionChunk( - id="chatcmpl-stream", - object="chat.completion.chunk", - created=1700000000, - model=model, - choices=[ - ChunkChoice( - index=0, - delta=ChoiceDelta(role="assistant", content=content), - finish_reason=None, - ) - ], - ), - ChatCompletionChunk( - id="chatcmpl-stream", - object="chat.completion.chunk", - created=1700000000, - model=model, - choices=[ChunkChoice(index=0, delta=ChoiceDelta(), finish_reason="stop")], - usage=CompletionUsage(prompt_tokens=5, completion_tokens=3, total_tokens=8), - ), - ] - - -# --------------------------------------------------------------------------- -# Fixtures -# --------------------------------------------------------------------------- - - -@pytest.fixture -def app() -> FastAPI: - switchyard = Switchyard( - backend=_MockLLMBackend(_make_completion()), - translator=TranslationEngine(), - ) - return build_switchyard_app(switchyard) - - -@pytest.fixture -def streaming_app() -> FastAPI: - switchyard = Switchyard( - backend=_StreamingMockLLMBackend(_make_chunks()), - translator=TranslationEngine(), - ) - return build_switchyard_app(switchyard) - - -@pytest.fixture -async def client(app: FastAPI) -> AsyncIterator[httpx.AsyncClient]: - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=app), - base_url="http://test", - ) as c: - yield c - - -@pytest.fixture -async def streaming_client(streaming_app: FastAPI) -> AsyncIterator[httpx.AsyncClient]: - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=streaming_app), - base_url="http://test", - ) as c: - yield c - - -@pytest.fixture -def tool_call_app() -> FastAPI: - """App backed by a mock that always returns a tool-calling completion.""" - switchyard = Switchyard( - backend=_MockLLMBackend(_make_tool_call_completion()), - translator=TranslationEngine(), - ) - return build_switchyard_app(switchyard) - - -@pytest.fixture -async def tool_call_client(tool_call_app: FastAPI) -> AsyncIterator[httpx.AsyncClient]: - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=tool_call_app), - base_url="http://test", - ) as c: - yield c - - -@pytest.fixture -def raising_app() -> FastAPI: - """App backed by a mock that always raises a generic backend error.""" - switchyard = Switchyard( - backend=_RaisingMockLLMBackend(RuntimeError("backend boom")), - translator=TranslationEngine(), - ) - return build_switchyard_app(switchyard) - - -@pytest.fixture -async def raising_client(raising_app: FastAPI) -> AsyncIterator[httpx.AsyncClient]: - # ``raise_app_exceptions=False`` mirrors uvicorn's behavior in production: - # an unhandled exception inside a route is mapped to a 500 response, not - # propagated up the call stack. Required for the error-path tests below. - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=raising_app, raise_app_exceptions=False), - base_url="http://test", - ) as c: - yield c - - -@pytest.fixture -def translating_app() -> FastAPI: - """App whose chat-only backend translates the inbound request to Chat, so an - invalid inbound role is rejected during translation before any upstream call.""" - switchyard = Switchyard( - backend=_TranslatingMockLLMBackend(), - translator=TranslationEngine(), - ) - return build_switchyard_app(switchyard) - - -# --------------------------------------------------------------------------- -# Tests -# --------------------------------------------------------------------------- - - -class TestInferenceE2E: - - async def test_health_liveness(self, client: httpx.AsyncClient) -> None: - resp = await client.get("/health") - assert resp.status_code == 200 - assert resp.json() == {"status": "ok"} - - async def test_openai_chat_completions(self, client: httpx.AsyncClient) -> None: - resp = await client.post( - "/v1/chat/completions", - json={ - "model": "any-model", - "messages": [{"role": "user", "content": "hello"}], - }, - ) - assert resp.status_code == 200 - data = resp.json() - choice = data["choices"][0] - assert choice["message"]["role"] == "assistant" - assert choice["message"]["content"] == _REPLY - assert choice["finish_reason"] == "stop" - - async def test_openai_chat_completions_streaming( - self, streaming_client: httpx.AsyncClient - ) -> None: - resp = await streaming_client.post( - "/v1/chat/completions", - json={ - "model": "any-model", - "messages": [{"role": "user", "content": "hello"}], - "stream": True, - }, - ) - assert resp.status_code == 200 - assert "text/event-stream" in resp.headers["content-type"] - - data_lines = [ - line[6:] # strip leading "data: " - for line in resp.text.split("\n") - if line.startswith("data: ") and line != "data: [DONE]" - ] - assert data_lines, "expected at least one data frame before [DONE]" - - chunks = [json.loads(line) for line in data_lines] - content = "".join( - c["choices"][0]["delta"].get("content", "") - for c in chunks - if c.get("choices") - ) - assert content == _REPLY - - async def test_anthropic_messages(self, client: httpx.AsyncClient) -> None: - resp = await client.post( - "/v1/messages", - json={ - "model": "any-model", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hello"}], - }, - ) - assert resp.status_code == 200 - data = resp.json() - assert data["type"] == "message" - assert data["role"] == "assistant" - assert data["content"][0]["type"] == "text" - assert data["content"][0]["text"] == _REPLY - assert data["stop_reason"] == "end_turn" - - async def test_openai_responses_api(self, client: httpx.AsyncClient) -> None: - resp = await client.post( - "/v1/responses", - json={ - "model": "any-model", - "input": "hello", - }, - ) - assert resp.status_code == 200 - data = resp.json() - assert "output" in data - msg = data["output"][0] - assert msg["type"] == "message" - assert msg["role"] == "assistant" - assert msg["content"][0]["type"] == "output_text" - assert msg["content"][0]["text"] == _REPLY - - async def test_anthropic_messages_streaming( - self, streaming_client: httpx.AsyncClient - ) -> None: - """Streaming through the Anthropic inbound — chunks must be Anthropic SSE shape.""" - resp = await streaming_client.post( - "/v1/messages", - json={ - "model": "any-model", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hello"}], - "stream": True, - }, - ) - assert resp.status_code == 200, resp.text - assert "text/event-stream" in resp.headers["content-type"] - - events = [ - json.loads(line[6:]) - for line in resp.text.split("\n") - if line.startswith("data: ") and line != "data: [DONE]" - ] - assert events, "expected at least one Anthropic SSE event" - - # Anthropic streaming uses an event-typed envelope (message_start / - # content_block_delta / message_stop). Pin the contract: at least - # one event carries the canonical Anthropic event 'type' field. - anthropic_event_types = {"message_start", "content_block_delta", "message_stop"} - seen_types = {e.get("type") for e in events} - assert seen_types & anthropic_event_types, ( - f"no Anthropic-shaped event types in stream: {seen_types}" - ) - - # The reply text should be reconstructible from text deltas. - text_deltas = [ - e["delta"]["text"] - for e in events - if e.get("type") == "content_block_delta" - and e.get("delta", {}).get("type") == "text_delta" - ] - if text_deltas: - assert "".join(text_deltas) == _REPLY - - async def test_openai_responses_streaming( - self, streaming_client: httpx.AsyncClient - ) -> None: - """Streaming through the Responses API inbound — chunks must be Responses SSE shape.""" - resp = await streaming_client.post( - "/v1/responses", - json={ - "model": "any-model", - "input": "hello", - "stream": True, - }, - ) - assert resp.status_code == 200, resp.text - assert "text/event-stream" in resp.headers["content-type"] - - events = [ - json.loads(line[6:]) - for line in resp.text.split("\n") - if line.startswith("data: ") and line != "data: [DONE]" - ] - assert events, "expected at least one Responses SSE event" - - # Responses streaming events all carry a 'type' starting with - # 'response.' — a regression that emits Chat Completions chunks - # on this endpoint would fail this assertion. - types = {e.get("type", "") for e in events} - assert any(t.startswith("response.") for t in types), ( - f"no Responses-shaped event types in stream: {types}" - ) - - async def test_backend_exception_returns_500( - self, raising_client: httpx.AsyncClient - ) -> None: - """A backend that raises must surface as an HTTP error, not 200 with garbage.""" - resp = await raising_client.post( - "/v1/chat/completions", - json={ - "model": "any-model", - "messages": [{"role": "user", "content": "hello"}], - }, - ) - assert resp.status_code == 500 - assert resp.headers["content-type"].startswith("application/json") - body = resp.json() - assert body["error"]["type"] == "internal_error" - assert body["error"]["code"] == "internal_chain_error" - assert "backend boom" in body["error"]["message"] - - -class TestToolCallRoundTrip: - """Backend returns OpenAI tool_calls; verify each inbound format renders correctly. - - Tool-call translation is a frequent break point in refactors because - the format-specific shapes diverge sharply (OpenAI: ``tool_calls`` on - the assistant message; Anthropic: a ``tool_use`` block in ``content``; - Responses: a top-level ``function_call`` output item). - """ - - async def test_openai_tool_calls_passthrough( - self, tool_call_client: httpx.AsyncClient - ) -> None: - resp = await tool_call_client.post( - "/v1/chat/completions", - json={ - "model": "any-model", - "messages": [{"role": "user", "content": "what's the weather?"}], - "tools": [ - { - "type": "function", - "function": { - "name": "get_weather", - "parameters": {"type": "object"}, - }, - } - ], - }, - ) - assert resp.status_code == 200, resp.text - choice = resp.json()["choices"][0] - assert choice["finish_reason"] == "tool_calls" - tool_calls = choice["message"]["tool_calls"] - assert len(tool_calls) == 1 - assert tool_calls[0]["function"]["name"] == "get_weather" - assert tool_calls[0]["function"]["arguments"] == '{"city": "Paris"}' - - async def test_anthropic_tool_use_translation( - self, tool_call_client: httpx.AsyncClient - ) -> None: - """OpenAI ``tool_calls`` from the backend must be translated to Anthropic ``tool_use``.""" - resp = await tool_call_client.post( - "/v1/messages", - json={ - "model": "any-model", - "max_tokens": 100, - "messages": [{"role": "user", "content": "what's the weather?"}], - "tools": [ - { - "name": "get_weather", - "input_schema": {"type": "object"}, - } - ], - }, - ) - assert resp.status_code == 200, resp.text - body = resp.json() - assert body["type"] == "message" - assert body["stop_reason"] == "tool_use" - - tool_use_blocks = [b for b in body["content"] if b.get("type") == "tool_use"] - assert len(tool_use_blocks) == 1, f"expected one tool_use block, got: {body['content']}" - block = tool_use_blocks[0] - assert block["name"] == "get_weather" - assert block["input"] == {"city": "Paris"} - - -class TestEndpointErrorContract: - """Endpoint error-mapping contracts previously covered only through the - removed latency-service backend, re-vehicled onto in-process backends.""" - - def test_post_dispatch_exception_returns_json_500(self, app: FastAPI) -> None: - """An exception raised AFTER dispatch (e.g. during result serialization) - must surface as a JSON 500 envelope, not FastAPI's plain-text 500.""" - - class _Unserializable: - def model_dump(self) -> None: - raise RuntimeError("serialization exploded") - - with patch( - "switchyard.lib.endpoints.openai_chat_endpoint.dispatch_chat_request", - new_callable=AsyncMock, - return_value=_Unserializable(), - ): - with TestClient(app, raise_server_exceptions=False) as client: - response = client.post( - "/v1/chat/completions", - json={"model": "any-model", "messages": [{"role": "user", "content": "hi"}]}, - ) - - assert response.status_code == 500 - assert response.headers["content-type"].startswith("application/json") - body = response.json() - assert body["error"]["type"] == "internal_error" - assert body["error"]["code"] == "internal_chain_error" - assert "serialization exploded" in body["error"]["message"] - - @pytest.mark.parametrize( - ("path", "payload"), - [ - ( - "/v1/responses", - {"model": "m", "input": [{"type": "message", "role": "api", "content": "ping"}]}, - ), - ( - "/v1/messages", - {"model": "m", "max_tokens": 16, "messages": [{"role": "api", "content": "ping"}]}, - ), - ], - ) - def test_invalid_inbound_role_surfaces_as_internal_error( - self, translating_app: FastAPI, path: str, payload: dict[str, object] - ) -> None: - """An unsupported inbound message role is rejected by the translation engine - when a chat-only backend translates the request, before any upstream call. - - NOTE: this currently surfaces as a 500 ``internal_error``. The removed - latency-service backend special-cased translation ``invalid_value`` errors - into a provider-compatible 400; no surviving backend reproduces that, so - the standalone-server contract is now a 500. Update this test (and the - endpoint/translation layer) if a 400 is restored. - """ - with TestClient(translating_app, raise_server_exceptions=False) as client: - response = client.post(path, json=payload) - - assert response.status_code == 500 - body = response.json() - assert body["error"]["type"] == "internal_error" - assert '"api"' in body["error"]["message"] - assert "role" in body["error"]["message"] diff --git a/tests/test_infra.py b/tests/test_infra.py deleted file mode 100644 index d4b61ebad..000000000 --- a/tests/test_infra.py +++ /dev/null @@ -1,146 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for core role helpers and plain processor components.""" - -import pytest -from openai.types.chat import ChatCompletion -from openai.types.chat.chat_completion import Choice -from openai.types.chat.chat_completion_message import ChatCompletionMessage - -from switchyard.lib.proxy_context import ProxyContext -from switchyard.lib.roles import LLMBackend -from switchyard_rust.core import ( - ChatRequest, - ChatRequestType, - ChatResponse, - ChatResponseType, - response_type_matches, -) - -# --------------------------------------------------------------------------- -# Fixtures -# --------------------------------------------------------------------------- - - -def make_ctx() -> ProxyContext: - return ProxyContext() - - -def make_request() -> ChatRequest: - return ChatRequest.openai_chat({"model": "gpt-4o", "messages": []}) - - -# --------------------------------------------------------------------------- -# Backend instantiation guard -# --------------------------------------------------------------------------- - - -class TestABCsCannotBeInstantiated: - def test_llm_backend(self): - with pytest.raises(TypeError): - LLMBackend() # type: ignore[abstract] - - -# --------------------------------------------------------------------------- -# Concrete implementations for testing -# --------------------------------------------------------------------------- - - -class PassthroughProcessor: - async def process(self, ctx, request): - return request - - -class TaggedRequestProcessor: - def __init__(self, tag: str) -> None: - self._tag = tag - - async def process(self, ctx, request): - ctx.metadata.setdefault("request_order", []).append(self._tag) - return request - - -class EchoBackend(LLMBackend): - @property - def supported_request_types(self): - return [ChatRequestType.OPENAI_CHAT] - - async def call(self, ctx, request): - from openai.types.completion_usage import CompletionUsage - - return ChatResponse.openai_completion( - ChatCompletion( - id="chatcmpl-test", - object="chat.completion", - created=1700000000, - model="gpt-4o", - choices=[ - Choice( - index=0, - message=ChatCompletionMessage(role="assistant", content="echo"), - finish_reason="stop", - ) - ], - usage=CompletionUsage(prompt_tokens=1, completion_tokens=1, total_tokens=2), - ) - ) - - -class NoopResponseProcessor: - async def process(self, ctx, response): - return response - - -class TaggedResponseProcessor: - def __init__(self, tag: str) -> None: - self._tag = tag - - async def process(self, ctx, response): - ctx.metadata.setdefault("response_order", []).append(self._tag) - return response - - -# --------------------------------------------------------------------------- -# Request-side component -# --------------------------------------------------------------------------- - - -class TestRequestProcessor: - async def test_process(self): - proc = PassthroughProcessor() - req = make_request() - result = await proc.process(make_ctx(), req) - assert result is req - - -# --------------------------------------------------------------------------- -# LLMBackend -# --------------------------------------------------------------------------- - - -class TestLLMBackend: - async def test_call(self): - backend = EchoBackend() - resp = await backend.call(make_ctx(), make_request()) - assert response_type_matches(resp, ChatResponseType.OPENAI_COMPLETION) - assert resp.body["choices"][0]["message"]["content"] == "echo" - - -# --------------------------------------------------------------------------- -# Response-side component -# --------------------------------------------------------------------------- - - -class TestResponseProcessor: - async def test_process(self): - - resp = ChatResponse.openai_completion( - ChatCompletion( - id="test", object="chat.completion", created=0, model="m", - choices=[Choice(index=0, message=ChatCompletionMessage(role="assistant", content="x"), finish_reason="stop")], - ) - ) - proc = NoopResponseProcessor() - result = await proc.process(make_ctx(), resp) - assert result is resp diff --git a/tests/test_init_all_exports.py b/tests/test_init_all_exports.py deleted file mode 100644 index 3f5c5c26c..000000000 --- a/tests/test_init_all_exports.py +++ /dev/null @@ -1,23 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Every name in switchyard.__all__ must be accessible from the top-level package. - -These tests fail when symbols are listed in __all__ but are not handled by the -module-level __getattr__ lazy-loader, causing AttributeError / ImportError. -""" - -from __future__ import annotations - -import switchyard - - -def test_all_symbols_accessible() -> None: - """All __all__ exports must be gettable without AttributeError.""" - missing = [] - for name in switchyard.__all__: - try: - getattr(switchyard, name) - except AttributeError: - missing.append(name) - assert missing == [], f"Symbols in __all__ not importable from switchyard: {missing}" diff --git a/tests/test_launchers.py b/tests/test_launchers.py index fa4ec9e2d..b0fc60bb7 100644 --- a/tests/test_launchers.py +++ b/tests/test_launchers.py @@ -22,8 +22,8 @@ def _subparsers(parser: argparse.ArgumentParser) -> dict[str, argparse.ArgumentP return action.choices # type: ignore[return-value] -def test_cli_exposes_only_serve_and_launch() -> None: - assert set(_subparsers(_build_parser())) == {"serve", "launch"} +def test_cli_exposes_only_launch() -> None: + assert set(_subparsers(_build_parser())) == {"launch"} @pytest.mark.parametrize("agent", ["claude", "codex", "openclaw"]) diff --git a/tests/test_live_stats_footer.py b/tests/test_live_stats_footer.py index 4a31b057a..d28e74377 100644 --- a/tests/test_live_stats_footer.py +++ b/tests/test_live_stats_footer.py @@ -9,8 +9,9 @@ from __future__ import annotations +from collections.abc import Mapping + from switchyard.cli.launchers.live_stats_footer import FOOTER_ROWS, LiveStatsFooter -from switchyard.lib.stats_accumulator import StatsAccumulator def _strip_ansi(text: str) -> str: @@ -37,38 +38,60 @@ def indicator(self) -> tuple[str, int]: return ("●", 1) -async def _stats_with_model_call( +class _Stats: + def __init__(self, snapshot: Mapping[str, object] | None = None) -> None: + self.snapshot = snapshot or { + "total_requests": 0, + "total_errors": 0, + "total_tokens": {}, + "models": {}, + } + + def snapshot_sync(self) -> Mapping[str, object]: + return self.snapshot + + +def _stats_with_model_call( model: str, *, prompt: int = 1234, completion: int = 567, cached: int = 200, -) -> StatsAccumulator: - acc = StatsAccumulator() - await acc.record_success(model=model) - await acc.record_usage( - model=model, - prompt_tokens=prompt, - completion_tokens=completion, - cached_tokens=cached, - ) - return acc - - -def _footer(stats: StatsAccumulator, *, model: str = "nvidia/some/default") -> LiveStatsFooter: +) -> _Stats: + return _Stats({ + "total_requests": 1, + "total_errors": 0, + "total_tokens": { + "prompt": prompt, + "completion": completion, + "cached": cached, + }, + "models": { + model: { + "calls": 1, + "errors": 0, + "prompt_tokens": prompt, + "completion_tokens": completion, + "cached_tokens": cached, + } + }, + }) + + +def _footer(stats: _Stats, *, model: str = "nvidia/some/default") -> LiveStatsFooter: return LiveStatsFooter(stats, model=model, health=_StubHealth()) # type: ignore[arg-type] -async def test_footer_height_at_zero_traffic() -> None: +def test_footer_height_at_zero_traffic() -> None: """Before any traffic: aggregate + 1 fallback row = 2 rows.""" - footer = _footer(StatsAccumulator()) + footer = _footer(_Stats()) assert footer.height == FOOTER_ROWS == 2 rows = footer.render(cols=80) assert len(rows) == 2 -async def test_aggregate_row_shows_totals_without_model_name() -> None: - stats = await _stats_with_model_call("vendor/some-model") +def test_aggregate_row_shows_totals_without_model_name() -> None: + stats = _stats_with_model_call("vendor/some-model") rows = _footer(stats).render(cols=80) agg = _strip_ansi(rows[0][0]) assert "switchyard" in agg @@ -79,8 +102,8 @@ async def test_aggregate_row_shows_totals_without_model_name() -> None: assert "some-model" not in agg -async def test_active_row_shows_model_with_recent_traffic() -> None: - stats = await _stats_with_model_call("vendor/winner-model") +def test_active_row_shows_model_with_recent_traffic() -> None: + stats = _stats_with_model_call("vendor/winner-model") rows = _footer(stats).render(cols=80) active = _strip_ansi(rows[1][0]) assert "winner-model" in active @@ -90,31 +113,40 @@ async def test_active_row_shows_model_with_recent_traffic() -> None: assert "200 cached" in active -async def test_active_row_falls_back_to_default_when_no_traffic() -> None: +def test_active_row_falls_back_to_default_when_no_traffic() -> None: """Before any backend call lands, the row labels with the launch default.""" - rows = _footer(StatsAccumulator(), model="vendor/launch-default").render(cols=80) + rows = _footer(_Stats(), model="vendor/launch-default").render(cols=80) active = _strip_ansi(rows[1][0]) assert "launch-default" in active assert "0 req" in active -async def test_new_tier_adds_a_row_on_next_render() -> None: +def test_new_tier_adds_a_row_on_next_render() -> None: """A new model in traffic adds a row; height grows from 2 to 3.""" - stats = StatsAccumulator() - await stats.record_success(model="vendor/first") - await stats.record_usage( - model="vendor/first", prompt_tokens=10, completion_tokens=20, - ) + stats = _stats_with_model_call("vendor/first", prompt=10, completion=20, cached=0) footer = _footer(stats) rows = footer.render(cols=80) assert len(rows) == 2 assert footer.height == 2 assert "first" in _strip_ansi(rows[1][0]) - await stats.record_success(model="vendor/second") - await stats.record_usage( - model="vendor/second", prompt_tokens=30, completion_tokens=40, - ) + stats.snapshot = { + "total_requests": 2, + "total_errors": 0, + "total_tokens": {"prompt": 40, "completion": 60, "cached": 0}, + "models": { + "vendor/first": { + "calls": 1, + "prompt_tokens": 10, + "completion_tokens": 20, + }, + "vendor/second": { + "calls": 1, + "prompt_tokens": 30, + "completion_tokens": 40, + }, + }, + } rows = footer.render(cols=80) assert len(rows) == 3 assert footer.height == 3 diff --git a/tests/test_llm_client.py b/tests/test_llm_client.py deleted file mode 100644 index 8369b1243..000000000 --- a/tests/test_llm_client.py +++ /dev/null @@ -1,205 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Unit tests for :class:`OpenAILLMClient`.""" - -from unittest.mock import AsyncMock, MagicMock - -import httpx -import openai -import pytest -import respx -from openai import AsyncOpenAI - -from switchyard.lib.llm_client import OpenAILLMClient, RawSSEFrameStream - - -def test_constructs_only_the_async_client() -> None: - """Only the async client is built. - - Backends call ``acompletion`` exclusively, so a sync ``OpenAI`` client - would only allocate a second, never-used httpx connection pool (1000 - connections by default) per instance. - """ - client = OpenAILLMClient(api_key="test-key") - assert isinstance(client.async_client, AsyncOpenAI) - assert not hasattr(client, "client") - - -def test_max_retries_reaches_the_sdk_client() -> None: - """``max_retries`` is forwarded to the underlying SDK client.""" - client = OpenAILLMClient(api_key="test-key", max_retries=0) - assert client.async_client.max_retries == 0 - - -class TestAcompletionApiKeyOverride: - """``acompletion`` overrides the construction-time key only for a real key. - - A blank or absent per-call ``api_key`` must fall back to the - construction-time key (the configured endpoint key) instead of overriding - it with nothing, which would unauthenticate the upstream call. - """ - - @staticmethod - def _client_with_spied_options() -> tuple[OpenAILLMClient, MagicMock]: - client = OpenAILLMClient(api_key="endpoint-key") - client.async_client = MagicMock() - client.async_client.chat.completions.create = AsyncMock(return_value="base") - client.async_client.responses.create = AsyncMock(return_value="responses-base") - overridden = MagicMock() - overridden.chat.completions.create = AsyncMock(return_value="overridden") - overridden.responses.create = AsyncMock(return_value="responses-overridden") - client.async_client.with_options.return_value = overridden - # Non-streaming ``aresponses`` fetches the raw HTTP response and - # returns its exact JSON body; wire raw spies per client. - for target, label in ((client.async_client, "base"), (overridden, "overridden")): - raw = MagicMock() - raw.http_response.json.return_value = {"src": f"responses-{label}"} - target.responses.with_raw_response.create = AsyncMock(return_value=raw) - return client, client.async_client - - async def test_real_caller_key_uses_with_options(self) -> None: - client, async_client = self._client_with_spied_options() - result = await client.acompletion(api_key="caller-key", model="m") - async_client.with_options.assert_called_once_with(api_key="caller-key") - assert result == "overridden" - - async def test_none_key_falls_back_to_construction_key(self) -> None: - client, async_client = self._client_with_spied_options() - result = await client.acompletion(api_key=None, model="m") - async_client.with_options.assert_not_called() - assert result == "base" - - async def test_blank_key_falls_back_to_construction_key(self) -> None: - client, async_client = self._client_with_spied_options() - result = await client.acompletion(api_key=" ", model="m") - async_client.with_options.assert_not_called() - assert result == "base" - - async def test_responses_real_caller_key_uses_with_options(self) -> None: - client, async_client = self._client_with_spied_options() - result = await client.aresponses(api_key="caller-key", model="m", input="hi") - async_client.with_options.assert_called_once_with(api_key="caller-key") - assert result == {"src": "responses-overridden"} - - async def test_responses_missing_key_falls_back_to_construction_key(self) -> None: - client, async_client = self._client_with_spied_options() - result = await client.aresponses(api_key=None, model="m", input="hi") - async_client.with_options.assert_not_called() - assert result == {"src": "responses-base"} - - async def test_responses_blank_key_falls_back_to_construction_key(self) -> None: - client, async_client = self._client_with_spied_options() - result = await client.aresponses(api_key=" ", model="m", input="hi") - async_client.with_options.assert_not_called() - assert result == {"src": "responses-base"} - - async def test_responses_streaming_uses_raw_streaming_path(self) -> None: - """Streaming fetches the raw SSE response (verbatim frames) and enters - the SDK context manager eagerly so error statuses raise at call time.""" - client, async_client = self._client_with_spied_options() - - async def _lines(): - yield "data: {}" - yield "" - - api_response = MagicMock() - api_response.http_response.aiter_lines = _lines - cm = MagicMock() - cm.__aenter__ = AsyncMock(return_value=api_response) - cm.__aexit__ = AsyncMock(return_value=False) - async_client.responses.with_streaming_response.create = MagicMock(return_value=cm) - - result = await client.aresponses(api_key=None, model="m", input="hi", stream=True) - - assert isinstance(result, RawSSEFrameStream) - cm.__aenter__.assert_awaited_once() - async_client.responses.create.assert_not_called() - async_client.responses.with_raw_response.create.assert_not_called() - await result.aclose() - cm.__aexit__.assert_awaited_once() - - -@respx.mock -async def test_aresponses_returns_exact_upstream_json() -> None: - """Non-streaming Responses calls return the upstream body as-is. - - Provider-specific extras and explicit-null fields must survive — the SDK - typed-model round-trip would normalize the former and ``exclude_none`` - serialization would drop the latter. - """ - upstream = { - "id": "resp_1", - "object": "response", - "created_at": 1719890000, - "model": "gpt-5", - "status": "completed", - "output": [], - "store": False, - "temperature": 1.0, - "top_p": 0.9, - "previous_response_id": None, - "provider_meta": {"azure_region": "eastus"}, - } - respx.post("http://upstream.test/v1/responses").mock( - return_value=httpx.Response(200, json=upstream) - ) - - client = OpenAILLMClient(api_key="k", base_url="http://upstream.test/v1") - result = await client.aresponses(model="gpt-5", input="hi") - - assert result == upstream - - -_SSE_FRAMES = [ - ( - 'event: response.created\n' - 'data: {"type":"response.created","response":{"id":"resp_1","store":false,' - '"temperature":1.0,"reasoning":{"effort":null},"provider_meta":{"az":"eastus"}}}\n\n' - ), - ": keep-alive\n\n", - ( - 'event: response.output_text.delta\n' - 'data: {"type":"response.output_text.delta","delta":"hi","vendor_extra":123}\n\n' - ), - ( - 'event: response.completed\n' - 'data: {"type":"response.completed","response":{"id":"resp_1","usage":' - '{"input_tokens":3,"output_tokens":2}}}\n\n' - ), -] - - -@respx.mock -async def test_aresponses_streaming_yields_verbatim_sse_frames() -> None: - """Streaming Responses calls yield the upstream SSE frames byte-for-byte - : unknown provider fields, explicit nulls, comment keep-alives, - and event names all survive because no typed-model parse happens. - """ - respx.post("http://upstream.test/v1/responses").mock( - return_value=httpx.Response( - 200, - headers={"content-type": "text/event-stream"}, - content="".join(_SSE_FRAMES).encode(), - ) - ) - - client = OpenAILLMClient(api_key="k", base_url="http://upstream.test/v1") - stream = await client.aresponses(model="gpt-5", input="hi", stream=True) - - frames = [frame async for frame in stream] - assert frames == _SSE_FRAMES - - -@respx.mock -async def test_aresponses_streaming_error_status_raises_at_call_time() -> None: - """A non-2xx on a streaming Responses call raises ``APIStatusError`` from - ``aresponses`` itself — before any frame is consumed — so the backend's - retry/failover contract is unchanged by the raw-frame path.""" - respx.post("http://upstream.test/v1/responses").mock( - return_value=httpx.Response(500, json={"error": {"message": "boom"}}) - ) - - client = OpenAILLMClient(api_key="k", base_url="http://upstream.test/v1") - with pytest.raises(openai.APIStatusError): - await client.aresponses(model="gpt-5", input="hi", stream=True) diff --git a/tests/test_metrics_endpoint.py b/tests/test_metrics_endpoint.py deleted file mode 100644 index e3dbd7678..000000000 --- a/tests/test_metrics_endpoint.py +++ /dev/null @@ -1,155 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""End-to-end FastAPI test that ``StatsEndpoint`` serves both -``/v1/stats`` (existing JSON) and ``/metrics`` (Prometheus exposition) -off the same shared :class:`StatsAccumulator`. - -Pins the contract the ticket calls out: existing ``/v1/stats`` behavior -stays intact, ``/metrics`` returns Prometheus text-format with the core -metric names. -""" - -from __future__ import annotations - -from collections.abc import AsyncIterator - -import httpx -import pytest -from fastapi import FastAPI -from prometheus_client.parser import text_string_to_metric_families - -from switchyard.lib.endpoints.stats_endpoint import PROMETHEUS_CONTENT_TYPE, StatsEndpoint -from switchyard.lib.stats_accumulator import StatsAccumulator - - -@pytest.fixture -def stats() -> StatsAccumulator: - return StatsAccumulator() - - -@pytest.fixture -async def client(stats: StatsAccumulator) -> AsyncIterator[httpx.AsyncClient]: - app = FastAPI() - StatsEndpoint(stats).register(app) - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=app), base_url="http://test" - ) as c: - yield c - - -async def test_v1_stats_returns_existing_json_shape( - client: httpx.AsyncClient, stats: StatsAccumulator -) -> None: - await stats.record_success(model="m", backend_latency_ms=10.0) - await stats.record_usage( - model="m", prompt_tokens=5, completion_tokens=3, total_latency_ms=20.0 - ) - - resp = await client.get("/v1/stats") - assert resp.status_code == 200 - body = resp.json() - # Existing schema is unchanged — keep the contract surface visible. - assert body["total_requests"] == 1 - assert body["models"]["m"]["calls"] == 1 - assert body["models"]["m"]["prompt_tokens"] == 5 - assert body["models"]["m"]["max_observed_context_tokens"] == 8 - assert "cost_estimate" in body - - -async def test_metrics_returns_prometheus_exposition( - client: httpx.AsyncClient, stats: StatsAccumulator -) -> None: - await stats.record_success(model="strong/m", backend_latency_ms=42.5, tier="strong") - await stats.record_usage( - model="strong/m", - prompt_tokens=120, - completion_tokens=30, - total_latency_ms=88.0, - routing_overhead_ms=8.0, - tier="strong", - ) - - resp = await client.get("/metrics") - assert resp.status_code == 200 - assert resp.headers["content-type"] == PROMETHEUS_CONTENT_TYPE - - body = resp.text - # Core metric headers the ticket calls out. - for line in ( - "# TYPE switchyard_requests_total counter", - "# TYPE switchyard_errors_total counter", - "# TYPE switchyard_model_call_latency_ms summary", - "# TYPE switchyard_total_latency_ms summary", - "# TYPE switchyard_routing_overhead_ms summary", - ): - assert line in body, f"missing exposition line: {line}" - - # Selected model/tier counter sample lands with the expected label set. - assert 'switchyard_requests_total{model="strong/m",tier="strong"} 1' in body - assert ( - 'switchyard_model_call_latency_ms_count{model="strong/m",tier="strong"} 1' - in body - ) - - -async def test_metrics_output_round_trips_through_official_prometheus_parser( - client: httpx.AsyncClient, stats: StatsAccumulator -) -> None: - """Spec compliance gate: ``prometheus_client.parser`` is the reference - parser used by every real scraper. Anything it accepts, Prometheus will.""" - await stats.record_success(model="openai/gpt-5.2", backend_latency_ms=42.5, tier="strong") - await stats.record_error(model="anth/claude", tier="weak") - await stats.record_success(model="anth/claude", backend_latency_ms=5.0, tier="weak") - await stats.record_usage( - model="openai/gpt-5.2", - prompt_tokens=120, - completion_tokens=30, - total_latency_ms=88.0, - routing_overhead_ms=8.0, - tier="strong", - ) - await stats.record_usage( - model="anth/claude", - prompt_tokens=40, - completion_tokens=5, - total_latency_ms=15.0, - routing_overhead_ms=3.0, - tier="weak", - ) - - resp = await client.get("/metrics") - assert resp.status_code == 200 - - families = {f.name: f for f in text_string_to_metric_families(resp.text)} - - # prometheus-client strips the ``_total`` suffix from counter family names - # but preserves it on individual samples — assert against the family form. - expected = { - "switchyard_total_requests": "gauge", - "switchyard_total_errors": "gauge", - "switchyard_requests": "counter", - "switchyard_errors": "counter", - "switchyard_prompt_tokens": "counter", - "switchyard_completion_tokens": "counter", - "switchyard_cached_tokens": "counter", - "switchyard_model_call_latency_ms": "summary", - "switchyard_total_latency_ms": "summary", - "switchyard_routing_overhead_ms": "summary", - } - for name, kind in expected.items(): - assert name in families, f"family {name} missing from parsed output" - assert families[name].type == kind, f"family {name} parsed as {families[name].type}" - - # Counter values survive the parse round-trip with the right labels. - req_samples = { - (s.labels["model"], s.labels["tier"]): s.value - for s in families["switchyard_requests"].samples - } - assert req_samples[("openai/gpt-5.2", "strong")] == 1 - assert req_samples[("anth/claude", "weak")] == 1 - - # Summary sum aggregates across all observations (8 + 3 = 11 ms overhead). - overhead = families["switchyard_routing_overhead_ms"] - overhead_sum = next(s.value for s in overhead.samples if s.name.endswith("_sum")) - assert overhead_sum == 11.0 diff --git a/tests/test_no_stale_module_paths.py b/tests/test_no_stale_module_paths.py index f67cd9256..a102acd79 100644 --- a/tests/test_no_stale_module_paths.py +++ b/tests/test_no_stale_module_paths.py @@ -1,31 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Regression guard for stale Python module-path references in source code. - -Runtime tests don't catch this class of bug — broken docstring paths -don't fail at import or at request time, only when someone follows the -reference (Sphinx build, IDE go-to-definition, copying the path into an -import). Hence this dedicated source-scan test. - -Patterns checked: - -* ``switchyard.core.`` — intermediate path that was flattened into - ``switchyard.lib.`` / ``switchyard.cli.`` during the open-source - cleanup. No file in the current tree should reference it. -* ``switchyard.foundation`` — pre-rename sub-package. Renamed to - ``switchyard.lib`` in the same cleanup. -* ``nemo_switchyard.`` — original Python package name. Renamed to - ``switchyard``. ``test_cli_stale_names.py`` covers user-facing CLI - strings (``nemo-switchyard`` with a dash); this guards Python-import - paths (``nemo_switchyard`` with an underscore). -* ``SwitchyardV2`` / ``switchyard_v2`` — V2 vs V1 was a transient - migration concept that has since been collapsed to a single, - unversioned ``Switchyard`` API. Make sure the V2 suffix does not - creep back into module paths or symbols. -* ``switchyard_v2_cli`` / ``switchyard_v2_app`` / ``build_switchyard_v2_app`` - — old module/function names from the V2 era. -""" +"""Prevent deleted Python server and compatibility paths from returning.""" from __future__ import annotations @@ -36,6 +12,11 @@ _STALE_PATH_PATTERNS = ( "switchyard.core.", "switchyard.foundation", + "switchyard.lib.", + "switchyard.server.", + "switchyard_rust.components", + "switchyard_rust.core", + "switchyard_rust.translation", "nemo_switchyard.", "SwitchyardV2", "switchyard_v2", diff --git a/tests/test_outcome_metrics.py b/tests/test_outcome_metrics.py deleted file mode 100644 index 3d8397785..000000000 --- a/tests/test_outcome_metrics.py +++ /dev/null @@ -1,304 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Unit + integration tests for the outcome counters.""" - -from __future__ import annotations - -import pytest -from fastapi.testclient import TestClient -from openai.types.chat import ChatCompletion -from openai.types.chat.chat_completion import Choice -from openai.types.chat.chat_completion_message import ChatCompletionMessage -from openai.types.completion_usage import CompletionUsage - -from switchyard.lib.endpoints import outcome_metrics -from switchyard.lib.endpoints.upstream_error import ( - record_upstream_attempt_failure, - record_upstream_attempt_success, -) -from switchyard.lib.proxy_context import ( - CTX_UPSTREAM_ATTEMPTS_RECORDED, - CTX_UPSTREAM_HTTP_STATUS, - ProxyContext, -) -from switchyard.lib.roles import LLMBackend -from switchyard.lib.switchyard import Switchyard -from switchyard.server.switchyard_app import build_switchyard_app -from switchyard_rust.core import ( - ChatRequest, - ChatRequestType, - ChatResponse, - SwitchyardUpstreamError, -) -from switchyard_rust.translation import TranslationEngine - - -@pytest.fixture(autouse=True) -def _reset_counters(): - outcome_metrics._reset_for_tests() - yield - outcome_metrics._reset_for_tests() - - -# --------------------------------------------------------------------------- -# Classification -# --------------------------------------------------------------------------- - - -class TestClassify: - @pytest.mark.parametrize("code", [200, 201, 204, 299]) - def test_2xx_is_success(self, code: int) -> None: - assert outcome_metrics.classify(code) == "success" - - @pytest.mark.parametrize("code", [429, 500, 504]) - def test_spec_codes_are_retryable_error(self, code: int) -> None: - """Exactly the codes the success criterion lists count as retryable.""" - assert outcome_metrics.classify(code) == "retryable_error" - - @pytest.mark.parametrize("code", [400, 401, 403, 404, 422, 502, 503]) - def test_other_codes_are_other_error(self, code: int) -> None: - """Bad-payload / bad-key / non-spec 5xx fall outside the criterion.""" - assert outcome_metrics.classify(code) == "other_error" - - -# --------------------------------------------------------------------------- -# Code label (the per-status dimension for the distribution dashboard) -# --------------------------------------------------------------------------- - - -class TestCodeLabel: - def test_none_is_the_no_status_sentinel(self) -> None: - """Non-HTTP failures have no status line → the ``none`` sentinel.""" - assert outcome_metrics.code_label(None) == outcome_metrics.NO_STATUS_CODE - assert outcome_metrics.code_label(None) == "none" - - @pytest.mark.parametrize("code", sorted(outcome_metrics.KNOWN_STATUS_CODES)) - def test_known_codes_emitted_verbatim(self, code: int) -> None: - assert outcome_metrics.code_label(code) == str(code) - - @pytest.mark.parametrize( - ("code", "expected"), - [(418, "4xx"), (451, "4xx"), (599, "5xx"), (100, "1xx"), (302, "3xx")], - ) - def test_unknown_codes_clamp_to_class(self, code: int, expected: str) -> None: - """An oddball upstream code collapses to its class, bounding cardinality.""" - assert outcome_metrics.code_label(code) == expected - - @pytest.mark.parametrize("code", [0, 99, 600, 700]) - def test_out_of_range_codes_clamp_to_other(self, code: int) -> None: - assert outcome_metrics.code_label(code) == "other" - - -# --------------------------------------------------------------------------- -# Render shape -# --------------------------------------------------------------------------- - - -class TestRender: - def test_render_initial_state_is_all_zero(self) -> None: - out = "\n".join(outcome_metrics.render_lines()) - assert 'switchyard_client_responses_total{outcome="success"} 0' in out - assert 'switchyard_client_responses_total{outcome="retryable_error"} 0' in out - assert 'switchyard_client_responses_total{outcome="other_error"} 0' in out - # Upstream attempts carry a code label; the canonical codes are - # seeded at 0 so their series exist before the first matching attempt. - assert 'switchyard_upstream_attempts_total{outcome="success",code="200"} 0' in out - assert ( - 'switchyard_upstream_attempts_total{outcome="retryable_error",code="429"} 0' - in out - ) - assert ( - 'switchyard_upstream_attempts_total{outcome="retryable_error",code="none"} 0' - in out - ) - assert "switchyard_router_retry_recovered_total 0" in out - - def test_render_includes_help_and_type_lines(self) -> None: - """Prometheus exposition needs HELP+TYPE before each metric family.""" - out = "\n".join(outcome_metrics.render_lines()) - for metric in ( - "switchyard_client_responses_total", - "switchyard_upstream_attempts_total", - "switchyard_router_retry_recovered_total", - ): - assert f"# HELP {metric}" in out - assert f"# TYPE {metric}" in out - - def test_render_reflects_recorded_state(self) -> None: - outcome_metrics.record_client_response(200) - outcome_metrics.record_client_response(429) - outcome_metrics.record_client_response(401) - outcome_metrics.record_upstream_attempt(500) - outcome_metrics.record_upstream_attempt(None) - outcome_metrics.record_retry_recovered() - - out = "\n".join(outcome_metrics.render_lines()) - assert 'switchyard_client_responses_total{outcome="success"} 1' in out - assert 'switchyard_client_responses_total{outcome="retryable_error"} 1' in out - assert 'switchyard_client_responses_total{outcome="other_error"} 1' in out - # The two retryable attempts split across their codes — the whole - # point of the new label — rather than collapsing into one bucket. - assert ( - 'switchyard_upstream_attempts_total{outcome="retryable_error",code="500"} 1' - in out - ) - assert ( - 'switchyard_upstream_attempts_total{outcome="retryable_error",code="none"} 1' - in out - ) - assert "switchyard_router_retry_recovered_total 1" in out - - def test_distinct_codes_get_distinct_series(self) -> None: - """429 / 500 / 504 must be separately countable, not merged.""" - for _ in range(3): - outcome_metrics.record_upstream_attempt(429) - outcome_metrics.record_upstream_attempt(500) - outcome_metrics.record_upstream_attempt(504) - # An unknown 4xx clamps to its class rather than spawning a new series. - outcome_metrics.record_upstream_attempt(418) - - out = "\n".join(outcome_metrics.render_lines()) - assert ( - 'switchyard_upstream_attempts_total{outcome="retryable_error",code="429"} 3' - in out - ) - assert ( - 'switchyard_upstream_attempts_total{outcome="retryable_error",code="500"} 1' - in out - ) - assert ( - 'switchyard_upstream_attempts_total{outcome="retryable_error",code="504"} 1' - in out - ) - assert ( - 'switchyard_upstream_attempts_total{outcome="other_error",code="4xx"} 1' - in out - ) - - -# --------------------------------------------------------------------------- -# Endpoint-layer fallback — wires the upstream-attempt counter for backends -# (Rust native / passthrough / multi) that issue one attempt and can't reach -# the Python-only outcome_metrics themselves. -# --------------------------------------------------------------------------- - - -def _upstream_count(out: str, outcome: str, code: str) -> str: - return f'switchyard_upstream_attempts_total{{outcome="{outcome}",code="{code}"}}' - - -class TestEndpointUpstreamAttemptFallback: - def test_success_records_one_200(self) -> None: - record_upstream_attempt_success(ProxyContext()) - out = "\n".join(outcome_metrics.render_lines()) - assert f"{_upstream_count(out, 'success', '200')} 1" in out - - def test_rust_upstream_http_error_records_its_status(self) -> None: - """A Rust backend's typed ``SwitchyardUpstreamError.status_code`` is used.""" - exc = SwitchyardUpstreamError("boom") - exc.status_code = 500 - record_upstream_attempt_failure(ProxyContext(), exc) - out = "\n".join(outcome_metrics.render_lines()) - assert f"{_upstream_count(out, 'retryable_error', '500')} 1" in out - - def test_python_backend_ctx_status_takes_priority(self) -> None: - """A Python backend's stashed ctx status is recorded even without a typed exc.""" - ctx = ProxyContext() - ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] = 401 - record_upstream_attempt_failure(ctx, RuntimeError("opaque")) - out = "\n".join(outcome_metrics.render_lines()) - assert f"{_upstream_count(out, 'other_error', '401')} 1" in out - - def test_status_less_upstream_error_is_retryable_none(self) -> None: - """An upstream failure with no HTTP status (network) maps to code=none.""" - record_upstream_attempt_failure(ProxyContext(), SwitchyardUpstreamError("conn reset")) - out = "\n".join(outcome_metrics.render_lines()) - assert f"{_upstream_count(out, 'retryable_error', 'none')} 1" in out - - def test_internal_error_is_not_an_upstream_attempt(self) -> None: - """A non-upstream chain failure (e.g. translation/processor) records nothing.""" - record_upstream_attempt_failure(ProxyContext(), ValueError("internal bug")) - out = "\n".join(outcome_metrics.render_lines()) - # Every seeded series stays at 0 — no attempt was attributed. - assert f"{_upstream_count(out, 'success', '200')} 0" in out - assert f"{_upstream_count(out, 'retryable_error', 'none')} 0" in out - - def test_dedup_flag_suppresses_fallback(self) -> None: - """A backend that records its own attempts opts the endpoint out.""" - ctx = ProxyContext() - ctx.metadata[CTX_UPSTREAM_ATTEMPTS_RECORDED] = True - record_upstream_attempt_success(ctx) - record_upstream_attempt_failure(ctx, SwitchyardUpstreamError("boom")) - out = "\n".join(outcome_metrics.render_lines()) - assert f"{_upstream_count(out, 'success', '200')} 0" in out - assert f"{_upstream_count(out, 'retryable_error', 'none')} 0" in out - - -# --------------------------------------------------------------------------- -# Client-response middleware — only the /v1/* LLM routes feed the client -# outcome counter; the operational routes (/metrics, /health, /v1/models) do -# not, so a scraper polling them cannot inflate the success count. -# --------------------------------------------------------------------------- - - -class _CannedBackend(LLMBackend): - """Passthrough-style backend: returns one canned OpenAI completion per call.""" - - def supported_request_types(self) -> list[ChatRequestType]: - return [ - ChatRequestType.OPENAI_CHAT, - ChatRequestType.OPENAI_RESPONSES, - ChatRequestType.ANTHROPIC, - ] - - async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: - return ChatResponse.openai_completion( - ChatCompletion( - id="chatcmpl-test", - object="chat.completion", - created=1700000000, - model="mock-model", - choices=[ - Choice( - index=0, - message=ChatCompletionMessage(role="assistant", content="hi"), - finish_reason="stop", - ) - ], - usage=CompletionUsage(prompt_tokens=5, completion_tokens=3, total_tokens=8), - ) - ) - - -def _canned_app() -> object: - return build_switchyard_app( - Switchyard(backend=_CannedBackend(), translator=TranslationEngine()) - ) - - -class TestClientResponseMiddleware: - def test_successful_chat_completion_counts_as_success(self) -> None: - """A served /v1/chat/completions success increments the client outcome counter.""" - with TestClient(_canned_app(), raise_server_exceptions=False) as client: - response = client.post( - "/v1/chat/completions", - json={"model": "mock-model", "messages": [{"role": "user", "content": "hi"}]}, - ) - assert response.status_code == 200 - - metrics = "\n".join(outcome_metrics.render_lines()) - assert 'switchyard_client_responses_total{outcome="success"} 1' in metrics - assert 'switchyard_client_responses_total{outcome="retryable_error"} 0' in metrics - - def test_operational_routes_are_not_counted_as_client_responses(self) -> None: - """Polling /metrics, /health, /v1/models must not feed the client counter — - otherwise a scraper on a fixed cadence would inflate the success count.""" - with TestClient(_canned_app(), raise_server_exceptions=False) as client: - for _ in range(5): - client.get("/metrics") - client.get("/health") - client.get("/v1/models") - - metrics = "\n".join(outcome_metrics.render_lines()) - assert 'switchyard_client_responses_total{outcome="success"} 0' in metrics diff --git a/tests/test_prometheus_emitter.py b/tests/test_prometheus_emitter.py deleted file mode 100644 index 698b5aca8..000000000 --- a/tests/test_prometheus_emitter.py +++ /dev/null @@ -1,56 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Unit tests for the module-level Prometheus emitter table.""" - -from __future__ import annotations - -from collections.abc import Iterator - -import pytest - -from switchyard.lib.endpoints import prometheus_emitter - - -@pytest.fixture(autouse=True) -def _clean_table() -> Iterator[None]: - """Isolate every test: emitters registered here must not leak.""" - prometheus_emitter._clear_for_tests() - yield - prometheus_emitter._clear_for_tests() - - -class TestTableLifecycle: - def test_render_empty_returns_empty_string(self) -> None: - """No emitters → empty string so callers can concat unconditionally.""" - assert prometheus_emitter.render() == "" - - def test_register_then_render_composes_lines(self) -> None: - prometheus_emitter.register(lambda: ["foo 1", "bar 2"]) - rendered = prometheus_emitter.render() - assert "foo 1" in rendered - assert "bar 2" in rendered - assert rendered.endswith("\n") - - def test_register_is_idempotent(self) -> None: - """Re-registering the same callable must not double-count.""" - emitter = lambda: ["x 1"] # noqa: E731 - prometheus_emitter.register(emitter) - prometheus_emitter.register(emitter) - assert prometheus_emitter.render().count("x 1") == 1 - - def test_unregister_removes_emitter(self) -> None: - emitter = lambda: ["x 1"] # noqa: E731 - prometheus_emitter.register(emitter) - prometheus_emitter.unregister(emitter) - assert prometheus_emitter.render() == "" - - def test_unregister_unknown_is_noop(self) -> None: - """Shutdown paths should not throw if registration was skipped.""" - prometheus_emitter.unregister(lambda: ["x"]) - - def test_multiple_emitters_compose_in_registration_order(self) -> None: - prometheus_emitter.register(lambda: ["a 1"]) - prometheus_emitter.register(lambda: ["b 2"]) - rendered = prometheus_emitter.render() - assert rendered.index("a 1") < rendered.index("b 2") diff --git a/tests/test_prometheus_exposition.py b/tests/test_prometheus_exposition.py deleted file mode 100644 index 4431a41ef..000000000 --- a/tests/test_prometheus_exposition.py +++ /dev/null @@ -1,176 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Unit tests for :mod:`switchyard.lib.prometheus_exposition`. - -Covers metric names, label sets, summary quantile rendering, and the -empty-accumulator edge case. Exposition is parsed line-by-line back into -samples for assertion rather than string-matched, so reorder-safe. -""" - -from __future__ import annotations - -import re - -from switchyard.lib.prometheus_exposition import render_prometheus -from switchyard.lib.stats_accumulator import StatsAccumulator - -_SAMPLE_RE = re.compile(r"^(?P[a-zA-Z_:][a-zA-Z0-9_:]*)(?P\{[^}]*\})? (?P.+)$") - - -def _parse(exposition: str) -> tuple[dict[str, str], dict[str, str], dict[tuple[str, frozenset[tuple[str, str]]], str]]: - """Return ``(help_map, type_map, samples)`` from exposition text. - - samples keys are ``(metric_name, frozenset(labels.items()))``. - """ - help_map: dict[str, str] = {} - type_map: dict[str, str] = {} - samples: dict[tuple[str, frozenset[tuple[str, str]]], str] = {} - for raw in exposition.splitlines(): - line = raw.strip() - if not line: - continue - if line.startswith("# HELP "): - name, _, text = line[len("# HELP ") :].partition(" ") - help_map[name] = text - continue - if line.startswith("# TYPE "): - name, _, kind = line[len("# TYPE ") :].partition(" ") - type_map[name] = kind - continue - match = _SAMPLE_RE.match(line) - assert match, f"unparseable exposition line: {line!r}" - name = match.group("name") - labels_blob = match.group("labels") or "" - labels: dict[str, str] = {} - if labels_blob: - inner = labels_blob[1:-1] - # Cheap parser — values never contain "," or "\"" in these tests. - for part in inner.split(","): - k, _, v = part.partition("=") - labels[k] = v.strip('"') - samples[(name, frozenset(labels.items()))] = match.group("value") - return help_map, type_map, samples - - -async def test_empty_accumulator_renders_zero_totals(): - snapshot = await StatsAccumulator().snapshot() - text = render_prometheus(snapshot) - - help_map, type_map, samples = _parse(text) - - assert type_map["switchyard_total_requests"] == "gauge" - assert type_map["switchyard_total_errors"] == "gauge" - assert samples[("switchyard_total_requests", frozenset())] == "0" - assert samples[("switchyard_total_errors", frozenset())] == "0" - # Routing overhead summary should still emit a header even with no data. - assert type_map["switchyard_routing_overhead_ms"] == "summary" - assert samples[("switchyard_routing_overhead_ms_count", frozenset())] == "0" - # Exposition must end in a single trailing newline (scraper requirement). - assert text.endswith("\n") - - -async def test_two_tier_snapshot_emits_expected_metric_names_and_labels(): - stats = StatsAccumulator() - await stats.record_success(model="strong/m", backend_latency_ms=42.5, tier="strong") - await stats.record_usage( - model="strong/m", - prompt_tokens=120, - completion_tokens=30, - cached_tokens=10, - total_latency_ms=88.0, - routing_overhead_ms=8.0, - tier="strong", - ) - await stats.record_error(model="weak/m", tier="weak") - await stats.record_success(model="weak/m", backend_latency_ms=5.0, tier="weak") - await stats.record_usage( - model="weak/m", - prompt_tokens=40, - completion_tokens=5, - total_latency_ms=15.0, - routing_overhead_ms=3.0, - tier="weak", - ) - - text = render_prometheus(await stats.snapshot()) - _, type_map, samples = _parse(text) - - # Every metric the ticket lists must be present with the expected type. - expected_types = { - "switchyard_total_requests": "gauge", - "switchyard_total_errors": "gauge", - "switchyard_requests_total": "counter", - "switchyard_errors_total": "counter", - "switchyard_prompt_tokens_total": "counter", - "switchyard_completion_tokens_total": "counter", - "switchyard_cached_tokens_total": "counter", - "switchyard_model_call_latency_ms": "summary", - "switchyard_total_latency_ms": "summary", - "switchyard_routing_overhead_ms": "summary", - } - for metric, kind in expected_types.items(): - assert type_map.get(metric) == kind, f"{metric} missing or wrong type" - - strong_labels = frozenset({("model", "strong/m"), ("tier", "strong")}) - weak_labels = frozenset({("model", "weak/m"), ("tier", "weak")}) - - # weak/m had one record_error then one record_success → calls=1, errors=1. - assert samples[("switchyard_requests_total", strong_labels)] == "1" - assert samples[("switchyard_requests_total", weak_labels)] == "1" - assert samples[("switchyard_errors_total", weak_labels)] == "1" - assert samples[("switchyard_prompt_tokens_total", strong_labels)] == "120" - assert samples[("switchyard_completion_tokens_total", weak_labels)] == "5" - assert samples[("switchyard_cached_tokens_total", strong_labels)] == "10" - - # Summaries must emit quantile=0.5 and quantile=0.99 plus _sum / _count. - strong_p50_key = ("switchyard_model_call_latency_ms", strong_labels | {("quantile", "0.5")}) - strong_p99_key = ("switchyard_model_call_latency_ms", strong_labels | {("quantile", "0.99")}) - assert strong_p50_key in samples and strong_p99_key in samples - assert samples[("switchyard_model_call_latency_ms_count", strong_labels)] == "1" - assert samples[("switchyard_total_latency_ms_count", weak_labels)] == "1" - - # Global routing-overhead summary has no labels. - assert samples[("switchyard_routing_overhead_ms_count", frozenset())] == "2" - - -def test_label_value_escapes_backslash_quote_and_newline(): - snapshot = { - "total_requests": 0, - "total_errors": 0, - "models": { - 'weird"name\\with\nnewline': { - "calls": 1, - "errors": 0, - "tier": None, - "prompt_tokens": 0, - "completion_tokens": 0, - "cached_tokens": 0, - "cache_creation_tokens": 0, - "reasoning_tokens": 0, - "model_call_latency": {"count": 0, "total_ms": 0, "p50_ms": 0, "p99_ms": 0}, - "total_latency": {"count": 0, "total_ms": 0, "p50_ms": 0, "p99_ms": 0}, - }, - }, - "routing_overhead": {"count": 0, "total_ms": 0, "p50_ms": 0, "p99_ms": 0}, - } - text = render_prometheus(snapshot) - # Escaped form per Prometheus exposition spec. - assert 'model="weird\\"name\\\\with\\nnewline"' in text - - -def test_build_info_gauge_present() -> None: - from importlib.metadata import version as pkg_version - ver = pkg_version("nemo-switchyard") - text = render_prometheus({"total_requests": 0, "total_errors": 0, "models": {}}) - _, type_map, samples = _parse(text) - assert type_map.get("switchyard_build_info") == "gauge" - key = ("switchyard_build_info", frozenset({("version", ver)})) - assert samples[key] == "1" - - -def test_build_info_gauge_carries_version_label() -> None: - from importlib.metadata import version as pkg_version - expected = pkg_version("nemo-switchyard") - text = render_prometheus({"total_requests": 0, "total_errors": 0, "models": {}}) - assert f'version="{expected}"' in text diff --git a/tests/test_python_server_passthrough.py b/tests/test_python_server_passthrough.py deleted file mode 100644 index 4dc5eabaa..000000000 --- a/tests/test_python_server_passthrough.py +++ /dev/null @@ -1,482 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""End-to-end Python server passthrough tests with a mocked upstream. - -This file is the highest-leverage entry in the mocked-test suite added after -the post-PR-#12 wiring regressions. Where ``test_inference_e2e.py`` swaps in -a fake ``LLMBackend`` subclass — bypassing the entire ``OpenAiPassthroughBackend`` + -openai-SDK + httpx code path — these tests exercise that path end-to-end: - - inbound HTTP request - → Switchyard chain (real RequestProcessors / ResponseProcessors) - → OpenAiPassthroughBackend (Rust HTTP call) - → local OpenAI-compatible upstream stub - → response back through the chain - → outbound HTTP response - -A regression anywhere in that pipeline (renamed module, missing kwarg, wrong -base_url forwarding, stale processor wiring, broken stats accumulator) -fails one of these tests instead of slipping to top-of-tree. - -All tests run offline: ASGITransport intercepts inbound HTTP, the upstream -is a local loopback HTTP server, and no external network is touched. -""" - -from __future__ import annotations - -import json -import threading -from collections.abc import AsyncIterator, Iterator -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from typing import Any - -import httpx -import pytest - -from switchyard.cli.route_bundle import build_route_bundle_table -from switchyard.lib.endpoints import outcome_metrics -from switchyard.server.switchyard_app import build_switchyard_app - -# --------------------------------------------------------------------------- -# Upstream payloads -# --------------------------------------------------------------------------- - - -def _completion_payload(*, content: str = "hello back") -> dict[str, object]: - """An OpenAI Chat Completion JSON body (non-streaming).""" - return { - "id": "chatcmpl-upstream", - "object": "chat.completion", - "created": 1700000000, - "model": "upstream-model", - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": content}, - "finish_reason": "stop", - } - ], - "usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8}, - } - - -def _completion_chunk(*, content: str = "", finish: str | None = None) -> dict[str, object]: - delta: dict[str, object] = {} - if content: - delta["content"] = content - return { - "id": "chatcmpl-upstream-stream", - "object": "chat.completion.chunk", - "created": 1700000000, - "model": "upstream-model", - "choices": [{"index": 0, "delta": delta, "finish_reason": finish}], - } - - -def _sse_stream_body(chunks: list[dict[str, object]]) -> bytes: - """Encode chunks as the upstream SSE wire format the OpenAI SDK reads.""" - out = [] - for c in chunks: - out.append(f"data: {json.dumps(c)}\n\n") - out.append("data: [DONE]\n\n") - return "".join(out).encode("utf-8") - - -class _OpenAICompatStub: - def __init__(self) -> None: - self._server: ThreadingHTTPServer | None = None - self._thread: threading.Thread | None = None - self._lock = threading.Lock() - self._requests: list[dict[str, Any]] = [] - self._responses: list[tuple[int, bytes, str]] = [] - - def __enter__(self) -> _OpenAICompatStub: - owner = self - - class Handler(BaseHTTPRequestHandler): - protocol_version = "HTTP/1.1" - - def do_POST(self) -> None: - length = int(self.headers.get("content-length", "0")) - raw = self.rfile.read(length) - body = json.loads(raw.decode("utf-8")) - with owner._lock: - owner._requests.append({ - "path": self.path, - "authorization": self.headers.get("authorization"), - "body": body, - }) - if owner._responses: - status, content, content_type = owner._responses.pop(0) - else: - status = 500 - content = b'{"error":{"message":"no stub response queued"}}' - content_type = "application/json" - - self.send_response(status) - self.send_header("content-type", content_type) - self.send_header("content-length", str(len(content))) - self.send_header("connection", "close") - self.end_headers() - self.wfile.write(content) - - def log_message(self, _format: str, *args: object) -> None: - return None - - self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) - self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) - self._thread.start() - return self - - def __exit__(self, *args: object) -> None: - if self._server is not None: - self._server.shutdown() - self._server.server_close() - if self._thread is not None: - self._thread.join(timeout=2) - - @property - def base_url(self) -> str: - if self._server is None: - raise RuntimeError("stub server is not running") - host, port = self._server.server_address - return f"http://{host}:{port}/v1" - - @property - def requests(self) -> list[dict[str, Any]]: - with self._lock: - return list(self._requests) - - def respond_json(self, status: int, body: dict[str, object]) -> None: - content = json.dumps(body).encode("utf-8") - with self._lock: - self._responses.append((status, content, "application/json")) - - def respond_sse(self, body: bytes) -> None: - with self._lock: - self._responses.append((200, body, "text/event-stream")) - - -# --------------------------------------------------------------------------- -# Fixtures -# --------------------------------------------------------------------------- - - -@pytest.fixture -def passthrough_upstream() -> Iterator[_OpenAICompatStub]: - with _OpenAICompatStub() as upstream: - yield upstream - - -@pytest.fixture -async def passthrough_client( - passthrough_upstream: _OpenAICompatStub, -) -> AsyncIterator[httpx.AsyncClient]: - """Return a client using the retained passthrough route.""" - table = build_route_bundle_table({ - "defaults": { - "api_key": "test-key-not-used", - "base_url": passthrough_upstream.base_url, - "format": "openai", - }, - "routes": { - "any-model": {"type": "passthrough", "target": "any-model"}, - }, - }) - app = build_switchyard_app(table) - # ``raise_app_exceptions=False`` mirrors what uvicorn does in - # production: an unhandled exception inside a route is mapped to a - # 500 response, not propagated up the call stack. Without this the - # error-path tests would see the openai SDK exception escape the - # transport entirely. - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=app, raise_app_exceptions=False), - base_url="http://test", - ) as client: - yield client - - -# --------------------------------------------------------------------------- -# Passthrough route — OpenAI inbound, OpenAI upstream -# --------------------------------------------------------------------------- - - -class TestPassthroughOpenAI: - """Inbound OpenAI Chat Completions through the passthrough route.""" - - async def test_non_streaming_round_trip( - self, - passthrough_client: httpx.AsyncClient, - passthrough_upstream: _OpenAICompatStub, - ) -> None: - passthrough_upstream.respond_json(200, _completion_payload(content="pong")) - - resp = await passthrough_client.post( - "/v1/chat/completions", - json={ - "model": "any-model", - "messages": [{"role": "user", "content": "ping"}], - }, - ) - - assert resp.status_code == 200, resp.text - body = resp.json() - assert body["choices"][0]["message"]["content"] == "pong" - assert passthrough_upstream.requests, "passthrough never invoked the upstream" - - async def test_streaming_round_trip( - self, - passthrough_client: httpx.AsyncClient, - passthrough_upstream: _OpenAICompatStub, - ) -> None: - chunks = [ - _completion_chunk(content="foo"), - _completion_chunk(content="bar"), - _completion_chunk(finish="stop"), - ] - passthrough_upstream.respond_sse(_sse_stream_body(chunks)) - - resp = await passthrough_client.post( - "/v1/chat/completions", - json={ - "model": "any-model", - "messages": [{"role": "user", "content": "ping"}], - "stream": True, - }, - ) - - assert resp.status_code == 200, resp.text - assert "text/event-stream" in resp.headers["content-type"] - assert passthrough_upstream.requests - - data_lines = [ - line[6:] - for line in resp.text.split("\n") - if line.startswith("data: ") and line != "data: [DONE]" - ] - assert data_lines, "expected at least one outbound data frame" - decoded = [json.loads(line) for line in data_lines] - joined = "".join( - c["choices"][0]["delta"].get("content", "") - for c in decoded - if c.get("choices") - ) - assert joined == "foobar" - - -# --------------------------------------------------------------------------- -# Passthrough route — Anthropic inbound, OpenAI upstream -# --------------------------------------------------------------------------- - - -class TestPassthroughAnthropic: - """Inbound Anthropic Messages, translated to OpenAI for the upstream call.""" - - async def test_non_streaming_translates_both_directions( - self, - passthrough_client: httpx.AsyncClient, - passthrough_upstream: _OpenAICompatStub, - ) -> None: - passthrough_upstream.respond_json(200, _completion_payload(content="hi")) - - resp = await passthrough_client.post( - "/v1/messages", - json={ - "model": "any-model", - "max_tokens": 16, - "messages": [{"role": "user", "content": "ping"}], - }, - ) - - assert resp.status_code == 200, resp.text - body = resp.json() - assert body["type"] == "message" - assert body["role"] == "assistant" - assert body["content"][0]["type"] == "text" - assert body["content"][0]["text"] == "hi" - assert body["stop_reason"] == "end_turn" - assert passthrough_upstream.requests, "Anthropic inbound did not reach the OpenAI upstream" - - # Verify the request the upstream SAW was already in OpenAI Chat - # Completions shape — i.e. the request translator ran. - upstream_body = passthrough_upstream.requests[-1]["body"] - assert "messages" in upstream_body - assert upstream_body["messages"][-1]["content"] == "ping" - - -# --------------------------------------------------------------------------- -# Passthrough route — Responses API inbound, OpenAI Chat upstream -# --------------------------------------------------------------------------- - - -class TestPassthroughResponsesApi: - """Inbound OpenAI Responses API, translated to Chat Completions upstream.""" - - async def test_non_streaming_translates_both_directions( - self, - passthrough_client: httpx.AsyncClient, - passthrough_upstream: _OpenAICompatStub, - ) -> None: - passthrough_upstream.respond_json(200, _completion_payload(content="resp-ok")) - - resp = await passthrough_client.post( - "/v1/responses", - json={"model": "any-model", "input": "ping"}, - ) - - assert resp.status_code == 200, resp.text - body = resp.json() - msg = body["output"][0] - assert msg["type"] == "message" - assert msg["role"] == "assistant" - assert msg["content"][0]["type"] == "output_text" - assert msg["content"][0]["text"] == "resp-ok" - assert passthrough_upstream.requests - - -# --------------------------------------------------------------------------- -# Backend errors propagate as HTTP errors, not 200 with garbage -# --------------------------------------------------------------------------- - - -class TestPassthroughBackendErrors: - """Upstream HTTP errors must surface as HTTP errors to the client. - - A regression where the chain swallows an upstream 4xx/5xx and still - returns 200 with malformed JSON is exactly the kind of silent failure - we want CI to catch. - """ - - async def test_upstream_500_does_not_return_200( - self, - passthrough_client: httpx.AsyncClient, - passthrough_upstream: _OpenAICompatStub, - ) -> None: - passthrough_upstream.respond_json(500, {"error": {"message": "boom"}}) - - resp = await passthrough_client.post( - "/v1/chat/completions", - json={ - "model": "any-model", - "messages": [{"role": "user", "content": "ping"}], - }, - ) - - # Whatever the exact mapping is, returning 200 on an upstream 500 is wrong. - assert resp.status_code != 200, ( - f"Upstream 500 leaked through as a successful response: body={resp.text!r}" - ) - assert resp.status_code >= 400 - - async def test_upstream_401_does_not_return_200( - self, - passthrough_client: httpx.AsyncClient, - passthrough_upstream: _OpenAICompatStub, - ) -> None: - passthrough_upstream.respond_json( - 401, - {"error": {"message": "bad key", "type": "invalid_api_key"}}, - ) - - resp = await passthrough_client.post( - "/v1/chat/completions", - json={ - "model": "any-model", - "messages": [{"role": "user", "content": "ping"}], - }, - ) - - assert resp.status_code != 200 - assert resp.status_code >= 400 - - -# --------------------------------------------------------------------------- -# Upstream-attempt outcome counters are wired for the Rust passthrough backend -# --------------------------------------------------------------------------- - - -class TestPassthroughUpstreamAttemptCounters: - """`switchyard_upstream_attempts_total` must populate for passthrough chains. - - The endpoint-layer fallback records one upstream attempt per request for - backends (here the Rust ``OpenAiPassthroughBackend``) that issue exactly - one upstream call and have no Python retry loop — they cannot, by - themselves, reach the Python-only ``outcome_metrics`` counters. - """ - - @pytest.fixture(autouse=True) - def _reset_counters(self) -> Iterator[None]: - outcome_metrics._reset_for_tests() - yield - outcome_metrics._reset_for_tests() - - async def test_success_records_one_200_attempt( - self, - passthrough_client: httpx.AsyncClient, - passthrough_upstream: _OpenAICompatStub, - ) -> None: - passthrough_upstream.respond_json(200, _completion_payload(content="ok")) - - resp = await passthrough_client.post( - "/v1/chat/completions", - json={ - "model": "any-model", - "messages": [{"role": "user", "content": "ping"}], - }, - ) - assert resp.status_code == 200, resp.text - - out = "\n".join(outcome_metrics.render_lines()) - assert 'switchyard_upstream_attempts_total{outcome="success",code="200"} 1' in out - assert "switchyard_router_retry_recovered_total 0" in out - - async def test_upstream_500_records_one_retryable_attempt( - self, - passthrough_client: httpx.AsyncClient, - passthrough_upstream: _OpenAICompatStub, - ) -> None: - passthrough_upstream.respond_json(500, {"error": {"message": "boom"}}) - - resp = await passthrough_client.post( - "/v1/chat/completions", - json={ - "model": "any-model", - "messages": [{"role": "user", "content": "ping"}], - }, - ) - assert resp.status_code >= 400 - - out = "\n".join(outcome_metrics.render_lines()) - assert ( - 'switchyard_upstream_attempts_total{outcome="retryable_error",code="500"} 1' - in out - ) - assert 'switchyard_upstream_attempts_total{outcome="success",code="200"} 0' in out - - async def test_upstream_401_records_one_other_error_attempt( - self, - passthrough_client: httpx.AsyncClient, - passthrough_upstream: _OpenAICompatStub, - ) -> None: - passthrough_upstream.respond_json( - 401, - {"error": {"message": "bad key", "type": "invalid_api_key"}}, - ) - - resp = await passthrough_client.post( - "/v1/chat/completions", - json={ - "model": "any-model", - "messages": [{"role": "user", "content": "ping"}], - }, - ) - assert resp.status_code >= 400 - - out = "\n".join(outcome_metrics.render_lines()) - assert ( - 'switchyard_upstream_attempts_total{outcome="other_error",code="401"} 1' - in out - ) - # A 4xx client error is not retryable and never recovers. - assert "switchyard_router_retry_recovered_total 0" in out diff --git a/tests/test_request_metadata.py b/tests/test_request_metadata.py deleted file mode 100644 index 62e3187aa..000000000 --- a/tests/test_request_metadata.py +++ /dev/null @@ -1,129 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Unit tests for header helpers in :mod:`switchyard.lib.request_metadata`.""" - -from __future__ import annotations - -import pytest - -from switchyard.lib.proxy_context import CTX_CALLER_API_KEY, ProxyContext -from switchyard.lib.request_metadata import ( - CTX_REQUEST_HEADERS, - RequestMetadata, - attach_caller_api_key, - attach_request_metadata, - extract_caller_api_key, - redact_sensitive_headers, -) - - -class TestExtractCallerApiKey: - """``extract_caller_api_key`` parses caller credentials from HTTP headers. - - Multi-tenant deploys forward each caller's key per request. The dedicated - ``x-switchyard-api-key`` header is preferred (it survives proxies such as - LiteLLM that strip ``Authorization``); ``Authorization: Bearer `` and - ``x-api-key`` remain supported for direct callers. The codex launcher sends - ``"switchyard"`` as a sentinel placeholder, which must not be forwarded. - """ - - @pytest.mark.parametrize( - "headers, expected", - [ - ({"Authorization": "Bearer nvapi-real"}, "nvapi-real"), - ({"authorization": "bearer nvapi-lowercase"}, "nvapi-lowercase"), - ({"x-api-key": "nvapi-via-x-api-key"}, "nvapi-via-x-api-key"), - ({"X-Api-Key": "nvapi-titlecase-header"}, "nvapi-titlecase-header"), - # The dedicated forwarded header is honored... - ({"x-switchyard-api-key": "nvapi-forwarded"}, "nvapi-forwarded"), - ({"X-Switchyard-Api-Key": "nvapi-fwd-titlecase"}, "nvapi-fwd-titlecase"), - # ...and wins over Authorization and x-api-key when several are set - # (the case behind a proxy that strips Authorization upstream). - ( - { - "x-switchyard-api-key": "forwarded-wins", - "Authorization": "Bearer lose", - "x-api-key": "lose-too", - }, - "forwarded-wins", - ), - # Authorization wins over x-api-key when both are present. - ( - {"Authorization": "Bearer first", "x-api-key": "second"}, - "first", - ), - # Surrounding whitespace is stripped. - ({"Authorization": "Bearer nvapi-spaces "}, "nvapi-spaces"), - ({"x-switchyard-api-key": " nvapi-fwd-spaces "}, "nvapi-fwd-spaces"), - ], - ) - def test_extraction(self, headers: dict[str, str], expected: str) -> None: - assert extract_caller_api_key(headers) == expected - - @pytest.mark.parametrize( - "headers", - [ - {}, - {"Authorization": ""}, - {"Authorization": "Bearer "}, - # Non-bearer schemes are not forwarded. - {"Authorization": "Basic dXNlcjpwYXNz"}, - # Codex launcher sentinel — in any supported header. - {"Authorization": "Bearer switchyard"}, - {"x-api-key": "switchyard"}, - {"x-switchyard-api-key": "switchyard"}, - {"x-switchyard-api-key": ""}, - # Case-insensitive sentinel match — both halves of the case - # space should be treated as the same placeholder. - {"Authorization": "Bearer Switchyard"}, - ], - ) - def test_no_key_returned(self, headers: dict[str, str]) -> None: - assert extract_caller_api_key(headers) is None - - -class TestRedactSensitiveHeaders: - """Credential headers are scrubbed before the header map is retained.""" - - def test_redacts_credential_headers(self) -> None: - headers = { - "Authorization": "Bearer nvapi-real", - "x-api-key": "nvapi-x", - "x-switchyard-api-key": "nvapi-forwarded", - "x-switchyard-intake-task": "demo", - "content-type": "application/json", - } - redacted = redact_sensitive_headers(headers) - assert redacted["Authorization"] == "[REDACTED]" - assert redacted["x-api-key"] == "[REDACTED]" - assert redacted["x-switchyard-api-key"] == "[REDACTED]" - # Non-credential headers pass through untouched. - assert redacted["x-switchyard-intake-task"] == "demo" - assert redacted["content-type"] == "application/json" - - def test_matching_is_case_insensitive(self) -> None: - redacted = redact_sensitive_headers({"X-Switchyard-Api-Key": "nvapi-real"}) - assert redacted["X-Switchyard-Api-Key"] == "[REDACTED]" - - -class TestCallerKeyForwardedButNotRetained: - """The endpoint extracts the caller key for upstream use, then retains a - redacted header map so the key cannot leak into logs or traces.""" - - def test_key_extracted_but_redacted_in_stored_headers(self) -> None: - headers = { - "x-switchyard-api-key": "nvapi-secret", - "x-switchyard-intake-task": "demo", - } - ctx = ProxyContext() - # Mirror the endpoint: both helpers receive the raw headers. - attach_request_metadata(ctx, RequestMetadata.from_headers(headers), headers) - attach_caller_api_key(ctx, headers) - - # Extracted for upstream forwarding... - assert ctx.metadata[CTX_CALLER_API_KEY] == "nvapi-secret" # pragma: allowlist secret - # ...but the retained header map carries no raw credential. - stored = ctx.metadata[CTX_REQUEST_HEADERS] - assert stored["x-switchyard-api-key"] == "[REDACTED]" - assert stored["x-switchyard-intake-task"] == "demo" diff --git a/tests/test_request_translation_engine.py b/tests/test_request_translation_engine.py deleted file mode 100644 index 1e01c831b..000000000 --- a/tests/test_request_translation_engine.py +++ /dev/null @@ -1,495 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for the unified TranslationEngine request wrapper.""" - -import copy -import json - -from switchyard_rust.core import ChatRequest, ChatRequestType, request_type_matches -from switchyard_rust.translation import TranslationEngine - -E = TranslationEngine() - - -# ========================================================================= -# to_openai_chat -# ========================================================================= - - -class TestToOpenAIChat: - def test_openai_passthrough(self): - """OpenAI chat requests pass through unchanged (same object).""" - body = {"model": "gpt-4o", "messages": [{"role": "user", "content": "Hi"}]} - req = ChatRequest.openai_chat(body) - result = E.request_to(ChatRequestType.OPENAI_CHAT, req) - assert result is req - - def test_anthropic_to_openai_basic(self): - """Anthropic requests are converted to OpenAI chat requests.""" - body = { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "Hello"}], - "max_tokens": 1024, - "system": "Be helpful.", - } - req = ChatRequest.anthropic(body) - result = E.request_to(ChatRequestType.OPENAI_CHAT, req) - - assert request_type_matches(result, ChatRequestType.OPENAI_CHAT) - msgs = result.body["messages"] - # System prompt becomes first message - assert msgs[0]["role"] == "system" - assert msgs[0]["content"] == "Be helpful." - # User message follows - assert msgs[1]["role"] == "user" - assert msgs[1]["content"] == "Hello" - assert result.body["model"] == "claude-sonnet-4-20250514" - - def test_anthropic_to_openai_with_tools(self): - """Anthropic tools are converted to OpenAI tool format.""" - body = { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "What's the weather?"}], - "max_tokens": 1024, - "tools": [ - { - "name": "get_weather", - "description": "Get weather for a location", - "input_schema": { - "type": "object", - "properties": {"location": {"type": "string"}}, - }, - } - ], - } - req = ChatRequest.anthropic(body) - result = E.request_to(ChatRequestType.OPENAI_CHAT, req) - - assert request_type_matches(result, ChatRequestType.OPENAI_CHAT) - tools = result.body.get("tools", []) - assert len(tools) == 1 - assert tools[0]["function"]["name"] == "get_weather" - - def test_anthropic_to_openai_does_not_mutate_original(self): - """The original Anthropic request body is not modified.""" - body = { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "Hi"}], - "max_tokens": 1024, - } - original = copy.deepcopy(body) - req = ChatRequest.anthropic(body) - E.request_to(ChatRequestType.OPENAI_CHAT, req) - assert req.body == original - - def test_responses_to_openai_basic(self): - """Responses requests are converted to OpenAI chat requests.""" - body = { - "model": "gpt-4o", - "input": "Hello world", - "instructions": "Be brief.", - } - req = ChatRequest.openai_responses(body) - result = E.request_to(ChatRequestType.OPENAI_CHAT, req) - - assert request_type_matches(result, ChatRequestType.OPENAI_CHAT) - msgs = result.body["messages"] - assert msgs[0]["role"] == "system" - assert msgs[0]["content"] == "Be brief." - assert msgs[1]["role"] == "user" - assert msgs[1]["content"] == "Hello world" - - def test_responses_to_openai_with_tools(self): - """Responses API tools are converted to OpenAI tool format.""" - body = { - "model": "gpt-4o", - "input": "Get weather", - "tools": [ - { - "name": "get_weather", - "description": "Get weather", - "parameters": { - "type": "object", - "properties": {"loc": {"type": "string"}}, - }, - } - ], - } - req = ChatRequest.openai_responses(body) - result = E.request_to(ChatRequestType.OPENAI_CHAT, req) - - tools = result.body.get("tools", []) - assert len(tools) == 1 - assert tools[0]["function"]["name"] == "get_weather" - - def test_responses_to_openai_does_not_mutate_original(self): - """The original Responses request body is not modified.""" - body = {"model": "gpt-4o", "input": "Hi"} - original = copy.deepcopy(body) - req = ChatRequest.openai_responses(body) - E.request_to(ChatRequestType.OPENAI_CHAT, req) - assert req.body == original - - def test_anthropic_only_top_level_fields_dropped(self): - """Anthropic-only fields must not leak into the OpenAI request — the - OpenAI SDK rejects them with TypeError at call time. - - Claude Code sends several of these on every request (``thinking`` for - extended thinking, ``cache_control`` for prompt caching, - ``context_management`` for long-context management, ``container`` for - Claude containers). Including a synthetic ``made_up_beta_field`` to - assert the whitelist strategy also handles future Anthropic-only - fields we don't know about yet. - """ - body = { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "Hi"}], - "max_tokens": 1024, - "thinking": {"type": "enabled", "budget_tokens": 8000}, - "cache_control": {"type": "ephemeral"}, - "container": "my-container", - "inference_geo": "us-east-1", - "output_config": {"some": "config"}, - "context_management": {"strategy": "auto"}, - "made_up_beta_field": "future-proofing", - } - req = ChatRequest.anthropic(body) - result = E.request_to(ChatRequestType.OPENAI_CHAT, req) - for field in ( - "thinking", "cache_control", "container", "inference_geo", - "output_config", "context_management", "made_up_beta_field", - ): - assert field not in result.body, ( - f"{field!r} leaked into OpenAI request — " - "OpenAI SDK would reject with TypeError" - ) - - def test_anthropic_thinking_content_blocks_do_not_leak_to_openai(self): - """Anthropic thinking blocks are preserved internally, not sent as Chat content.""" - body = { - "model": "claude-opus-4-20250514", - "messages": [ - {"role": "user", "content": "Use the tool."}, - { - "role": "assistant", - "content": [ - { - "type": "thinking", - "thinking": "I should call the tool.", - "signature": "sig-abc", - }, - {"type": "redacted_thinking", "data": "encrypted"}, - { - "type": "tool_use", - "id": "toolu_1", - "name": "lookup", - "input": {"query": "status"}, - }, - ], - }, - ], - "tools": [{"name": "lookup", "input_schema": {"type": "object"}}], - "max_tokens": 2048, - } - req = ChatRequest.anthropic(body) - result = E.request_to(ChatRequestType.OPENAI_CHAT, req) - - assistant = result.body["messages"][1] - assert assistant["role"] == "assistant" - assert assistant["content"] is None - assert "reasoning_content" not in assistant - assert assistant["tool_calls"][0]["function"]["name"] == "lookup" - assert "thinking" not in str(result.body) - assert "redacted_thinking" not in str(result.body) - - -# ========================================================================= -# to_anthropic -# ========================================================================= - - -class TestToAnthropic: - def test_anthropic_passthrough(self): - """Anthropic requests pass through unchanged.""" - body = { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "Hi"}], - "max_tokens": 1024, - } - req = ChatRequest.anthropic(body) - result = E.request_to(ChatRequestType.ANTHROPIC, req) - assert result is req - - def test_openai_to_anthropic(self): - """OpenAI chat requests are converted to Anthropic requests.""" - body = { - "model": "gpt-4o", - "messages": [ - {"role": "system", "content": "Be helpful."}, - {"role": "user", "content": "Hello"}, - ], - "max_tokens": 1024, - } - req = ChatRequest.openai_chat(body) - result = E.request_to(ChatRequestType.ANTHROPIC, req) - - assert request_type_matches(result, ChatRequestType.ANTHROPIC) - assert result.body["system"] == "Be helpful." - assert result.body["model"] == "gpt-4o" - - def test_openai_to_anthropic_developer_and_system_concat(self): - """OpenAI developer/system messages must not leak as invalid Anthropic roles.""" - body = { - "model": "gpt-4o", - "messages": [ - {"role": "system", "content": "System rules."}, - {"role": "developer", "content": "Developer rules."}, - {"role": "user", "content": "Hello"}, - ], - } - req = ChatRequest.openai_chat(body) - result = E.request_to(ChatRequestType.ANTHROPIC, req) - - assert result.body["system"] == "System rules.\n\nDeveloper rules." - assert [m["role"] for m in result.body["messages"]] == ["user"] - - def test_openai_to_anthropic_uses_max_completion_tokens(self): - body = { - "model": "gpt-4o", - "messages": [{"role": "user", "content": "Hello"}], - "max_completion_tokens": 512, - } - req = ChatRequest.openai_chat(body) - result = E.request_to(ChatRequestType.ANTHROPIC, req) - assert result.body["max_tokens"] == 512 - - def test_openai_to_anthropic_maps_reasoning_effort(self): - body = { - "model": "gpt-4o", - "messages": [{"role": "user", "content": "Hello"}], - "reasoning_effort": "high", - } - req = ChatRequest.openai_chat(body) - result = E.request_to(ChatRequestType.ANTHROPIC, req) - assert result.body["thinking"] == {"type": "adaptive"} - assert result.body["output_config"] == {"effort": "high"} - - def test_openai_to_anthropic_maps_image_url_content(self): - body = { - "model": "gpt-4o", - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": "Describe"}, - { - "type": "image_url", - "image_url": { - "url": "https://example.test/image.png", - }, - }, - ], - } - ], - } - req = ChatRequest.openai_chat(body) - result = E.request_to(ChatRequestType.ANTHROPIC, req) - content = result.body["messages"][0]["content"] - assert content == [ - {"type": "text", "text": "Describe"}, - { - "type": "image", - "source": { - "type": "url", - "url": "https://example.test/image.png", - }, - }, - ] - - def test_openai_to_anthropic_merges_consecutive_tool_results(self): - body = { - "model": "gpt-4o", - "messages": [ - {"role": "user", "content": "call tools"}, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": {"name": "a", "arguments": "{}"}, - }, - { - "id": "call_2", - "type": "function", - "function": {"name": "b", "arguments": "{}"}, - }, - ], - }, - {"role": "tool", "tool_call_id": "call_1", "content": "one"}, - {"role": "tool", "tool_call_id": "call_2", "content": "two"}, - ], - } - req = ChatRequest.openai_chat(body) - result = E.request_to(ChatRequestType.ANTHROPIC, req) - - tool_result_msg = result.body["messages"][2] - assert tool_result_msg["role"] == "user" - assert tool_result_msg["content"] == [ - {"type": "tool_result", "tool_use_id": "call_1", "content": "one"}, - {"type": "tool_result", "tool_use_id": "call_2", "content": "two"}, - ] - - def test_anthropic_tool_result_followup_text_is_preserved(self): - body = { - "model": "claude-sonnet-4-20250514", - "messages": [ - { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "toolu_1", - "content": "72F", - }, - {"type": "text", "text": "Now summarize it."}, - ], - } - ], - "max_tokens": 1024, - } - req = ChatRequest.anthropic(body) - result = E.request_to(ChatRequestType.OPENAI_CHAT, req) - assert result.body["messages"] == [ - {"role": "tool", "tool_call_id": "toolu_1", "content": "72F"}, - {"role": "user", "content": "Now summarize it."}, - ] - - def test_openai_to_anthropic_sanitizes_tool_call_ids(self): - """Tool call IDs and matching tool results must stay valid together.""" - invalid_id = "call.bad:id/with space" - body = { - "model": "gpt-4o", - "messages": [ - {"role": "user", "content": "search"}, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": invalid_id, - "type": "function", - "function": { - "name": "search", - "arguments": '{"q":"test"}', - }, - }, - ], - }, - { - "role": "tool", - "tool_call_id": invalid_id, - "content": "ok", - }, - ], - "max_tokens": 1024, - } - req = ChatRequest.openai_chat(body) - - result = E.request_to(ChatRequestType.ANTHROPIC, req) - - messages = result.body["messages"] - sanitized_id = "call_bad_id_with_space" - assert messages[1]["content"][0]["id"] == sanitized_id - assert messages[2]["content"][0]["tool_use_id"] == sanitized_id - - def test_openai_unknown_content_does_not_leak_to_anthropic(self): - """Unknown OpenAI content is kept as text instead of raw Anthropic blocks.""" - unknown_part = {"type": "future_openai_part", "payload": {"keep": True}} - body = { - "model": "gpt-4o", - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": "hi"}, - unknown_part, - ], - } - ], - } - req = ChatRequest.openai_chat(body) - - result = E.request_to(ChatRequestType.ANTHROPIC, req) - - content = result.body["messages"][0]["content"] - assert content[1]["type"] == "text" - assert json.loads(content[1]["text"]) == unknown_part - assert "future_openai_part" not in [ - block.get("type") for block in content if isinstance(block, dict) - ] - - def test_responses_to_anthropic(self): - """Responses requests can be converted to Anthropic requests.""" - body = {"model": "gpt-4o", "input": "Hi"} - req = ChatRequest.openai_responses(body) - result = E.request_to(ChatRequestType.ANTHROPIC, req) - assert request_type_matches(result, ChatRequestType.ANTHROPIC) - assert result.body["messages"] == [{"role": "user", "content": "Hi"}] - - def test_responses_tool_arguments_to_anthropic_are_object_shaped(self): - """Responses tool-call argument strings become valid Anthropic tool input objects.""" - body = { - "model": "gpt-4o", - "input": [ - {"type": "message", "role": "user", "content": "List files"}, - { - "type": "function_call", - "name": "exec_command", - "call_id": "call_1", - "arguments": '{"cmd":"ls","limit":2}', - }, - ], - } - req = ChatRequest.openai_responses(body) - result = E.request_to(ChatRequestType.ANTHROPIC, req) - - tool_use = result.body["messages"][1]["content"][0] - assert tool_use["type"] == "tool_use" - assert tool_use["input"] == {"cmd": "ls", "limit": 2} - - -# ========================================================================= -# to_responses -# ========================================================================= - - -class TestToResponses: - def test_responses_passthrough(self): - """Responses requests pass through unchanged.""" - body = {"model": "gpt-4o", "input": "Hi"} - req = ChatRequest.openai_responses(body) - result = E.request_to(ChatRequestType.OPENAI_RESPONSES, req) - assert result is req - - def test_openai_to_responses(self): - """OpenAI chat requests can be converted to Responses requests.""" - body = {"model": "gpt-4o", "messages": [{"role": "user", "content": "Hi"}]} - req = ChatRequest.openai_chat(body) - result = E.request_to(ChatRequestType.OPENAI_RESPONSES, req) - assert request_type_matches(result, ChatRequestType.OPENAI_RESPONSES) - assert result.body["input"] == "Hi" - - def test_anthropic_to_responses(self): - """Anthropic requests can be converted to Responses requests.""" - body = { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "Hi"}], - "max_tokens": 1024, - } - req = ChatRequest.anthropic(body) - result = E.request_to(ChatRequestType.OPENAI_RESPONSES, req) - assert request_type_matches(result, ChatRequestType.OPENAI_RESPONSES) - assert result.body["input"] == "Hi" diff --git a/tests/test_request_translation_engine_to_any_of.py b/tests/test_request_translation_engine_to_any_of.py deleted file mode 100644 index f145f843c..000000000 --- a/tests/test_request_translation_engine_to_any_of.py +++ /dev/null @@ -1,162 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Unit tests for :meth:`TranslationEngine.request_to_any_of`. - -Covers the four behaviors: - -- Passthrough when the inbound type is in ``supported``. -- Translation to ``supported[0]`` when the inbound type isn't supported. -- ``ValueError`` on empty ``supported``. -- Any built-in request format can translate to any other built-in format. -""" - -import pytest - -from switchyard_rust.core import ChatRequest, ChatRequestType, request_type_matches -from switchyard_rust.translation import TranslationEngine - -ENGINE = TranslationEngine() - -# --------------------------------------------------------------------------- -# Fixtures -# --------------------------------------------------------------------------- - - -def _openai_req() -> ChatRequest: - return ChatRequest.openai_chat({ # type: ignore[arg-type] - "model": "gpt-4o", - "messages": [{"role": "user", "content": "hi"}], - }) - - -def _anthropic_req() -> ChatRequest: - return ChatRequest.anthropic({ # type: ignore[arg-type] - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 1024, - }) - - -def _responses_req() -> ChatRequest: - return ChatRequest.openai_responses({ # type: ignore[arg-type] - "model": "gpt-4o", - "input": "hi", - }) - - -# --------------------------------------------------------------------------- -# Passthrough (inbound type is in `supported`) -# --------------------------------------------------------------------------- - - -class TestPassthrough: - def test_openai_in_singleton_supported(self): - req = _openai_req() - out = ENGINE.request_to_any_of( - req, [ChatRequestType.OPENAI_CHAT], - ) - assert out is req - - def test_anthropic_in_singleton_supported(self): - req = _anthropic_req() - out = ENGINE.request_to_any_of( - req, [ChatRequestType.ANTHROPIC], - ) - assert out is req - - def test_responses_in_singleton_supported(self): - req = _responses_req() - out = ENGINE.request_to_any_of( - req, [ChatRequestType.OPENAI_RESPONSES], - ) - assert out is req - - def test_openai_in_multi_element_supported(self): - """Passthrough still wins even when supported lists more formats.""" - req = _openai_req() - out = ENGINE.request_to_any_of( - req, - [ChatRequestType.OPENAI_RESPONSES, ChatRequestType.OPENAI_CHAT], - ) - assert out is req - - def test_anthropic_in_multi_element_supported(self): - req = _anthropic_req() - out = ENGINE.request_to_any_of( - req, - [ChatRequestType.OPENAI_CHAT, ChatRequestType.ANTHROPIC], - ) - assert out is req - - -# --------------------------------------------------------------------------- -# Translation (inbound type not in `supported`, translate to supported[0]) -# --------------------------------------------------------------------------- - - -class TestTranslation: - def test_openai_chat_to_anthropic(self): - req = _openai_req() - out = ENGINE.request_to_any_of( - req, [ChatRequestType.ANTHROPIC], - ) - assert request_type_matches(out, ChatRequestType.ANTHROPIC) - assert out is not req - - def test_anthropic_to_openai_chat(self): - req = _anthropic_req() - out = ENGINE.request_to_any_of( - req, [ChatRequestType.OPENAI_CHAT], - ) - assert request_type_matches(out, ChatRequestType.OPENAI_CHAT) - assert out is not req - - def test_responses_to_openai_chat(self): - req = _responses_req() - out = ENGINE.request_to_any_of( - req, [ChatRequestType.OPENAI_CHAT], - ) - assert request_type_matches(out, ChatRequestType.OPENAI_CHAT) - assert out is not req - - def test_translation_target_is_supported_zero(self): - """When multiple unsupported formats translate, supported[0] is chosen.""" - req = _anthropic_req() - # Anthropic is not in supported; supported[0] is OPENAI_CHAT, so we - # should translate to OpenAI Chat (not to Responses, the second entry). - out = ENGINE.request_to_any_of( - req, - [ChatRequestType.OPENAI_CHAT, ChatRequestType.OPENAI_RESPONSES], - ) - assert request_type_matches(out, ChatRequestType.OPENAI_CHAT) - - -# --------------------------------------------------------------------------- -# Error paths -# --------------------------------------------------------------------------- - - -class TestErrors: - def test_empty_supported_raises_value_error(self): - req = _openai_req() - with pytest.raises(ValueError, match="must be non-empty"): - ENGINE.request_to_any_of(req, []) - - def test_openai_chat_to_responses(self): - """OpenAI Chat can translate to Responses.""" - req = _openai_req() - out = ENGINE.request_to_any_of(req, [ChatRequestType.OPENAI_RESPONSES]) - assert request_type_matches(out, ChatRequestType.OPENAI_RESPONSES) - - def test_anthropic_to_responses(self): - """Anthropic can translate to Responses.""" - req = _anthropic_req() - out = ENGINE.request_to_any_of(req, [ChatRequestType.OPENAI_RESPONSES]) - assert request_type_matches(out, ChatRequestType.OPENAI_RESPONSES) - - def test_responses_to_anthropic(self): - """Responses can translate to Anthropic.""" - req = _responses_req() - out = ENGINE.request_to_any_of(req, [ChatRequestType.ANTHROPIC]) - assert request_type_matches(out, ChatRequestType.ANTHROPIC) diff --git a/tests/test_response_translation_engine.py b/tests/test_response_translation_engine.py deleted file mode 100644 index 6ef605757..000000000 --- a/tests/test_response_translation_engine.py +++ /dev/null @@ -1,540 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for the unified TranslationEngine response wrapper.""" - -from __future__ import annotations - -from collections.abc import AsyncIterator -from types import SimpleNamespace -from unittest.mock import MagicMock - -import pytest -from openai.types.chat import ChatCompletion -from openai.types.chat.chat_completion import Choice -from openai.types.chat.chat_completion_message import ChatCompletionMessage -from openai.types.completion_usage import CompletionUsage -from openai.types.responses import Response as OpenAIResponse - -from switchyard.lib.chat_response.anthropic import AnthropicResponseStream -from switchyard.lib.chat_response.openai_chat import ResponseStream -from switchyard_rust.core import ( - ChatRequest, - ChatRequestType, - ChatResponse, - ChatResponseType, - ProxyContext, - response_type_matches, -) -from switchyard_rust.translation import TranslationEngine, sse_frame_payloads - -E = TranslationEngine() - - -def _make_completion(content: str = "Hello!", model: str = "gpt-4o") -> ChatCompletion: - return ChatCompletion( - id="chatcmpl-test", - object="chat.completion", - created=1700000000, - model=model, - choices=[ - Choice( - index=0, - message=ChatCompletionMessage(role="assistant", content=content), - finish_reason="stop", - ) - ], - usage=CompletionUsage(prompt_tokens=10, completion_tokens=5, total_tokens=15), - ) - - -def _make_anthropic_message(content: str = "Hi there", model: str = "claude-sonnet-4-20250514"): - from anthropic.types import Message, TextBlock, Usage - - return Message( - id="msg_test123", - type="message", - role="assistant", - content=[TextBlock(type="text", text=content)], - model=model, - stop_reason="end_turn", - stop_sequence=None, - usage=Usage(input_tokens=10, output_tokens=5), - ) - - -def _make_responses_api_response( - *, content: str = "Hello from Responses", model: str = "gpt-4o", -) -> OpenAIResponse: - return OpenAIResponse( - id="resp_test", - created_at=1700000000, - model=model, - object="response", - output=[ - { - "type": "message", - "id": "msg_test", - "status": "completed", - "role": "assistant", - "content": [{"type": "output_text", "text": content, "annotations": []}], - } - ], - tool_choice="auto", - tools=[], - status="completed", - parallel_tool_calls=True, - text={"format": {"type": "text"}}, - ) - - -# ========================================================================= -# to_openai_chat -# ========================================================================= - - -class TestToOpenAIChat: - def test_completion_passthrough(self): - """OpenAI completion responses pass through unchanged.""" - resp = ChatResponse.openai_completion(_make_completion()) - result = E.response_to(ChatRequestType.OPENAI_CHAT, resp) - assert result is resp - - def test_streaming_passthrough(self): - """OpenAI streaming responses pass through unchanged.""" - stream = MagicMock(spec=ResponseStream) - resp = ChatResponse.openai_stream(stream) - result = E.response_to(ChatRequestType.OPENAI_CHAT, resp) - assert result is resp - - def test_anthropic_to_openai(self): - """Anthropic responses are converted to OpenAI completion responses.""" - resp = ChatResponse.anthropic_completion(_make_anthropic_message(content="Hello world")) - result = E.response_to(ChatRequestType.OPENAI_CHAT, resp) - - assert response_type_matches(result, ChatResponseType.OPENAI_COMPLETION) - assert result.body["choices"][0]["message"]["content"] == "Hello world" - assert result.body["choices"][0]["finish_reason"] == "stop" - - def test_responses_api_to_openai(self): - """Responses API responses are converted to OpenAI completion responses.""" - resp = ChatResponse.openai_responses_completion( - _make_responses_api_response(content="Hello from Responses"), - ) - result = E.response_to(ChatRequestType.OPENAI_CHAT, resp) - - assert response_type_matches(result, ChatResponseType.OPENAI_COMPLETION) - assert result.body["model"] == "gpt-4o" - assert result.body["choices"][0]["message"]["content"] == "Hello from Responses" - assert result.body["choices"][0]["finish_reason"] == "stop" - - -# ========================================================================= -# to_anthropic -# ========================================================================= - - -class TestToAnthropic: - def test_anthropic_passthrough(self): - """Anthropic completion responses pass through unchanged.""" - resp = ChatResponse.anthropic_completion(_make_anthropic_message()) - result = E.response_to(ChatRequestType.ANTHROPIC, resp) - assert result is resp - - def test_anthropic_streaming_passthrough(self): - """Anthropic streaming responses pass through unchanged.""" - from switchyard.lib.chat_response.anthropic import AnthropicResponseStream - - stream = MagicMock(spec=AnthropicResponseStream) - resp = ChatResponse.anthropic_stream(stream) - result = E.response_to(ChatRequestType.ANTHROPIC, resp) - assert result is resp - - def test_openai_to_anthropic(self): - """OpenAI completion responses are converted to Anthropic responses.""" - resp = ChatResponse.openai_completion(_make_completion(content="Hello world")) - result = E.response_to(ChatRequestType.ANTHROPIC, resp) - - assert response_type_matches(result, ChatResponseType.ANTHROPIC_COMPLETION) - body = result.body - assert body["role"] == "assistant" - assert body["stop_reason"] == "end_turn" - assert len(body["content"]) == 1 - assert body["content"][0]["text"] == "Hello world" - - def test_responses_api_to_anthropic(self): - """Responses API responses are converted to Anthropic responses.""" - resp = ChatResponse.openai_responses_completion(_make_responses_api_response(content="Hello world")) - result = E.response_to(ChatRequestType.ANTHROPIC, resp) - - assert response_type_matches(result, ChatResponseType.ANTHROPIC_COMPLETION) - assert result.body["content"][0]["text"] == "Hello world" - - -# ========================================================================= -# to_responses -# ========================================================================= - - -class TestToResponses: - def test_responses_passthrough(self): - """Responses API completion responses pass through unchanged.""" - resp = ChatResponse.openai_responses_completion(_make_responses_api_response()) - result = E.response_to(ChatRequestType.OPENAI_RESPONSES, resp) - assert result is resp - - def test_responses_streaming_passthrough(self): - """Responses API streaming responses pass through unchanged.""" - from switchyard.lib.chat_response.openai_responses import ResponsesApiStream - - resp = ChatResponse.openai_responses_stream(ResponsesApiStream(_async_iter([]))) - result = E.response_to(ChatRequestType.OPENAI_RESPONSES, resp) - assert result is resp - - def test_openai_to_responses(self): - """OpenAI completion responses are converted to Responses API responses.""" - resp = ChatResponse.openai_completion(_make_completion(content="Hello!")) - result = E.response_to(ChatRequestType.OPENAI_RESPONSES, resp) - - assert response_type_matches(result, ChatResponseType.OPENAI_RESPONSES_COMPLETION) - body = result.body - assert body["object"] == "response" - assert body["status"] == "completed" - assert len(body["output"]) == 1 - assert body["output"][0]["type"] == "message" - - def test_anthropic_to_responses(self): - """Anthropic responses are converted to Responses API responses.""" - resp = ChatResponse.anthropic_completion(_make_anthropic_message()) - result = E.response_to(ChatRequestType.OPENAI_RESPONSES, resp) - - assert response_type_matches(result, ChatResponseType.OPENAI_RESPONSES_COMPLETION) - assert result.body["output"][0]["content"][0]["text"] == "Hi there" - - -# ========================================================================= -# stream_openai_to_anthropic — usage propagation -# ========================================================================= - - -def _chunk( - content: str | None = None, - reasoning_content: str | None = None, - finish_reason: str | None = None, - usage: SimpleNamespace | None = None, -) -> SimpleNamespace: - """Build a minimal OpenAI-shaped streaming chunk.""" - delta = SimpleNamespace( - content=content, - reasoning=None, - reasoning_content=reasoning_content, - tool_calls=None, - ) - choice = SimpleNamespace(delta=delta, finish_reason=finish_reason) - chunk = SimpleNamespace(choices=[choice]) - if usage is not None: - chunk.usage = usage - return chunk - - -async def _async_iter(items: list) -> AsyncIterator: - for item in items: - yield item - - -async def _collect_events(chunks: list) -> list[dict]: - events: list[dict] = [] - async for ev in E.translate_stream( - ChatRequestType.OPENAI_CHAT, - ChatRequestType.ANTHROPIC, - _async_iter(chunks), - model="test-model", - ): - events.append(ev) - return events - - -def _find_event(events: list[dict], event_type: str) -> dict: - return next(ev for ev in events if ev["type"] == event_type) - - -# ========================================================================= -# terminal translate() role — served-model stamping (SWITCH-922) -# ========================================================================= - - -_ROUTE_ID = "switchyard" -_SERVED_MODEL = "served/model" - - -def _routed_ctx(served_model: str | None = _SERVED_MODEL) -> ProxyContext: - """A context shaped like one a router leaves behind for the terminal role.""" - ctx = ProxyContext() - ctx.selected_model = served_model - return ctx - - -class TestTranslateStampsTheServedModel: - """The client-visible ``model`` must name the model that answered. - - Routers rewrite the model on their own copy of the request, so the request - reaching the terminal role still names the route the client addressed - (``switchyard`` for the benchmark bundles). Reading it from there labelled - every routed turn with the route id. - """ - - @pytest.mark.asyncio - async def test_anthropic_stream_reports_the_served_model(self): - request = ChatRequest.anthropic( - {"model": _ROUTE_ID, "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]} - ) - response = ChatResponse.openai_stream(ResponseStream(_async_iter([_chunk(content="hi")]))) - - events = [ev async for ev in await E.translate(_routed_ctx(), request, response)] - - assert events[0]["message"]["model"] == _SERVED_MODEL - - @pytest.mark.asyncio - async def test_responses_stream_reports_the_served_model(self): - request = ChatRequest.openai_responses({"model": _ROUTE_ID, "input": "hi"}) - response = ChatResponse.openai_stream(ResponseStream(_async_iter([_chunk(content="hi")]))) - - frames = [frame async for frame in await E.translate(_routed_ctx(), request, response)] - payloads = [ - payload for frame in frames for payload in sse_frame_payloads(frame) - ] - - assert payloads[0]["response"]["model"] == _SERVED_MODEL - - @pytest.mark.asyncio - async def test_openai_chat_stream_reports_the_served_model(self): - request = ChatRequest.openai_chat( - {"model": _ROUTE_ID, "messages": [{"role": "user", "content": "hi"}]} - ) - response = ChatResponse.anthropic_stream( - AnthropicResponseStream( - _async_iter( - [ - { - "type": "message_start", - "message": { - "id": "msg_1", - "model": "upstream/model", - "role": "assistant", - "content": [], - }, - }, - { - "type": "content_block_delta", - "index": 0, - "delta": {"type": "text_delta", "text": "hi"}, - }, - ] - ) - ) - ) - - chunks = [chunk async for chunk in await E.translate(_routed_ctx(), request, response)] - - # The OpenAI-chat target yields SDK chunk objects, not dicts. - assert chunks[0].model == _SERVED_MODEL - - @pytest.mark.asyncio - async def test_buffered_response_reports_the_served_model(self): - request = ChatRequest.anthropic( - {"model": _ROUTE_ID, "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]} - ) - response = ChatResponse.openai_completion(_make_completion(model=_SERVED_MODEL)) - - result = await E.translate(_routed_ctx(), request, response) - - assert result["model"] == _SERVED_MODEL - - @pytest.mark.asyncio - async def test_stream_falls_back_to_the_upstream_model_without_a_selection(self): - """Passthrough routes keep reporting whatever the provider announced.""" - request = ChatRequest.openai_chat( - {"model": _ROUTE_ID, "messages": [{"role": "user", "content": "hi"}]} - ) - response = ChatResponse.anthropic_stream( - AnthropicResponseStream( - _async_iter( - [ - { - "type": "message_start", - "message": { - "id": "msg_1", - "model": "upstream/model", - "role": "assistant", - "content": [], - }, - }, - { - "type": "content_block_delta", - "index": 0, - "delta": {"type": "text_delta", "text": "hi"}, - }, - ] - ) - ) - ) - - chunks = [ - chunk async for chunk in await E.translate(_routed_ctx(None), request, response) - ] - - assert chunks[0].model == "upstream/model" - - -class TestStreamOpenAIToAnthropicUsage: - @pytest.mark.asyncio - async def test_backend_usage_preferred_over_heuristic(self): - """Real usage from the backend replaces the per-delta +1 counter.""" - usage = SimpleNamespace( - prompt_tokens=500, - completion_tokens=42, - total_tokens=542, - prompt_tokens_details=None, - ) - chunks = [ - _chunk(content="Hello"), - _chunk(content=" world", usage=usage), - _chunk(finish_reason="stop"), - ] - events = await _collect_events(chunks) - msg_delta = _find_event(events, "message_delta") - - assert msg_delta["usage"]["input_tokens"] == 500 - assert msg_delta["usage"]["output_tokens"] == 42 - - @pytest.mark.asyncio - async def test_cached_tokens_propagated(self): - """Anthropic usage splits uncached input from cache-read tokens.""" - usage = SimpleNamespace( - prompt_tokens=1000, - completion_tokens=10, - total_tokens=1010, - prompt_tokens_details=SimpleNamespace(cached_tokens=800), - ) - chunks = [_chunk(content="Hi", usage=usage)] - events = await _collect_events(chunks) - msg_delta = _find_event(events, "message_delta") - - assert msg_delta["usage"]["cache_read_input_tokens"] == 800 - assert msg_delta["usage"]["input_tokens"] == 200 - - @pytest.mark.asyncio - async def test_heuristic_fallback_when_no_backend_usage(self): - """Without backend usage, output_tokens falls back to delta counting.""" - chunks = [ - _chunk(content="a"), - _chunk(content="b"), - _chunk(content="c"), - _chunk(finish_reason="stop"), - ] - events = await _collect_events(chunks) - msg_delta = _find_event(events, "message_delta") - - assert msg_delta["usage"]["output_tokens"] == 3 - assert "input_tokens" not in msg_delta["usage"] - assert "cache_read_input_tokens" not in msg_delta["usage"] - - @pytest.mark.asyncio - async def test_reasoning_stream_events_validate_against_anthropic_sdk(self): - from anthropic.types import ( # noqa: PLC0415 - RawContentBlockDeltaEvent, - RawContentBlockStartEvent, - ) - - events = await _collect_events([ - _chunk(reasoning_content="private"), - _chunk(content="visible"), - _chunk(finish_reason="stop"), - ]) - - thinking_start = next( - event for event in events - if event["type"] == "content_block_start" - and event["content_block"]["type"] == "thinking" - ) - signature_delta = next( - event for event in events - if event["type"] == "content_block_delta" - and event["delta"]["type"] == "signature_delta" - ) - assert thinking_start["content_block"]["signature"] == "" - assert signature_delta["delta"]["signature"] == "" - RawContentBlockStartEvent.model_validate(thinking_start) - RawContentBlockDeltaEvent.model_validate(signature_delta) - - -# --------------------------------------------------------------------------- -# Raw SSE frame parsing for cross-format stream translation -# --------------------------------------------------------------------------- - - -class TestSseFramePayloads: - """``sse_frame_payloads`` feeds verbatim-passthrough frame strings back - into the translator; the SSE grammar corners must parse correctly.""" - - def test_single_frame_with_event_name(self): - from switchyard_rust.translation import sse_frame_payloads - - payloads = sse_frame_payloads( - 'event: response.created\ndata: {"type":"response.created"}\n\n' - ) - assert payloads == [{"type": "response.created"}] - - def test_multi_data_lines_join_with_newline(self): - from switchyard_rust.translation import sse_frame_payloads - - payloads = sse_frame_payloads('data: {"a":\ndata: 1}\n\n') - assert payloads == [{"a": 1}] - - def test_comment_done_and_non_json_frames_yield_nothing(self): - from switchyard_rust.translation import sse_frame_payloads - - assert sse_frame_payloads(": keep-alive\n\n") == [] - assert sse_frame_payloads("data: [DONE]\n\n") == [] - assert sse_frame_payloads("data: not-json\n\n") == [] - - def test_multiple_frames_in_one_string_parse_in_order(self): - from switchyard_rust.translation import sse_frame_payloads - - payloads = sse_frame_payloads('data: {"a":1}\n\ndata: {"b":2}\n\n') - assert payloads == [{"a": 1}, {"b": 2}] - - -async def test_translate_stream_accepts_raw_sse_frame_strings(): - """Cross-format translation parses raw Responses frames into chat chunks.""" - - async def _frames() -> AsyncIterator[str]: - yield ( - 'event: response.created\n' - 'data: {"type":"response.created","response":{"id":"r1","model":"m"}}\n\n' - ) - yield ( - 'event: response.output_text.delta\n' - 'data: {"type":"response.output_text.delta","delta":"hello"}\n\n' - ) - yield ": keep-alive\n\n" - yield ( - 'event: response.completed\n' - 'data: {"type":"response.completed","response":{"id":"r1","usage":' - '{"input_tokens":1,"output_tokens":1}}}\n\n' - ) - - chunks = [ - chunk - async for chunk in E.translate_stream( - "openai_responses", "openai_chat", _frames(), model="m", - ) - ] - - deltas = [ - choice.get("delta", {}).get("content") - for chunk in chunks - if isinstance(chunk, dict) - for choice in chunk.get("choices", []) - ] - assert "hello" in deltas diff --git a/tests/test_responses_openai_translation.py b/tests/test_responses_openai_translation.py deleted file mode 100644 index dff117c0e..000000000 --- a/tests/test_responses_openai_translation.py +++ /dev/null @@ -1,1137 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for OpenAI Responses <-> OpenAI Chat translation.""" - -from unittest.mock import MagicMock - -import pytest - -from switchyard_rust.translation import TranslationEngine - -ENGINE = TranslationEngine() - - -def _responses_request_to_chat(body: object) -> dict: - return ENGINE.translate_request("openai_responses", "openai_chat", body) - - -def _chat_response_to_responses(response: object) -> dict: - return ENGINE.translate_response("openai_chat", "openai_responses", response) - -# --------------------------------------------------------------------------- -# Request conversion tests -# --------------------------------------------------------------------------- - - -class TestConvertResponsesRequestToChatCompletions: - """Tests for convert_responses_request_to_chat_completions.""" - - def test_simple_string_input(self): - body = { - "model": "gpt-4", - "input": "Hello, world!", - } - result = _responses_request_to_chat(body) - - assert result["model"] == "gpt-4" - assert len(result["messages"]) == 1 - assert result["messages"][0] == {"role": "user", "content": "Hello, world!"} - - def test_instructions_becomes_system_message(self): - body = { - "model": "gpt-4", - "input": "What is 2+2?", - "instructions": "You are a math tutor.", - } - result = _responses_request_to_chat(body) - - assert len(result["messages"]) == 2 - assert result["messages"][0] == {"role": "system", "content": "You are a math tutor."} - assert result["messages"][1] == {"role": "user", "content": "What is 2+2?"} - - def test_max_output_tokens_becomes_max_completion_tokens(self): - body = { - "model": "gpt-4", - "input": "Hi", - "max_output_tokens": 1024, - } - result = _responses_request_to_chat(body) - - assert result["max_completion_tokens"] == 1024 - assert "max_tokens" not in result - assert "max_output_tokens" not in result - - def test_tools_conversion(self): - body = { - "model": "gpt-4", - "input": "Get weather", - "tools": [ - { - "type": "function", - "name": "get_weather", - "description": "Get the weather", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string"}, - }, - "required": ["location"], - }, - } - ], - } - result = _responses_request_to_chat(body) - - assert len(result["tools"]) == 1 - tool = result["tools"][0] - assert tool["type"] == "function" - assert tool["function"]["name"] == "get_weather" - assert tool["function"]["description"] == "Get the weather" - assert tool["function"]["parameters"]["type"] == "object" - - def test_tools_preserve_strict_and_nested_chat_shape(self): - body = { - "model": "gpt-4", - "input": "Get weather", - "tools": [ - { - "type": "function", - "function": { - "name": "lookup", - "description": "Lookup data", - "parameters": {"type": "object"}, - "strict": True, - }, - } - ], - } - result = _responses_request_to_chat(body) - - assert result["tools"] == [ - { - "type": "function", - "function": { - "name": "lookup", - "description": "Lookup data", - "parameters": {"type": "object"}, - "strict": True, - }, - } - ] - - def test_codex_tools_format(self): - """Codex CLI sends tools with id/inputSchema instead of name/parameters.""" - body = { - "model": "gpt-4", - "input": "List files", - "tools": [ - { - "id": "exec_command", - "description": "Runs a command in a PTY.", - "inputSchema": { - "jsonSchema": { - "type": "object", - "properties": { - "cmd": {"type": "string", "description": "Shell command"}, - }, - "required": ["cmd"], - "additionalProperties": False, - } - }, - }, - { - "id": "view_image", - "description": "View a local image.", - "inputSchema": { - "jsonSchema": { - "type": "object", - "properties": { - "path": {"type": "string"}, - }, - "required": ["path"], - } - }, - }, - ], - } - result = _responses_request_to_chat(body) - - assert len(result["tools"]) == 2 - tool0 = result["tools"][0] - assert tool0["type"] == "function" - assert tool0["function"]["name"] == "exec_command" - assert tool0["function"]["description"] == "Runs a command in a PTY." - assert tool0["function"]["parameters"]["type"] == "object" - assert "cmd" in tool0["function"]["parameters"]["properties"] - - tool1 = result["tools"][1] - assert tool1["function"]["name"] == "view_image" - assert tool1["function"]["parameters"]["required"] == ["path"] - - def test_empty_tools_filtered(self): - """Ghost tools with empty id/name should be filtered out.""" - body = { - "model": "gpt-4", - "input": "Hi", - "tools": [ - { - "id": "exec_command", - "description": "Run a command", - "inputSchema": {"jsonSchema": {"type": "object", "properties": {}}}, - }, - { - "id": "", - "description": "", - "inputSchema": {"jsonSchema": {}}, - }, - ], - } - result = _responses_request_to_chat(body) - - assert len(result["tools"]) == 1 - assert result["tools"][0]["function"]["name"] == "exec_command" - - def test_tool_choice_dropped_when_no_tools(self): - # tool_choice without tools is invalid for OpenAI — must not be forwarded - body = { - "model": "gpt-4", - "input": "Hi", - "tool_choice": "required", - } - result = _responses_request_to_chat(body) - assert "tool_choice" not in result - - def test_function_tool_choice_maps_to_chat_shape(self): - body = { - "model": "gpt-4", - "input": "Hi", - "tools": [{"type": "function", "name": "lookup", "parameters": {}}], - "tool_choice": {"type": "function", "name": "lookup"}, - } - result = _responses_request_to_chat(body) - assert result["tool_choice"] == { - "type": "function", - "function": {"name": "lookup"}, - } - - def test_unsupported_hosted_tools_and_tool_choice_are_dropped(self): - body = { - "model": "gpt-4", - "input": "Search the web", - "tools": [{"type": "web_search_preview"}], - "tool_choice": {"type": "web_search_preview"}, - } - result = _responses_request_to_chat(body) - assert "tools" not in result - assert "tool_choice" not in result - - def test_passthrough_params(self): - body = { - "model": "gpt-4", - "input": "Hi", - "temperature": 0.7, - "top_p": 0.9, - "stream": True, - "parallel_tool_calls": False, - "metadata": {"trace": "abc"}, - "store": False, - "stream_options": {"include_usage": True}, - "prompt_cache_key": "session-1", - "service_tier": "flex", - "user": "u-123", - } - result = _responses_request_to_chat(body) - - assert result["temperature"] == 0.7 - assert result["top_p"] == 0.9 - assert result["stream"] is True - assert result["parallel_tool_calls"] is False - assert result["metadata"] == {"trace": "abc"} - assert result["store"] is False - assert result["stream_options"] == {"include_usage": True} - assert result["prompt_cache_key"] == "session-1" - assert result["service_tier"] == "flex" - assert result["user"] == "u-123" - - def test_reasoning_and_text_format_map_to_chat_fields(self): - body = { - "model": "gpt-5", - "input": "Return JSON", - "reasoning": {"effort": "high"}, - "text": { - "format": { - "type": "json_schema", - "name": "answer", - "schema": {"type": "object"}, - "strict": True, - } - }, - } - result = _responses_request_to_chat(body) - assert result["reasoning_effort"] == "high" - assert result["response_format"] == { - "type": "json_schema", - "json_schema": { - "name": "answer", - "schema": {"type": "object"}, - "strict": True, - }, - } - - @pytest.mark.parametrize( - ("field", "value"), - [ - ("previous_response_id", "resp_123"), - ("conversation", "conv_123"), - ("conversation", {"id": "conv_123"}), - ], - ) - def test_stateful_responses_fields_are_forgotten(self, field, value): - body = { - "model": "gpt-4", - "input": "Continue", - field: value, - } - result = _responses_request_to_chat(body) - - assert result["messages"] == [{"role": "user", "content": "Continue"}] - assert result["model"] == "gpt-4" - assert field not in result - - def test_multi_turn_input_with_messages(self): - body = { - "model": "gpt-4", - "input": [ - {"type": "message", "role": "user", "content": "Hello"}, - { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": "Hi there!"}], - }, - {"type": "message", "role": "user", "content": "How are you?"}, - ], - } - result = _responses_request_to_chat(body) - - assert len(result["messages"]) == 3 - assert result["messages"][0] == {"role": "user", "content": "Hello"} - assert result["messages"][1] == {"role": "assistant", "content": "Hi there!"} - assert result["messages"][2] == {"role": "user", "content": "How are you?"} - - def test_multi_turn_with_function_calls_and_outputs(self): - body = { - "model": "gpt-4", - "input": [ - {"type": "message", "role": "user", "content": "Get weather in SF and NY"}, - { - "type": "function_call", - "name": "get_weather", - "call_id": "call_abc", - "arguments": '{"location": "SF"}', - }, - { - "type": "function_call", - "name": "get_weather", - "call_id": "call_def", - "arguments": '{"location": "NY"}', - }, - { - "type": "function_call_output", - "call_id": "call_abc", - "output": "72F sunny", - }, - { - "type": "function_call_output", - "call_id": "call_def", - "output": "65F cloudy", - }, - ], - } - result = _responses_request_to_chat(body) - - msgs = result["messages"] - assert msgs[0] == {"role": "user", "content": "Get weather in SF and NY"} - - # Consecutive function_calls should be merged into one assistant message - assert msgs[1]["role"] == "assistant" - assert msgs[1]["content"] is None - assert len(msgs[1]["tool_calls"]) == 2 - assert msgs[1]["tool_calls"][0]["id"] == "call_abc" - assert msgs[1]["tool_calls"][0]["function"]["name"] == "get_weather" - assert msgs[1]["tool_calls"][1]["id"] == "call_def" - - # function_call_output -> tool messages - assert msgs[2]["role"] == "tool" - assert msgs[2]["tool_call_id"] == "call_abc" - assert msgs[2]["content"] == "72F sunny" - - assert msgs[3]["role"] == "tool" - assert msgs[3]["tool_call_id"] == "call_def" - assert msgs[3]["content"] == "65F cloudy" - - def test_interleaved_function_calls_produce_separate_turns(self): - """Sequential function_call/output pairs represent separate turns. - - Codex sends interleaved pairs (call A, output A, call B, output B) - when each call was a separate LLM turn. The transition from - function_call_output -> function_call marks a turn boundary. - Each turn should produce its own assistant message. - """ - body = { - "model": "gpt-4", - "input": [ - {"type": "message", "role": "user", "content": "Explore the repo"}, - { - "type": "function_call", - "name": "exec_command", - "call_id": "call_1", - "arguments": '{"cmd": "ls -la"}', - }, - { - "type": "function_call_output", - "call_id": "call_1", - "output": "file1.py file2.py", - }, - { - "type": "function_call", - "name": "str_replace_editor", - "call_id": "call_2", - "arguments": '{"filename": "AGENTS.md", "command": "view"}', - }, - { - "type": "function_call_output", - "call_id": "call_2", - "output": "# Agents\n...", - }, - { - "type": "function_call", - "name": "exec_command", - "call_id": "call_3", - "arguments": '{"cmd": "cat README.md"}', - }, - { - "type": "function_call_output", - "call_id": "call_3", - "output": "# README", - }, - ], - } - result = _responses_request_to_chat(body) - msgs = result["messages"] - - # user message - assert msgs[0] == {"role": "user", "content": "Explore the repo"} - - # Turn 1: call_1 - assert msgs[1]["role"] == "assistant" - assert msgs[1]["content"] is None - assert len(msgs[1]["tool_calls"]) == 1 - assert msgs[1]["tool_calls"][0]["id"] == "call_1" - assert msgs[1]["tool_calls"][0]["function"]["name"] == "exec_command" - assert msgs[2] == {"role": "tool", "tool_call_id": "call_1", "content": "file1.py file2.py"} - - # Turn 2: call_2 - assert msgs[3]["role"] == "assistant" - assert len(msgs[3]["tool_calls"]) == 1 - assert msgs[3]["tool_calls"][0]["id"] == "call_2" - assert msgs[3]["tool_calls"][0]["function"]["name"] == "str_replace_editor" - assert msgs[4] == {"role": "tool", "tool_call_id": "call_2", "content": "# Agents\n..."} - - # Turn 3: call_3 - assert msgs[5]["role"] == "assistant" - assert len(msgs[5]["tool_calls"]) == 1 - assert msgs[5]["tool_calls"][0]["id"] == "call_3" - assert msgs[5]["tool_calls"][0]["function"]["name"] == "exec_command" - assert msgs[6] == {"role": "tool", "tool_call_id": "call_3", "content": "# README"} - - # Total: 1 user + 3 * (assistant + tool) = 7 messages - assert len(msgs) == 7 - - def test_tool_blocks_separated_by_message(self): - """Tool blocks separated by a message should produce separate assistant messages.""" - body = { - "model": "gpt-4", - "input": [ - {"type": "message", "role": "user", "content": "Do task"}, - { - "type": "function_call", - "name": "tool_a", - "call_id": "call_a", - "arguments": "{}", - }, - { - "type": "function_call_output", - "call_id": "call_a", - "output": "result_a", - }, - # Assistant text response separates the two blocks - { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": "I found something."}], - }, - { - "type": "function_call", - "name": "tool_b", - "call_id": "call_b", - "arguments": "{}", - }, - { - "type": "function_call_output", - "call_id": "call_b", - "output": "result_b", - }, - ], - } - result = _responses_request_to_chat(body) - msgs = result["messages"] - - # user, assistant(tool_a), tool_a, assistant_text, assistant(tool_b), tool_b - assert msgs[0]["role"] == "user" - assert msgs[1]["role"] == "assistant" - assert len(msgs[1]["tool_calls"]) == 1 - assert msgs[1]["tool_calls"][0]["id"] == "call_a" - assert msgs[2]["role"] == "tool" - assert msgs[3] == {"role": "assistant", "content": "I found something."} - assert msgs[4]["role"] == "assistant" - assert len(msgs[4]["tool_calls"]) == 1 - assert msgs[4]["tool_calls"][0]["id"] == "call_b" - assert msgs[5]["role"] == "tool" - assert len(msgs) == 6 - - def test_message_between_function_call_and_output_is_deferred(self): - """Codex can inject warnings between a function_call and its output. - - Chat/Anthropic-compatible histories need the tool result immediately - after the assistant tool call, so preserve the warning as later context. - """ - body = { - "model": "gpt-4", - "input": [ - {"type": "message", "role": "user", "content": "Do task"}, - { - "type": "function_call", - "name": "shell", - "call_id": "tooluse_1", - "arguments": '{"command":["apply_patch","..."]}', - }, - { - "type": "message", - "role": "user", - "content": [ - { - "type": "input_text", - "text": ( - "Warning: apply_patch was requested via shell. " - "Use the apply_patch tool instead." - ), - } - ], - }, - { - "type": "function_call_output", - "call_id": "tooluse_1", - "output": "Success", - }, - ], - } - result = _responses_request_to_chat(body) - - assert result["messages"] == [ - {"role": "user", "content": "Do task"}, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "tooluse_1", - "type": "function", - "function": { - "name": "shell", - "arguments": '{"command":["apply_patch","..."]}', - }, - } - ], - }, - {"role": "tool", "tool_call_id": "tooluse_1", "content": "Success"}, - { - "role": "user", - "content": ( - "Warning: apply_patch was requested via shell. " - "Use the apply_patch tool instead." - ), - }, - ] - - def test_content_blocks_flattened(self): - """Content blocks with input_text/output_text types should be flattened.""" - body = { - "model": "gpt-4", - "input": [ - { - "type": "message", - "role": "user", - "content": [ - {"type": "input_text", "text": "Hello "}, - {"type": "input_text", "text": "world"}, - ], - }, - ], - } - result = _responses_request_to_chat(body) - assert result["messages"][0]["content"] == "Hello \nworld" - - def test_multimodal_content_blocks_preserved_for_chat(self): - body = { - "model": "gpt-4o", - "input": [ - { - "type": "message", - "role": "user", - "content": [ - {"type": "input_text", "text": "Describe this"}, - { - "type": "input_image", - "image_url": "https://example.test/cat.png", - "detail": "low", - }, - ], - }, - ], - } - result = _responses_request_to_chat(body) - - assert result["messages"][0]["content"] == [ - {"type": "text", "text": "Describe this"}, - { - "type": "image_url", - "image_url": { - "url": "https://example.test/cat.png", - "detail": "low", - }, - }, - ] - - def test_orphan_function_call_output_becomes_user_context(self): - body = { - "model": "gpt-4", - "input": [ - { - "type": "function_call_output", - "call_id": "call_orphan", - "output": "result", - }, - ], - } - result = _responses_request_to_chat(body) - - assert result["messages"] == [ - {"role": "user", "content": "Tool result call_orphan: result"} - ] - - def test_non_json_tool_payloads_fall_back_to_text(self): - recursive: list[object] = [] - recursive.append(recursive) - body = { - "model": "gpt-4", - "input": [ - { - "type": "function_call", - "name": "lookup", - "call_id": "call_recursive", - "arguments": recursive, - }, - { - "type": "function_call_output", - "call_id": "call_recursive", - "output": recursive, - }, - ], - } - result = _responses_request_to_chat(body) - - messages = result["messages"] - assert messages[0]["tool_calls"][0]["function"]["arguments"] == "[[...]]" - assert messages[1]["content"] == "[[...]]" - - def test_codex_reasoning_items_do_not_become_empty_assistant_messages(self): - """Codex replays each model turn as message + reasoning + function_call items. - - The reasoning item must ride with the turn's tool-call message instead of - surfacing as a fabricated ``{"role": "assistant", "content": ""}`` chat - message; those empty turns accumulate every round-trip and degrade the - upstream model until it stops emitting tool calls. - """ - body = { - "model": "big-reasoner", - "instructions": "You are a coding agent.", - "input": [ - { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "Fix the broken pip install."}], - }, - { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": "\n\n"}], - }, - { - "type": "reasoning", - "summary": [], - "content": [ - {"type": "reasoning_text", "text": "Let me check the python setup."} - ], - "encrypted_content": None, - }, - { - "type": "function_call", - "name": "shell_command", - "arguments": '{"command":"pip3 --version","workdir":"/app"}', - "call_id": "call-1", - }, - { - "type": "function_call_output", - "call_id": "call-1", - "output": "ModuleNotFoundError: No module named 'pip'", - }, - { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": "\n\n"}], - }, - { - "type": "reasoning", - "summary": [], - "content": [{"type": "reasoning_text", "text": "pip is missing; ensurepip."}], - "encrypted_content": None, - }, - { - "type": "function_call", - "name": "shell_command", - "arguments": '{"command":"python3 -m ensurepip","workdir":"/app"}', - "call_id": "call-2", - }, - { - "type": "function_call_output", - "call_id": "call-2", - "output": "Successfully installed pip", - }, - ], - } - result = _responses_request_to_chat(body) - - assert result["messages"] == [ - {"role": "system", "content": "You are a coding agent."}, - {"role": "user", "content": "Fix the broken pip install."}, - {"role": "assistant", "content": "\n\n"}, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call-1", - "type": "function", - "function": { - "name": "shell_command", - "arguments": '{"command":"pip3 --version","workdir":"/app"}', - }, - } - ], - }, - { - "role": "tool", - "tool_call_id": "call-1", - "content": "ModuleNotFoundError: No module named 'pip'", - }, - {"role": "assistant", "content": "\n\n"}, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call-2", - "type": "function", - "function": { - "name": "shell_command", - "arguments": '{"command":"python3 -m ensurepip","workdir":"/app"}', - }, - } - ], - }, - { - "role": "tool", - "tool_call_id": "call-2", - "content": "Successfully installed pip", - }, - ] - - def test_reasoning_item_merges_into_following_assistant_message(self): - """Native item ordering puts reasoning before the assistant message it belongs to.""" - body = { - "model": "gpt-5", - "input": [ - {"type": "message", "role": "user", "content": "Check the file"}, - { - "type": "reasoning", - "summary": [{"type": "summary_text", "text": "Reading it now."}], - }, - { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": "Let me check."}], - }, - { - "type": "function_call", - "name": "read_file", - "call_id": "call-r", - "arguments": '{"path":"a.py"}', - }, - {"type": "function_call_output", "call_id": "call-r", "output": "print(1)"}, - ], - } - result = _responses_request_to_chat(body) - - assert result["messages"] == [ - {"role": "user", "content": "Check the file"}, - {"role": "assistant", "content": "Let me check."}, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call-r", - "type": "function", - "function": {"name": "read_file", "arguments": '{"path":"a.py"}'}, - } - ], - }, - {"role": "tool", "tool_call_id": "call-r", "content": "print(1)"}, - ] - - def test_reasoning_only_turn_does_not_add_empty_assistant_message(self): - """A reasoning item directly before a tool call must not split the turn.""" - body = { - "model": "gpt-5", - "input": [ - {"type": "message", "role": "user", "content": "List files"}, - { - "type": "reasoning", - "summary": [], - "content": [{"type": "reasoning_text", "text": "Simple ls."}], - }, - { - "type": "function_call", - "name": "shell", - "call_id": "call-ls", - "arguments": '{"command":"ls"}', - }, - {"type": "function_call_output", "call_id": "call-ls", "output": "a.py"}, - ], - } - result = _responses_request_to_chat(body) - - empty_assistants = [ - message - for message in result["messages"] - if message.get("role") == "assistant" and message.get("content") == "" - ] - assert empty_assistants == [] - assert result["messages"][1]["tool_calls"][0]["id"] == "call-ls" - - -# --------------------------------------------------------------------------- -# Response conversion tests -# --------------------------------------------------------------------------- - - -class TestConvertChatResponseToResponses: - """Tests for convert_chat_response_to_responses.""" - - def test_text_response(self): - response = { - "id": "chatcmpl-123", - "model": "gpt-4", - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": "Hello!"}, - "finish_reason": "stop", - } - ], - "usage": { - "prompt_tokens": 10, - "completion_tokens": 5, - "total_tokens": 15, - }, - } - result = _chat_response_to_responses(response) - - assert result["id"] == "chatcmpl-123" - assert result["object"] == "response" - assert result["status"] == "completed" - assert result["model"] == "gpt-4" - - assert len(result["output"]) == 1 - out = result["output"][0] - assert out["type"] == "message" - assert out["role"] == "assistant" - assert out["content"][0]["type"] == "output_text" - assert out["content"][0]["text"] == "Hello!" - - def test_tool_calls_response(self): - response = { - "id": "chatcmpl-456", - "model": "gpt-4", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_abc", - "type": "function", - "function": { - "name": "get_weather", - "arguments": '{"location": "SF"}', - }, - }, - { - "id": "call_def", - "type": "function", - "function": { - "name": "get_weather", - "arguments": '{"location": "NY"}', - }, - }, - ], - }, - "finish_reason": "tool_calls", - } - ], - "usage": { - "prompt_tokens": 20, - "completion_tokens": 30, - "total_tokens": 50, - }, - } - result = _chat_response_to_responses(response) - - assert result["status"] == "completed" - # No text message, just function calls - assert len(result["output"]) == 2 - assert result["output"][0]["type"] == "function_call" - assert result["output"][0]["call_id"] == "call_abc" - assert result["output"][0]["name"] == "get_weather" - assert result["output"][0]["arguments"] == '{"location": "SF"}' - - assert result["output"][1]["type"] == "function_call" - assert result["output"][1]["call_id"] == "call_def" - - def test_usage_mapping(self): - response = { - "id": "chatcmpl-789", - "model": "gpt-4", - "choices": [ - { - "message": {"role": "assistant", "content": "Hi"}, - "finish_reason": "stop", - } - ], - "usage": { - "prompt_tokens": 100, - "completion_tokens": 50, - "total_tokens": 150, - "completion_tokens_details": {"reasoning_tokens": 12}, - }, - } - result = _chat_response_to_responses(response) - - assert result["usage"]["input_tokens"] == 100 - assert result["usage"]["output_tokens"] == 50 - assert result["usage"]["total_tokens"] == 150 - assert result["usage"]["output_tokens_details"] == {"reasoning_tokens": 12} - - def test_response_object_with_model_dump(self): - """Test with a response object that has model_dump (like litellm responses).""" - mock_response = MagicMock() - mock_response.model_dump.return_value = { - "id": "chatcmpl-obj", - "model": "gpt-4", - "choices": [ - { - "message": {"role": "assistant", "content": "From object"}, - "finish_reason": "stop", - } - ], - "usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8}, - } - - result = _chat_response_to_responses(mock_response) - assert result["output"][0]["content"][0]["text"] == "From object" - - def test_mixed_text_and_tool_calls(self): - response = { - "id": "chatcmpl-mixed", - "model": "gpt-4", - "choices": [ - { - "message": { - "role": "assistant", - "content": "Let me check that.", - "tool_calls": [ - { - "id": "call_123", - "type": "function", - "function": { - "name": "lookup", - "arguments": '{"query": "test"}', - }, - } - ], - }, - "finish_reason": "tool_calls", - } - ], - "usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, - } - result = _chat_response_to_responses(response) - - assert len(result["output"]) == 2 - assert result["output"][0]["type"] == "message" - assert result["output"][0]["content"][0]["text"] == "Let me check that." - assert result["output"][1]["type"] == "function_call" - assert result["output"][1]["name"] == "lookup" - - def test_reasoning_content_with_tool_calls_has_no_empty_message_item(self): - """Reasoning + tool calls with empty text must not fabricate an empty message item.""" - response = { - "id": "chatcmpl-reason", - "model": "big-reasoner", - "choices": [ - { - "message": { - "role": "assistant", - "content": None, - "reasoning_content": "Inspect the repository first.", - "tool_calls": [ - { - "id": "call-9", - "type": "function", - "function": { - "name": "shell", - "arguments": '{"command":"ls"}', - }, - } - ], - }, - "finish_reason": "tool_calls", - } - ], - "usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, - } - result = _chat_response_to_responses(response) - - assert [item["type"] for item in result["output"]] == ["reasoning", "function_call"] - reasoning = result["output"][0] - assert reasoning["content"] == [ - {"type": "reasoning_text", "text": "Inspect the repository first."} - ] - function_call = result["output"][1] - assert function_call["call_id"] == "call-9" - assert function_call["name"] == "shell" - assert function_call["arguments"] == '{"command": "ls"}' - - -# --------------------------------------------------------------------------- -# Streaming conversion tests (chat chunks -> Responses SSE events) -# --------------------------------------------------------------------------- - - -def _chat_chunk(delta: dict, finish: str | None = None) -> dict: - return { - "id": "chatcmpl-stream", - "object": "chat.completion.chunk", - "model": "big-reasoner", - "choices": [{"index": 0, "delta": delta, "finish_reason": finish}], - } - - -async def _translate_chat_stream_to_responses(chunks: list[dict]) -> list[dict]: - async def _source(): - for chunk in chunks: - yield chunk - - return [ - event - async for event in ENGINE.translate_stream("openai_chat", "openai_responses", _source()) - ] - - -class TestChatStreamToResponsesSse: - """Pins the SSE contract codex consumes for streamed tool calls.""" - - async def test_tool_call_stream_produces_well_formed_function_call_events(self): - events = await _translate_chat_stream_to_responses( - [ - _chat_chunk({"role": "assistant", "content": ""}), - _chat_chunk({"reasoning_content": "Need to inspect the repo."}), - _chat_chunk({"content": "\n\n"}), - _chat_chunk( - { - "tool_calls": [ - { - "index": 0, - "id": "call-9", - "type": "function", - "function": {"name": "shell", "arguments": '{"command":'}, - } - ] - } - ), - _chat_chunk( - {"tool_calls": [{"index": 0, "function": {"arguments": '"ls"}'}}]} - ), - _chat_chunk({}, finish="tool_calls"), - ] - ) - - added = [ - event["item"] - for event in events - if event["type"] == "response.output_item.added" - and event["item"]["type"] == "function_call" - ] - assert len(added) == 1 - assert added[0]["call_id"] == "call-9" - assert added[0]["name"] == "shell" - - argument_deltas = [ - event["delta"] - for event in events - if event["type"] == "response.function_call_arguments.delta" - ] - assert "".join(argument_deltas) == '{"command":"ls"}' - - done = [ - event["item"] - for event in events - if event["type"] == "response.output_item.done" - and event["item"]["type"] == "function_call" - ] - assert len(done) == 1 - assert done[0]["call_id"] == "call-9" - assert done[0]["name"] == "shell" - assert done[0]["arguments"] == '{"command":"ls"}' - assert done[0]["status"] == "completed" - - completed = [event for event in events if event["type"] == "response.completed"] - assert len(completed) == 1 - output_types = [item["type"] for item in completed[0]["response"]["output"]] - assert output_types == ["reasoning", "message", "function_call"] - function_item = completed[0]["response"]["output"][2] - assert function_item["call_id"] == "call-9" - assert function_item["arguments"] == '{"command":"ls"}' diff --git a/tests/test_rl_logging.py b/tests/test_rl_logging.py deleted file mode 100644 index dffe157e2..000000000 --- a/tests/test_rl_logging.py +++ /dev/null @@ -1,336 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for local RL trace logging (`--enable-rl-logging`).""" - -from __future__ import annotations - -import argparse -import json -from collections.abc import AsyncIterator -from pathlib import Path - -import pytest -from openai.types.chat import ChatCompletionChunk -from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice -from openai.types.chat.chat_completion_chunk import ChoiceDelta -from openai.types.completion_usage import CompletionUsage - -from switchyard.cli.switchyard_cli import _build_parser -from switchyard.lib.chat_response import ResponseStream -from switchyard.lib.processors.rl_logging_request_processor import ( - CTX_RL_LOGGING_REQUEST, - RlLoggingRequestProcessor, -) -from switchyard.lib.processors.rl_logging_response_processor import ( - RlLoggingResponseProcessor, - build_rl_logging_processors, -) -from switchyard.server.server_util import resolve_rl_log_dir -from switchyard_rust.core import ChatRequest, ChatResponse, ProxyContext - - -def _request() -> ChatRequest: - return ChatRequest.openai_chat({ - "model": "gpt-test", - "messages": [ - {"role": "system", "content": "be brief"}, - {"role": "user", "content": "hello"}, - ], - "tools": [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Look up weather", - "parameters": {"type": "object", "properties": {}}, - }, - }, - ], - "tool_choice": "auto", - }) - - -def _completion(*, content: str | None = "hi there", tool_calls: list | None = None, - choices: list | None = None) -> ChatResponse: - message: dict = {"role": "assistant"} - if content is not None: - message["content"] = content - if tool_calls is not None: - message["tool_calls"] = tool_calls - if choices is None: - choices = [{"index": 0, "message": message, "finish_reason": "stop"}] - return ChatResponse.openai_completion({ - "id": "chatcmpl-test", - "object": "chat.completion", - "created": 1700000000, - "model": "gpt-test", - "choices": choices, - "usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8}, - }) - - -def _read_only_trace(log_dir: Path) -> dict: - files = list(log_dir.glob("*.json")) - assert len(files) == 1, f"expected exactly one trace file, got {files}" - return json.loads(files[0].read_text()) - - -async def _run(log_dir: Path, response: ChatResponse, *, snapshot: bool = True) -> ProxyContext: - ctx = ProxyContext() - if snapshot: - await RlLoggingRequestProcessor().process(ctx, _request()) - await RlLoggingResponseProcessor(log_dir).process(ctx, response) - return ctx - - -async def test_request_processor_snapshots_openai_body() -> None: - ctx = ProxyContext() - await RlLoggingRequestProcessor().process(ctx, _request()) - snapshot = ctx.metadata[CTX_RL_LOGGING_REQUEST] - assert isinstance(snapshot, dict) - assert [m["role"] for m in snapshot["messages"]] == ["system", "user"] - - -async def test_non_streaming_writes_message_history_trace(tmp_path: Path) -> None: - await _run(tmp_path, _completion(content="hi there")) - entry = _read_only_trace(tmp_path) - - assert entry["is_valid"] is True - assert entry["tool_choice"] == "auto" - assert entry["token_count"] == { - "prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8, - } - # Request history + appended assistant turn. - assert [m["role"] for m in entry["messages"]] == ["system", "user", "assistant"] - assert entry["messages"][-1]["content"] == "hi there" - # Tools rewritten to the message_history shape. - assert entry["tools"] == [{ - "id": "get_weather", - "description": "Look up weather", - "inputSchema": {"jsonSchema": {"type": "object", "properties": {}}}, - }] - - -async def test_assistant_tool_calls_are_logged(tmp_path: Path) -> None: - tool_calls = [{ - "id": "call_1", - "type": "function", - "function": {"name": "get_weather", "arguments": "{}"}, - }] - await _run(tmp_path, _completion(content=None, tool_calls=tool_calls)) - entry = _read_only_trace(tmp_path) - assistant = entry["messages"][-1] - assert assistant["role"] == "assistant" - assert assistant["tool_calls"] == tool_calls - assert "content" not in assistant - - -async def test_empty_string_assistant_content_is_preserved(tmp_path: Path) -> None: - """An empty-string completion is valid content and must not be dropped.""" - await _run(tmp_path, _completion(content="")) - entry = _read_only_trace(tmp_path) - assert entry["messages"][-1] == {"role": "assistant", "content": ""} - - -async def test_empty_choices_writes_nothing(tmp_path: Path) -> None: - await _run(tmp_path, _completion(choices=[])) - assert list(tmp_path.glob("*.json")) == [] - - -async def test_missing_request_snapshot_writes_nothing(tmp_path: Path) -> None: - await _run(tmp_path, _completion(), snapshot=False) - assert list(tmp_path.glob("*.json")) == [] - - -def _anthropic_completion(*, content: str, input_tokens: int, output_tokens: int) -> ChatResponse: - return ChatResponse.anthropic_completion({ - "id": "msg_test", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": content}], - "model": "claude-test", - "stop_reason": "end_turn", - "stop_sequence": None, - "usage": {"input_tokens": input_tokens, "output_tokens": output_tokens}, - }) - - -def _responses_completion(*, content: str, input_tokens: int, output_tokens: int) -> ChatResponse: - return ChatResponse.openai_responses_completion({ - "id": "resp_test", - "object": "response", - "created_at": 1700000000, - "status": "completed", - "model": "codex-test", - "output": [{ - "type": "message", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": content}], - }], - "parallel_tool_calls": False, - "tool_choice": "auto", - "tools": [], - "usage": { - "input_tokens": input_tokens, - "output_tokens": output_tokens, - "total_tokens": input_tokens + output_tokens, - }, - }) - - -async def test_anthropic_response_is_translated_and_logged(tmp_path: Path) -> None: - """claude's backend can answer in Anthropic format; it must translate to message_history.""" - await _run(tmp_path, _anthropic_completion(content="hello world", input_tokens=7, output_tokens=3)) - entry = _read_only_trace(tmp_path) - assert [m["role"] for m in entry["messages"]] == ["system", "user", "assistant"] - assert entry["messages"][-1]["content"] == "hello world" - assert entry["token_count"]["prompt_tokens"] == 7 - assert entry["token_count"]["completion_tokens"] == 3 - assert entry["is_valid"] is True - - -async def test_openai_responses_response_is_translated_and_logged(tmp_path: Path) -> None: - """codex talks the Responses API; a Responses completion must translate too.""" - await _run(tmp_path, _responses_completion(content="hello world", input_tokens=6, output_tokens=2)) - entry = _read_only_trace(tmp_path) - assert entry["messages"][-1] == {"role": "assistant", "content": "hello world"} - assert entry["token_count"]["prompt_tokens"] == 6 - assert entry["token_count"]["completion_tokens"] == 2 - assert entry["is_valid"] is True - - -async def test_writes_one_file_per_turn(tmp_path: Path) -> None: - """A reused processor writes one independent file per completed turn (no overwrite).""" - request_processor = RlLoggingRequestProcessor() - response_processor = RlLoggingResponseProcessor(tmp_path) - for i in range(3): - ctx = ProxyContext() - await request_processor.process(ctx, _request()) - await response_processor.process(ctx, _completion(content=f"turn {i}")) - - files = list(tmp_path.glob("*.json")) - assert len(files) == 3 # distinct filename per turn, nothing overwritten - last_contents = sorted( - json.loads(f.read_text())["messages"][-1]["content"] for f in files - ) - assert last_contents == ["turn 0", "turn 1", "turn 2"] - - -async def test_write_failure_does_not_break_the_response( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, -) -> None: - """A trace-write failure is swallowed — the proxy response must flow untouched.""" - processor = RlLoggingResponseProcessor(tmp_path) - ctx = ProxyContext() - await RlLoggingRequestProcessor().process(ctx, _request()) - - def _boom(_entry: dict) -> None: - raise OSError("disk full") - - monkeypatch.setattr(processor, "_write_entry", _boom) - response = _completion() - assert await processor.process(ctx, response) is response - assert list(tmp_path.glob("*.json")) == [] - - -def test_resolve_rl_log_dir() -> None: - off = argparse.Namespace(enable_rl_logging=False, rl_log_dir="./rl_data") - assert resolve_rl_log_dir(off) is None - - on = argparse.Namespace(enable_rl_logging=True, rl_log_dir="/tmp/traces") - assert resolve_rl_log_dir(on) == Path("/tmp/traces") - - -def test_build_rl_logging_processors() -> None: - assert build_rl_logging_processors(None) == ([], []) - - req, resp = build_rl_logging_processors(Path("/tmp/x")) - assert [type(p).__name__ for p in req] == ["RlLoggingRequestProcessor"] - assert [type(p).__name__ for p in resp] == ["RlLoggingResponseProcessor"] - - -async def test_streaming_logs_after_drain(tmp_path: Path) -> None: - """Streaming turns are captured on stream completion, not before.""" - ctx = ProxyContext() - await RlLoggingRequestProcessor().process(ctx, _request()) - - content_chunk = ChatCompletionChunk( - id="chatcmpl-test", object="chat.completion.chunk", created=1700000000, - model="gpt-test", - choices=[ChunkChoice(index=0, delta=ChoiceDelta(content="hi there"), finish_reason="stop")], - ) - usage_chunk = ChatCompletionChunk( - id="chatcmpl-test", object="chat.completion.chunk", created=1700000000, - model="gpt-test", choices=[], - usage=CompletionUsage(prompt_tokens=5, completion_tokens=3, total_tokens=8), - ) - - async def _iter() -> AsyncIterator[ChatCompletionChunk]: - yield content_chunk - yield usage_chunk - - response = ChatResponse.openai_stream(ResponseStream(_iter())) - out = await RlLoggingResponseProcessor(tmp_path).process(ctx, response) - # Nothing is written until the stream actually drains. - assert list(tmp_path.glob("*.json")) == [] - - forwarded = [chunk async for chunk in out.stream] - assert len(forwarded) == 2 # stream still forwards every chunk to the client - - entry = _read_only_trace(tmp_path) - assert entry["messages"][-1] == {"role": "assistant", "content": "hi there"} - assert entry["token_count"] == { - "prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8, - } - - -def test_rl_flags_are_scoped_to_serve() -> None: - parser = _build_parser() - args = parser.parse_args( - [ - "serve", - "--routes", - "routes.yaml", - "--enable-rl-logging", - "--rl-log-dir", - "/tmp/x", - ], - ) - assert args.command == "serve" - assert args.enable_rl_logging is True - assert resolve_rl_log_dir(args) == Path("/tmp/x") - - -def test_serve_attaches_rl_logging_processors(monkeypatch, tmp_path: Path) -> None: - """`serve --routes --enable-rl-logging` wires the trace logger into the chain.""" - import switchyard.cli.switchyard_cli as cli - - captured: dict[str, list] = {} - - class _FakeTable: - def registered_models(self) -> list[str]: - return ["m"] - - def default_model(self) -> str | None: - return None - - def _fake_load(routes, *, pre_routing_request_processors=(), - extra_response_processors=(), **_kwargs): - captured["request"] = list(pre_routing_request_processors) - captured["response"] = list(extra_response_processors) - return _FakeTable() - - monkeypatch.setattr(cli, "load_route_bundle_table", _fake_load) - monkeypatch.setattr(cli, "build_and_serve", lambda *a, **k: None) - - args = argparse.Namespace( - routes="routes.yaml", - enable_rl_logging=True, rl_log_dir=str(tmp_path), - routing_log_file=None, - ) - cli._cmd_serve(args) - - assert [type(p).__name__ for p in captured["request"]] == ["RlLoggingRequestProcessor"] - assert [type(p).__name__ for p in captured["response"]] == ["RlLoggingResponseProcessor"] diff --git a/tests/test_rl_logging_e2e.py b/tests/test_rl_logging_e2e.py deleted file mode 100644 index 4a44b988b..000000000 --- a/tests/test_rl_logging_e2e.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""In-process end-to-end test for ``--enable-rl-logging`` on the serve chain. - -Drives a real HTTP request through the production serve path — route-bundle -table → ``build_switchyard_app`` → endpoint → chain → backend — with the -RL-logging processors wired in exactly as ``switchyard serve ---enable-rl-logging`` wires them, and asserts a ``message_history`` trace file -is written. The upstream is a real loopback HTTP server (``_OpenAICompatStub``) -because the Rust backend uses ``reqwest``, which ``respx`` cannot intercept. - -No API key, no ``claude`` binary, no outbound network — safe for CI. -""" - -from __future__ import annotations - -import json -from pathlib import Path - -import httpx - -from switchyard.cli.route_bundle import build_route_bundle_table -from switchyard.lib.processors.rl_logging_response_processor import build_rl_logging_processors -from switchyard.server.switchyard_app import build_switchyard_app -from tests._chain_test_helpers import _backend_payload, _OpenAICompatStub, _sse_body, _stream_chunk - - -def _build_app(stub: _OpenAICompatStub, log_dir: Path): - """Build the serve chain pointed at ``stub`` with RL-logging attached. - - Mirrors ``_cmd_serve``: ``build_rl_logging_processors`` produces the paired - snapshot + writer processors, which feed the route-bundle table builder. - """ - rl_request, rl_response = build_rl_logging_processors(log_dir) - table = build_route_bundle_table( - { - "defaults": { - "api_key": "dummy", - "base_url": stub.base_url, - "format": "openai", - }, - "routes": { - "mock-model": {"type": "passthrough", "target": "mock-model"} - }, - }, - pre_routing_request_processors=rl_request, - extra_response_processors=rl_response, - ) - return build_switchyard_app(table) - - -def _client(app) -> httpx.AsyncClient: - return httpx.AsyncClient( - transport=httpx.ASGITransport(app=app, raise_app_exceptions=False), - base_url="http://test", - ) - - -def _only_trace(log_dir: Path) -> dict: - files = list(log_dir.glob("*.json")) - assert len(files) == 1, f"expected one trace file, got {files}" - return json.loads(files[0].read_text()) - - -async def test_serve_chain_writes_trace_non_streaming(tmp_path: Path) -> None: - with _OpenAICompatStub() as stub: - stub.respond_json(_backend_payload(content="hello world", model="mock-model")) - app = _build_app(stub, tmp_path) - async with _client(app) as client: - resp = await client.post( - "/v1/chat/completions", - json={"model": "mock-model", "messages": [{"role": "user", "content": "hi"}]}, - headers={"authorization": "Bearer test"}, - ) - assert resp.status_code == 200 - assert stub.called - - entry = _only_trace(tmp_path) - assert entry["is_valid"] is True - assert [m["role"] for m in entry["messages"]] == ["user", "assistant"] - assert entry["messages"][-1]["content"] == "hello world" - assert entry["token_count"] == { - "prompt_tokens": 8, "completion_tokens": 4, "total_tokens": 12, - } - - -async def test_serve_chain_writes_trace_streaming(tmp_path: Path) -> None: - usage_chunk = { - "id": "chatcmpl-backend-stream", - "object": "chat.completion.chunk", - "created": 1700000002, - "model": "mock-model", - "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 8, "completion_tokens": 4, "total_tokens": 12}, - } - with _OpenAICompatStub() as stub: - stub.respond_sse(_sse_body([_stream_chunk(content="hello world"), usage_chunk])) - app = _build_app(stub, tmp_path) - async with _client(app) as client: - # The trace is written when the stream drains, so the body must be - # fully consumed before asserting. - async with client.stream( - "POST", - "/v1/chat/completions", - json={ - "model": "mock-model", - "stream": True, - "messages": [{"role": "user", "content": "hi"}], - }, - headers={"authorization": "Bearer test"}, - ) as resp: - assert resp.status_code == 200 - async for _ in resp.aiter_bytes(): - pass - - entry = _only_trace(tmp_path) - assert entry["is_valid"] is True - assert [m["role"] for m in entry["messages"]] == ["user", "assistant"] - assert entry["messages"][-1]["content"] == "hello world" - assert entry["token_count"]["total_tokens"] == 12 diff --git a/tests/test_route_bundle.py b/tests/test_route_bundle.py deleted file mode 100644 index d8773c273..000000000 --- a/tests/test_route_bundle.py +++ /dev/null @@ -1,176 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for the minimal Python server route bundle.""" - -from __future__ import annotations - -import sys -from pathlib import Path - -import httpx -import pytest - -import switchyard.cli.switchyard_cli as cli -from switchyard.cli.launchers.launcher_runtime import route_bundle_strategy_summary -from switchyard.cli.route_bundle import RouteBundleConfigError, build_route_bundle_table -from switchyard.lib.route_table import RouteTable -from switchyard.server.switchyard_app import build_switchyard_app -from switchyard_rust.components import StatsLlmBackend - - -async def test_noop_route_returns_ok_without_an_upstream() -> None: - table = build_route_bundle_table({ - "routes": {"test/noop": {"type": "noop"}}, - }) - - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=build_switchyard_app(table)), - base_url="http://test", - ) as client: - response = await client.post( - "/v1/chat/completions", - json={ - "model": "test/noop", - "messages": [{"role": "user", "content": "hello"}], - }, - ) - - assert response.status_code == 200 - assert response.json()["choices"][0]["message"]["content"] == "OK" - - -@pytest.mark.parametrize( - ("path", "body", "expected"), - [ - ( - "/v1/messages", - { - "model": "test/noop", - "max_tokens": 16, - "messages": [{"role": "user", "content": "hello"}], - }, - ("content", 0, "text"), - ), - ( - "/v1/responses", - {"model": "test/noop", "input": "hello"}, - ("output", 0, "content"), - ), - ], -) -async def test_noop_route_translates_to_inbound_format( - path: str, - body: dict[str, object], - expected: tuple[str, int, str], -) -> None: - table = build_route_bundle_table({"routes": {"test/noop": {"type": "noop"}}}) - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=build_switchyard_app(table)), - base_url="http://test", - ) as client: - response = await client.post(path, json=body) - - assert response.status_code == 200 - value: object = response.json() - for part in expected: - value = value[part] # type: ignore[index] - assert value - - -def test_passthrough_route_builds_one_native_backend(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("UPSTREAM_KEY", "secret") - table = build_route_bundle_table({ - "defaults": { - "api_key": "${UPSTREAM_KEY}", - "base_url": "https://example.invalid/v1", - "format": "openai", - }, - "routes": { - "direct": { - "type": "passthrough", - "target": {"model": "upstream/model"}, - "display_name": "Direct model", - } - }, - }) - - assert table.registered_models() == ["direct"] - assert table.default_model() == "direct" - assert table.registered_model_entries()[0]["display_name"] == "Direct model" - components = table.lookup_switchyard("direct").iter_components() - stats_backend = next(component for component in components if isinstance(component, StatsLlmBackend)) - assert isinstance(stats_backend, StatsLlmBackend) - - -def test_passthrough_summary_labels_the_model(tmp_path: Path) -> None: - path = tmp_path / "routes.yaml" - path.write_text("routes:\n direct:\n type: passthrough\n target: upstream/model\n") - - assert route_bundle_strategy_summary(str(path), "direct") == ( - "passthrough: model=upstream/model" - ) - - -@pytest.mark.parametrize( - "bundle, match", - [ - ({}, "routes must be a mapping"), - ({"routes": {}}, "at least one route"), - ({"routes": {"r": {}}}, "missing string 'type'"), - ( - {"routes": {"r": {"type": "random"}}}, - "expected 'noop' or 'passthrough'", - ), - ( - {"routes": {"r": {"type": "noop", "target": "unused"}}}, - "unknown key", - ), - ( - {"routes": {"r": {"type": "passthrough"}}}, - "target must be a mapping", - ), - ], -) -def test_invalid_bundles_fail_closed(bundle: object, match: str) -> None: - with pytest.raises(RouteBundleConfigError, match=match): - build_route_bundle_table(bundle) - - -def test_missing_environment_variable_is_rejected() -> None: - with pytest.raises(RouteBundleConfigError, match="MISSING_ROUTE_KEY"): - build_route_bundle_table({ - "defaults": {"api_key": "${MISSING_ROUTE_KEY}"}, - "routes": {"direct": {"type": "passthrough", "target": "model"}}, - }) - - -def test_main_reports_missing_bundle_without_traceback( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - missing = tmp_path / "missing.yaml" - monkeypatch.setattr(sys, "argv", ["switchyard", "serve", "--routes", str(missing)]) - - with pytest.raises(SystemExit) as error: - cli.main() - - assert error.value.code == f"error: invalid route bundle: {missing}: file not found" - - -def test_serve_loads_noop_bundle( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - path = tmp_path / "routes.yaml" - path.write_text("routes:\n test/noop:\n type: noop\n") - captured: dict[str, object] = {} - - def fake_serve(args: object, switchyard: object, **kwargs: object) -> None: - captured.update(args=args, switchyard=switchyard, **kwargs) - - monkeypatch.setattr(cli, "build_and_serve", fake_serve) - parser = cli._build_parser() - args = parser.parse_args(["serve", "--routes", str(path)]) - args.func(args) - - assert isinstance(captured["switchyard"], RouteTable) - assert captured["switchyard"].registered_models() == ["test/noop"] diff --git a/tests/test_route_selection_headers.py b/tests/test_route_selection_headers.py deleted file mode 100644 index 4b1dabacd..000000000 --- a/tests/test_route_selection_headers.py +++ /dev/null @@ -1,233 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Route-selection spend-attribution response headers. - -For tokenomics reporting, a front proxy (e.g. LiteLLM) must be able to tie -its provider spend-log rows back to the Switchyard logical route that -selected them: the Switchyard response returns ``x-switchyard-*`` headers with -the successful attempt's selection plus a per-request correlation id, so the -parent spend-log row can be enriched to match the provider row. -""" - -from __future__ import annotations - -import json -from typing import Any - -from fastapi.responses import JSONResponse - -from switchyard.lib.endpoints.dispatch import serialize_chain_result -from switchyard.lib.endpoints.route_selection import ( - ROUTER_CORRELATION_ID_HEADER, - ROUTER_MODEL_HEADER, - SELECTED_MODEL_HEADER, - SELECTED_PROVIDER_HEADER, - route_selection_headers, -) -from switchyard.lib.endpoints.upstream_error import handle_chain_exception -from switchyard.lib.proxy_context import CTX_ROUTE_SELECTION, ProxyContext - -ROUTE_MODEL = "nvidia/switchyard/test-route" -ENDPOINT_ID = "openai/test-model" -UPSTREAM_MODEL = "openai/openai/test-model" - - -def _selection(**overrides: object) -> dict[str, object]: - """Recorded route selection, with *overrides* applied.""" - payload: dict[str, object] = { - "router_model": ROUTE_MODEL, - "router_selected_endpoint": ENDPOINT_ID, - "router_selected_model": UPSTREAM_MODEL, - "router_selected_provider": "openai", - "router_correlation_id": "11111111-2222-3333-4444-555555555555", - } - payload.update(overrides) - return payload - - -# --------------------------------------------------------------------------- -# Unit: ctx → response-header mapping -# --------------------------------------------------------------------------- - - -class TestRouteSelectionHeaders: - """Mapping of the ctx selection record to ``x-switchyard-*`` headers.""" - - def test_empty_without_selection(self) -> None: - """A ctx without a recorded selection yields no headers.""" - assert route_selection_headers(ProxyContext()) == {} - - def test_maps_selection_to_response_headers(self) -> None: - """Every exposed selection field maps to its response header.""" - ctx = ProxyContext() - ctx.metadata[CTX_ROUTE_SELECTION] = _selection() - - assert route_selection_headers(ctx) == { - ROUTER_MODEL_HEADER: ROUTE_MODEL, - SELECTED_MODEL_HEADER: UPSTREAM_MODEL, - SELECTED_PROVIDER_HEADER: "openai", - ROUTER_CORRELATION_ID_HEADER: "11111111-2222-3333-4444-555555555555", - } - - def test_skips_absent_fields_instead_of_stamping_placeholders(self) -> None: - """An absent field is skipped — never stamped as a placeholder value.""" - ctx = ProxyContext() - ctx.metadata[CTX_ROUTE_SELECTION] = _selection(router_model=None) - - headers = route_selection_headers(ctx) - - assert ROUTER_MODEL_HEADER not in headers - assert headers[SELECTED_MODEL_HEADER] == UPSTREAM_MODEL - - def test_ignores_non_mapping_value(self) -> None: - """A malformed (non-mapping) ctx record yields no headers.""" - ctx = ProxyContext() - ctx.metadata[CTX_ROUTE_SELECTION] = "bogus" - - assert route_selection_headers(ctx) == {} - - def test_skips_values_unsafe_as_header_material(self) -> None: - """The client-controlled router_model must be re-validated as a header. - - A CRLF-bearing value would be a response-splitting vector, and a - non-latin-1 value would crash Starlette response construction after - the upstream call already succeeded and was billed. - """ - ctx = ProxyContext() - ctx.metadata[CTX_ROUTE_SELECTION] = _selection(router_model="gpt\r\nx-evil: 1") - headers = route_selection_headers(ctx) - assert ROUTER_MODEL_HEADER not in headers - assert headers[SELECTED_MODEL_HEADER] == UPSTREAM_MODEL - - ctx = ProxyContext() - ctx.metadata[CTX_ROUTE_SELECTION] = _selection(router_model="gpt-4中文") - headers = route_selection_headers(ctx) - assert ROUTER_MODEL_HEADER not in headers - assert headers[ROUTER_CORRELATION_ID_HEADER] == ( - "11111111-2222-3333-4444-555555555555" - ) - - -class TestSerializeChainResultHeaders: - """``serialize_chain_result`` stamps the selection on every branch.""" - - @staticmethod - async def _sse_iter(_result: Any) -> Any: - """Minimal SSE iterator satisfying the serializer signature.""" - yield "data: {}\n\n" - - def test_json_response_carries_selection_headers(self) -> None: - """A JSON-serialized result carries the selection headers.""" - ctx = ProxyContext() - ctx.metadata[CTX_ROUTE_SELECTION] = _selection() - - response = serialize_chain_result( - {"ok": True}, stream=False, sse_iter=self._sse_iter, ctx=ctx - ) - - assert response.headers[SELECTED_MODEL_HEADER] == UPSTREAM_MODEL - assert ( - response.headers[ROUTER_CORRELATION_ID_HEADER] - == "11111111-2222-3333-4444-555555555555" - ) - - def test_streaming_response_carries_selection_headers(self) -> None: - """The SSE branch stamps the headers too. - - The backend call completes before the StreamingResponse is built, so - the selection is final by the time SSE headers are committed. - """ - class _EmptyStream: - """Async iterator yielding nothing, standing in for a backend stream.""" - - def __aiter__(self) -> _EmptyStream: - return self - - async def __anext__(self) -> str: - raise StopAsyncIteration - - ctx = ProxyContext() - ctx.metadata[CTX_ROUTE_SELECTION] = _selection() - - response = serialize_chain_result( - _EmptyStream(), stream=True, sse_iter=self._sse_iter, ctx=ctx - ) - - assert response.media_type == "text/event-stream" - assert response.headers[ROUTER_MODEL_HEADER] == ROUTE_MODEL - - def test_selection_free_ctx_no_selection_headers(self) -> None: - """A selection-free ctx adds no ``x-switchyard-*`` headers.""" - response = serialize_chain_result( - {"ok": True}, stream=False, sse_iter=self._sse_iter, ctx=ProxyContext() - ) - - assert ROUTER_CORRELATION_ID_HEADER not in response.headers - - def test_prebuilt_response_passes_through_with_selection_merged(self) -> None: - """A result that is already a ``Response`` gains the selection headers. - - No current chain path yields a pre-built Response after a billed - upstream success, but the serializer contract — a recorded selection - is never dropped — must hold on that branch too, with the response's - own status, body, and headers preserved. - """ - ctx = ProxyContext() - ctx.metadata[CTX_ROUTE_SELECTION] = _selection() - prebuilt = JSONResponse( - content={"ok": True}, status_code=404, headers={"x-existing": "1"} - ) - - response = serialize_chain_result( - prebuilt, stream=False, sse_iter=self._sse_iter, ctx=ctx - ) - - assert response is prebuilt - assert response.status_code == 404 - assert response.headers["x-existing"] == "1" - assert json.loads(response.body) == {"ok": True} - assert response.headers[SELECTED_MODEL_HEADER] == UPSTREAM_MODEL - assert response.headers[ROUTER_CORRELATION_ID_HEADER] == ( - "11111111-2222-3333-4444-555555555555" - ) - - -class TestErrorPathSelectionHeaders: - """``handle_chain_exception`` keeps a billed selection on error responses.""" - - def test_failure_after_billed_success_keeps_selection_headers(self) -> None: - """A post-backend failure must still expose the billed selection. - - The upstream call succeeded (provider spend-log row written with the - stamped correlation id) before e.g. response translation raised; the - error response must carry the selection headers or that provider row - becomes unjoinable. - """ - ctx = ProxyContext() - ctx.metadata[CTX_ROUTE_SELECTION] = _selection() - - response = handle_chain_exception( - RuntimeError("response translation failed"), - ctx, - inbound="openai", - log_msg="test", - ) - - assert response.status_code == 500 - assert response.headers[ROUTER_CORRELATION_ID_HEADER] == ( - "11111111-2222-3333-4444-555555555555" - ) - assert response.headers[SELECTED_MODEL_HEADER] == UPSTREAM_MODEL - - def test_failure_without_selection_has_no_selection_headers(self) -> None: - """No billed success → the error response claims no selection.""" - response = handle_chain_exception( - RuntimeError("backend never succeeded"), - ProxyContext(), - inbound="openai", - log_msg="test", - ) - - assert response.status_code == 500 - assert ROUTER_CORRELATION_ID_HEADER not in response.headers diff --git a/tests/test_route_table.py b/tests/test_route_table.py deleted file mode 100644 index e11b98f08..000000000 --- a/tests/test_route_table.py +++ /dev/null @@ -1,282 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for RouteTable model-based HTTP dispatch.""" - -from unittest.mock import AsyncMock, MagicMock - -import pytest -from fastapi import FastAPI -from fastapi.testclient import TestClient - -from switchyard.lib.endpoints.anthropic_messages_endpoint import ( - AnthropicMessagesEndpoint, -) -from switchyard.lib.endpoints.openai_chat_endpoint import OpenAIChatEndpoint -from switchyard.lib.endpoints.responses_endpoint import ResponsesEndpoint -from switchyard.lib.proxy_context import CTX_CALLER_API_KEY -from switchyard.lib.route_table import RouteTable -from switchyard.lib.switchyard import Switchyard -from switchyard.server.switchyard_app import build_switchyard_app - - -def _make_chain(name: str = "chain") -> MagicMock: - chain = MagicMock(spec=Switchyard) - chain.call = AsyncMock(return_value={"chain": name}) - chain.iter_components.return_value = [MagicMock(name=f"{name}-component")] - return chain - - -def _make_app(table: RouteTable) -> TestClient: - app = FastAPI() - app.state.switchyard = table - OpenAIChatEndpoint().register(app) - AnthropicMessagesEndpoint().register(app) - ResponsesEndpoint().register(app) - return TestClient(app, raise_server_exceptions=False) - - -class TestRouteTable: - def test_lookup_returns_registered_chain(self) -> None: - chain = _make_chain("specific") - table = RouteTable() - table.register("gpt-4o", chain) - - assert table.lookup_switchyard("gpt-4o") is chain - - def test_lookup_raises_key_error_for_unknown_model(self) -> None: - table = RouteTable() - table.register("gpt-4o", _make_chain()) - - with pytest.raises(KeyError): - table.lookup_switchyard("not-registered") - - def test_register_overwrites_existing_key(self) -> None: - first = _make_chain("first") - second = _make_chain("second") - table = RouteTable() - table.register("m", first) - table.register("m", second) - - assert table.lookup_switchyard("m") is second - - def test_registered_models_preserves_registration_order(self) -> None: - table = RouteTable() - table.register("first", _make_chain("first")) - table.register("second", _make_chain("second")) - - assert table.registered_models() == ["first", "second"] - - def test_explicit_default_can_differ_from_registration_order(self) -> None: - table = RouteTable() - table.register("first", _make_chain("first")) - table.register("second", _make_chain("second"), default=True) - - assert table.default_model() == "second" - - table.set_default_model("first") - assert table.default_model() == "first" - - with pytest.raises(KeyError): - table.set_default_model("missing") - - def test_state_key_matches_switchyard(self) -> None: - assert RouteTable.state_key == Switchyard.state_key == "switchyard" - - def test_iter_components_deduplicates_shared_instances(self) -> None: - shared = MagicMock(name="shared") - unique_a = MagicMock(name="unique-a") - unique_b = MagicMock(name="unique-b") - first = MagicMock(spec=Switchyard) - first.iter_components.return_value = [shared, unique_a] - second = MagicMock(spec=Switchyard) - second.iter_components.return_value = [shared, unique_b] - table = RouteTable() - table.register("first", first) - table.register("second", second) - - components = table.iter_components() - - assert components.count(shared) == 1 - assert unique_a in components - assert unique_b in components - assert len(components) == 3 - - -def test_build_switchyard_app_accepts_table() -> None: - table = RouteTable() - app = build_switchyard_app(table) - - assert app.state.switchyard is table - - -def test_models_endpoint_lists_registered_models() -> None: - table = RouteTable() - table.register( - "switchyard-default-random-12345678", - _make_chain("random"), - metadata={ - "display_name": "Switchyard random routing", - "description": "Random routes strong and weak.", - "switchyard": { - "algorithm": "random", - "strong_model": "strong/model", - "weak_model": "weak/model", - "strong_probability": 0.5, - }, - }, - ) - table.register("strong/model", _make_chain("strong")) - - with TestClient(build_switchyard_app(table), raise_server_exceptions=False) as client: - response = client.get("/v1/models?limit=1000") - - assert response.status_code == 200 - body = response.json() - assert body["object"] == "list" - assert body["has_more"] is False - assert body["default_model"] == "switchyard-default-random-12345678" - assert body["model_pool"] == [ - "switchyard-default-random-12345678", - "strong/model", - ] - assert [item["id"] for item in body["data"]] == [ - "switchyard-default-random-12345678", - "strong/model", - ] - assert body["data"][0]["display_name"] == "Switchyard random routing" - assert body["data"][0]["description"] == "Random routes strong and weak." - assert body["data"][0]["switchyard"]["algorithm"] == "random" - assert body["data"][0]["capabilities"]["streaming"] is True - assert body["data"][0]["capabilities"]["supported_inbound_formats"] == [ - "openai-chat-completions", - "openai-responses", - "anthropic-messages", - ] - - -def test_models_endpoint_uses_table_default_model() -> None: - table = RouteTable() - table.register("strong/model", _make_chain("strong")) - table.register("weak/model", _make_chain("weak")) - table.register("switchyard-route", _make_chain("random"), default=True) - - with TestClient(build_switchyard_app(table), raise_server_exceptions=False) as client: - response = client.get("/v1/models?limit=1000") - - assert response.status_code == 200 - body = response.json() - assert body["first_id"] == "strong/model" - assert body["default_model"] == "switchyard-route" - assert body["model_pool"] == [ - "strong/model", - "weak/model", - "switchyard-route", - ] - assert [item["id"] for item in body["data"]] == [ - "strong/model", - "weak/model", - "switchyard-route", - ] - - -@pytest.mark.parametrize( - ("path", "body"), - [ - ("/v1/chat/completions", {"model": "registered", "messages": [{"role": "user", "content": "hi"}]}), - ("/v1/messages", {"model": "registered", "max_tokens": 1, "messages": [{"role": "user", "content": "hi"}]}), - ("/v1/responses", {"model": "registered", "input": "hi"}), - ], -) -def test_http_dispatch_uses_registered_chain(path: str, body: dict[str, object]) -> None: - registered = _make_chain("registered") - table = RouteTable() - table.register("registered", registered) - - with _make_app(table) as client: - response = client.post(path, json=body) - - assert response.status_code == 200 - assert response.json() == {"chain": "registered"} - registered.call.assert_awaited_once() - - -@pytest.mark.parametrize( - ("path", "body"), - [ - ("/v1/chat/completions", {"model": "registered", "messages": [{"role": "user", "content": "hi"}]}), - ("/v1/messages", {"model": "registered", "max_tokens": 1, "messages": [{"role": "user", "content": "hi"}]}), - ("/v1/responses", {"model": "registered", "input": "hi"}), - ], -) -def test_http_dispatch_attaches_caller_api_key_to_context( - path: str, - body: dict[str, object], -) -> None: - registered = _make_chain("registered") - table = RouteTable() - table.register("registered", registered) - - with _make_app(table) as client: - response = client.post( - path, - json=body, - headers={"Authorization": "Bearer caller-key"}, - ) - - assert response.status_code == 200 - ctx = registered.call.await_args.kwargs["ctx"] - assert ctx.metadata[CTX_CALLER_API_KEY] == "caller-key" # pragma: allowlist secret - - -@pytest.mark.parametrize( - ("path", "body"), - [ - ("/v1/chat/completions", {"model": "unknown", "messages": [{"role": "user", "content": "hi"}]}), - ("/v1/messages", {"model": "unknown", "max_tokens": 1, "messages": [{"role": "user", "content": "hi"}]}), - ("/v1/responses", {"model": "unknown", "input": "hi"}), - ], -) -def test_http_dispatch_returns_404_for_unknown_model( - path: str, body: dict[str, object], -) -> None: - registered = _make_chain("registered") - table = RouteTable() - table.register("registered", registered) - - with _make_app(table) as client: - response = client.post(path, json=body) - - assert response.status_code == 404 - assert response.json() == { - "error": { - "message": "No route registered for model unknown", - "type": "model_not_found", - "code": "model_not_found", - } - } - registered.call.assert_not_awaited() - - -@pytest.mark.parametrize( - ("path", "body"), - [ - ("/v1/chat/completions", {"model": "missing", "messages": [{"role": "user", "content": "hi"}]}), - ("/v1/messages", {"model": "missing", "max_tokens": 1, "messages": [{"role": "user", "content": "hi"}]}), - ("/v1/responses", {"model": "missing", "input": "hi"}), - ], -) -def test_http_dispatch_returns_404_without_default(path: str, body: dict[str, object]) -> None: - table = RouteTable() - - with _make_app(table) as client: - response = client.post(path, json=body) - - assert response.status_code == 404 - assert response.json() == { - "error": { - "message": "No route registered for model missing", - "type": "model_not_found", - "code": "model_not_found", - } - } diff --git a/tests/test_routing_log_response_processor.py b/tests/test_routing_log_response_processor.py deleted file mode 100644 index c63d321e3..000000000 --- a/tests/test_routing_log_response_processor.py +++ /dev/null @@ -1,297 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for the per-request JSONL routing log (`serve --routing-log-file`).""" - -from __future__ import annotations - -import json -from collections.abc import AsyncIterator -from pathlib import Path - -from fastapi import FastAPI -from fastapi.testclient import TestClient -from openai.types.chat import ChatCompletionChunk -from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice -from openai.types.chat.chat_completion_chunk import ChoiceDelta -from openai.types.completion_usage import CompletionUsage - -from switchyard.cli.switchyard_cli import _build_parser -from switchyard.lib.chat_response import ResponseStream -from switchyard.lib.endpoints import RoutingLogStatsEndpoint -from switchyard.lib.processors.routing_log_response_processor import ( - RoutingLogResponseProcessor, -) -from switchyard.lib.proxy_context import CTX_PROXY_ACTUAL_MODEL -from switchyard.lib.request_metadata import RequestMetadata, attach_request_metadata -from switchyard_rust.core import ChatResponse, ProxyContext - -TASK_HEADERS = { - "x-switchyard-intake-task": "hello-world-abc1", - "x-switchyard-trial-id": "hello-world-abc1-Xy7", - "proxy_x_session_id": "trial-session-1", -} - - -def _ctx(*, headers: dict[str, str] | None = None, model: str = "gpt-test") -> ProxyContext: - ctx = ProxyContext() - if headers is not None: - attach_request_metadata(ctx, RequestMetadata.from_headers(headers), headers) - ctx.metadata[CTX_PROXY_ACTUAL_MODEL] = model - return ctx - - -def _openai_completion() -> ChatResponse: - return ChatResponse.openai_completion({ - "id": "chatcmpl-test", - "object": "chat.completion", - "created": 1700000000, - "model": "gpt-test", - "choices": [{ - "index": 0, - "message": {"role": "assistant", "content": "hi"}, - "finish_reason": "stop", - }], - "usage": { - "prompt_tokens": 8, - "completion_tokens": 3, - "total_tokens": 11, - "prompt_tokens_details": { - "cached_tokens": 6, - "cache_creation_tokens": 1, - }, - "completion_tokens_details": {"reasoning_tokens": 2}, - }, - }) - - -def _anthropic_completion() -> ChatResponse: - return ChatResponse.anthropic_completion({ - "id": "msg_test", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "hi"}], - "model": "claude-test", - "stop_reason": "end_turn", - "stop_sequence": None, - "usage": { - "input_tokens": 7, - "output_tokens": 3, - "cache_creation_input_tokens": 2, - "cache_read_input_tokens": 4, - }, - }) - - -def _responses_completion() -> ChatResponse: - return ChatResponse.openai_responses_completion({ - "id": "resp_test", - "object": "response", - "created_at": 1700000000, - "status": "completed", - "model": "codex-test", - "output": [{ - "type": "message", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": "hi"}], - }], - "parallel_tool_calls": False, - "tool_choice": "auto", - "tools": [], - "usage": { - "input_tokens": 6, - "output_tokens": 2, - "total_tokens": 8, - "input_tokens_details": {"cached_tokens": 3}, - "output_tokens_details": {"reasoning_tokens": 1}, - }, - }) - - -def _records(log_file: Path) -> list[dict]: - return [json.loads(line) for line in log_file.read_text().splitlines()] - - -async def test_openai_completion_record_carries_task_and_usage(tmp_path: Path) -> None: - log_file = tmp_path / "routing_requests.jsonl" - ctx = _ctx(headers=TASK_HEADERS) - ctx.selected_model = "configured-route" - ctx.selected_target = "weak" - await RoutingLogResponseProcessor(log_file).process(ctx, _openai_completion()) - - (record,) = _records(log_file) - assert record["task"] == "hello-world-abc1" - assert record["trial_id"] == "hello-world-abc1-Xy7" - assert record["session_id"] == "trial-session-1" - assert record["model"] == "gpt-test" - assert record["tier"] == "weak" - assert record["prompt_tokens"] == 8 - assert record["cached_tokens"] == 6 - assert record["cache_creation_tokens"] == 1 - assert record["completion_tokens"] == 3 - assert record["reasoning_tokens"] == 2 - assert record["total_tokens"] == 11 - - -async def test_anthropic_usage_sums_cache_siblings(tmp_path: Path) -> None: - log_file = tmp_path / "routing_requests.jsonl" - await RoutingLogResponseProcessor(log_file).process( - _ctx(headers=TASK_HEADERS, model="claude-test"), _anthropic_completion(), - ) - - (record,) = _records(log_file) - assert record["prompt_tokens"] == 13 # input + cache_creation + cache_read - assert record["cached_tokens"] == 4 - assert record["cache_creation_tokens"] == 2 - assert record["completion_tokens"] == 3 - assert record["reasoning_tokens"] == 0 - - -async def test_responses_completion_uses_input_output_tokens(tmp_path: Path) -> None: - log_file = tmp_path / "routing_requests.jsonl" - await RoutingLogResponseProcessor(log_file).process( - _ctx(headers=TASK_HEADERS, model="codex-test"), _responses_completion(), - ) - - (record,) = _records(log_file) - assert record["prompt_tokens"] == 6 - assert record["cached_tokens"] == 3 - assert record["cache_creation_tokens"] == 0 - assert record["completion_tokens"] == 2 - assert record["reasoning_tokens"] == 1 - - -async def test_missing_headers_log_null_task_and_session(tmp_path: Path) -> None: - log_file = tmp_path / "routing_requests.jsonl" - await RoutingLogResponseProcessor(log_file).process(_ctx(), _openai_completion()) - - (record,) = _records(log_file) - assert record["task"] is None - assert record["trial_id"] is None - assert record["session_id"] is None - - -async def test_streaming_appends_after_drain(tmp_path: Path) -> None: - log_file = tmp_path / "routing_requests.jsonl" - content_chunk = ChatCompletionChunk( - id="chatcmpl-test", object="chat.completion.chunk", created=1700000000, - model="gpt-test", - choices=[ChunkChoice(index=0, delta=ChoiceDelta(content="hi"), finish_reason="stop")], - ) - usage_chunk = ChatCompletionChunk( - id="chatcmpl-test", object="chat.completion.chunk", created=1700000000, - model="gpt-test", choices=[], - usage=CompletionUsage(prompt_tokens=5, completion_tokens=3, total_tokens=8), - ) - - async def _iter() -> AsyncIterator[ChatCompletionChunk]: - yield content_chunk - yield usage_chunk - - response = ChatResponse.openai_stream(ResponseStream(_iter())) - out = await RoutingLogResponseProcessor(log_file).process( - _ctx(headers=TASK_HEADERS), response, - ) - assert not log_file.exists() # nothing until the stream drains - - forwarded = [chunk async for chunk in out.stream] - assert len(forwarded) == 2 - - (record,) = _records(log_file) - assert record["task"] == "hello-world-abc1" - assert record["total_tokens"] == 8 - - -async def test_appends_one_line_per_request(tmp_path: Path) -> None: - log_file = tmp_path / "routing_requests.jsonl" - processor = RoutingLogResponseProcessor(log_file) - for _ in range(3): - await processor.process(_ctx(headers=TASK_HEADERS), _openai_completion()) - assert len(_records(log_file)) == 3 - - -def test_snapshot_session_aggregates_models_and_ignores_bad_records(tmp_path: Path) -> None: - log_file = tmp_path / "routing_requests.jsonl" - log_file.write_text( - "\n".join([ - json.dumps({ - "session_id": "trial-session-1", "model": "model-a", - "prompt_tokens": 8, "cached_tokens": 2, - "cache_creation_tokens": 1, "completion_tokens": 3, - }), - json.dumps({ - "session_id": "trial-session-1", "model": "model-b", - "prompt_tokens": 13, "cached_tokens": 4, - "cache_creation_tokens": 2, "completion_tokens": 5, - }), - json.dumps({"session_id": "other", "model": "model-a"}), - json.dumps(["not", "an", "object"]), - "not json", - ]) + "\n", - encoding="utf-8", - ) - - snapshot = RoutingLogResponseProcessor(log_file).snapshot_session("trial-session-1") - - assert snapshot == { - "session_id": "trial-session-1", - "total_calls": 2, - "total_prompt_tokens": 21, - "total_cached_tokens": 6, - "total_cache_creation_tokens": 3, - "total_completion_tokens": 8, - "models": { - "model-a": { - "calls": 1, "prompt_tokens": 8, "cached_tokens": 2, - "cache_creation_tokens": 1, "completion_tokens": 3, - }, - "model-b": { - "calls": 1, "prompt_tokens": 13, "cached_tokens": 4, - "cache_creation_tokens": 2, "completion_tokens": 5, - }, - }, - } - - -def test_session_stats_endpoint_returns_snapshot_and_404(tmp_path: Path) -> None: - log_file = tmp_path / "routing_requests.jsonl" - processor = RoutingLogResponseProcessor(log_file) - app = FastAPI() - endpoint = processor.get_endpoint() - assert isinstance(endpoint, RoutingLogStatsEndpoint) - endpoint.register(app) - - with TestClient(app) as client: - assert client.get( - "/v1/routing/session-stats", params={"session_id": "missing"} - ).status_code == 404 - - log_file.write_text( - json.dumps({ - "session_id": "trial-session-1", "model": "model-a", - "prompt_tokens": 8, "cached_tokens": 2, - "cache_creation_tokens": 1, "completion_tokens": 3, - }) + "\n", - encoding="utf-8", - ) - with TestClient(app) as client: - response = client.get( - "/v1/routing/session-stats", params={"session_id": "trial-session-1"} - ) - assert response.status_code == 200 - assert response.json()["models"]["model-a"]["cached_tokens"] == 2 - - -def test_serve_parser_accepts_routing_log_file() -> None: - parser = _build_parser() - args = parser.parse_args( - [ - "serve", - "--routes", - "routes.yaml", - "--routing-log-file", - "tmp/routing.jsonl", - ] - ) - assert args.routing_log_file == "tmp/routing.jsonl" diff --git a/tests/test_shell_tui.py b/tests/test_shell_tui.py index d898c9ac6..6904e5f9e 100644 --- a/tests/test_shell_tui.py +++ b/tests/test_shell_tui.py @@ -38,7 +38,7 @@ import pytest -from switchyard.server.shell_tui import CSI, ShellTUI +from switchyard.cli.launchers.shell_tui import CSI, ShellTUI def _make_tui( @@ -56,7 +56,7 @@ def _make_tui( @pytest.fixture def fixed_winsize(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( - "switchyard.server.shell_tui._get_winsize", + "switchyard.cli.launchers.shell_tui._get_winsize", lambda _fd: (24, 80), ) @@ -370,7 +370,7 @@ def test_handle_winch_propagates_size_and_repaints( fcntl.fcntl(master_fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) monkeypatch.setattr( - "switchyard.server.shell_tui._get_winsize", + "switchyard.cli.launchers.shell_tui._get_winsize", lambda _fd: (40, 100), ) @@ -437,7 +437,7 @@ def test_handle_winch_bumps_activity_ts_to_block_footer_thread_barge( fcntl.fcntl(master_fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) monkeypatch.setattr( - "switchyard.server.shell_tui._get_winsize", + "switchyard.cli.launchers.shell_tui._get_winsize", lambda _fd: (24, 80), ) diff --git a/tests/test_sse_stream_close.py b/tests/test_sse_stream_close.py deleted file mode 100644 index 0376c60df..000000000 --- a/tests/test_sse_stream_close.py +++ /dev/null @@ -1,159 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Streaming teardown: the upstream stream is closed on every exit path. - -Regression coverage for the connection-pool leak where an interrupted SSE -response (client disconnect) never closed the upstream SDK ``AsyncStream``, so -its httpx connection was never returned to the pool. The fix closes the stream -from the SSE helpers' ``finally`` and gives ``ChatResponseStream`` an ``aclose`` -that releases the original Python source. -""" - -from __future__ import annotations - -from typing import Any - -from switchyard.lib.chat_response.openai_chat import ResponseStream -from switchyard.lib.endpoints.sse_helpers import ( - _aclose_stream, - iter_anthropic_sse, - iter_chat_completion_sse, - iter_preframed_sse, -) - - -class _FakeAsyncStream: - """Mimics the OpenAI/Anthropic SDK ``AsyncStream``. - - Async-iterable and closable via an async ``close()`` (the SDK shape — not - the async-generator ``aclose``). Records whether ``close()`` ran so tests - can assert the upstream connection would have been released. - """ - - def __init__(self, chunks: list[Any]) -> None: - self._chunks = list(chunks) - self._index = 0 - self.closed = False - - def __aiter__(self) -> _FakeAsyncStream: - return self - - async def __anext__(self) -> Any: - if self._index >= len(self._chunks): - raise StopAsyncIteration - chunk = self._chunks[self._index] - self._index += 1 - return chunk - - - async def close(self) -> None: - self.closed = True - - -# --------------------------------------------------------------------------- -# ChatResponseStream.aclose (Rust binding) -# --------------------------------------------------------------------------- - - -async def test_chat_response_stream_aclose_closes_source() -> None: - """``aclose`` releases the original Python source even before iteration.""" - fake = _FakeAsyncStream([{"id": "1"}]) - stream = ResponseStream(fake) - await stream.aclose() - assert fake.closed - - -async def test_chat_response_stream_aclose_is_idempotent() -> None: - fake = _FakeAsyncStream([{"id": "1"}]) - stream = ResponseStream(fake) - await stream.aclose() - await stream.aclose() - assert fake.closed - - -# --------------------------------------------------------------------------- -# SSE helpers close their stream on early termination and normal completion -# --------------------------------------------------------------------------- - - -async def test_chat_sse_closes_stream_on_client_disconnect() -> None: - """A client disconnect makes the server ``aclose()`` the SSE generator; - the helper's ``finally`` must then close the upstream stream.""" - fake = _FakeAsyncStream([{"id": "1"}, {"id": "2"}, {"id": "3"}]) - generator = iter_chat_completion_sse(fake) - first = await generator.__anext__() - assert first.startswith("data:") - await generator.aclose() # simulate disconnect mid-stream - assert fake.closed - - -async def test_chat_sse_closes_stream_on_normal_completion() -> None: - fake = _FakeAsyncStream([{"id": "1"}]) - frames = [frame async for frame in iter_chat_completion_sse(fake)] - assert frames[-1] == "data: [DONE]\n\n" - assert fake.closed - - -async def test_chat_sse_closes_rust_stream_source_on_disconnect() -> None: - """End-to-end: SSE over a ``ChatResponseStream`` → disconnect → the wrapped - SDK ``AsyncStream`` (and its connection) is released. This is the exact leak - scenario from the OOM incident.""" - fake = _FakeAsyncStream([{"id": "1"}, {"id": "2"}]) - generator = iter_chat_completion_sse(ResponseStream(fake)) - await generator.__anext__() - await generator.aclose() - assert fake.closed - - -async def test_anthropic_sse_closes_stream_on_disconnect() -> None: - fake = _FakeAsyncStream([{"type": "message_start"}, {"type": "content_block_delta"}]) - generator = iter_anthropic_sse(fake) - await generator.__anext__() - await generator.aclose() - assert fake.closed - - -async def test_preframed_sse_closes_stream_on_disconnect() -> None: - fake = _FakeAsyncStream(["event: ping\ndata: {}\n\n", "event: ping\ndata: {}\n\n"]) - generator = iter_preframed_sse(fake) - await generator.__anext__() - await generator.aclose() - assert fake.closed - - -async def test_preframed_sse_frames_responses_mapping_events() -> None: - fake = _FakeAsyncStream([ - { - "type": "response.created", - "response": {"id": "resp-test"}, - } - ]) - frames = [frame async for frame in iter_preframed_sse(fake)] - assert frames == [ - 'event: response.created\ndata: {"type": "response.created", "response": {"id": "resp-test"}}\n\n' - ] - assert fake.closed - - -# --------------------------------------------------------------------------- -# _aclose_stream contract -# --------------------------------------------------------------------------- - - -async def test_aclose_stream_prefers_aclose() -> None: - class WithAclose: - def __init__(self) -> None: - self.aclosed = False - - async def aclose(self) -> None: - self.aclosed = True - - obj = WithAclose() - await _aclose_stream(obj) - assert obj.aclosed - - -async def test_aclose_stream_tolerates_missing_closer() -> None: - # An object with neither ``aclose`` nor ``close`` must not raise. - await _aclose_stream(object()) diff --git a/tests/test_stats_accumulator.py b/tests/test_stats_accumulator.py deleted file mode 100644 index c8b881a31..000000000 --- a/tests/test_stats_accumulator.py +++ /dev/null @@ -1,247 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -import pytest - -from switchyard.lib.stats_accumulator import StatsAccumulator - - -async def test_snapshot_sync_matches_async_snapshot_and_includes_tier_tokens(): - stats = StatsAccumulator() - await stats.record_success( - model="strong/model", - backend_latency_ms=12.0, - tier="strong", - ) - await stats.record_usage( - model="strong/model", - prompt_tokens=100, - completion_tokens=25, - cached_tokens=10, - total_latency_ms=20.0, - routing_overhead_ms=8.0, - tier="strong", - ) - await stats.record_success(model="weak/model", tier="weak") - await stats.record_usage( - model="weak/model", - prompt_tokens=40, - completion_tokens=5, - tier="weak", - ) - - sync_snapshot = stats.snapshot_sync() - - assert sync_snapshot == await stats.snapshot() - assert sync_snapshot["total_requests"] == 2 - assert sync_snapshot["total_tokens"]["prompt"] == 140 - assert sync_snapshot["total_tokens"]["completion"] == 30 - assert sync_snapshot["models"]["strong/model"]["tier"] == "strong" - assert sync_snapshot["tiers"]["strong"]["prompt_tokens"] == 100 - assert sync_snapshot["tiers"]["strong"]["completion_tokens"] == 25 - assert sync_snapshot["tiers"]["weak"]["prompt_tokens"] == 40 - assert sync_snapshot["tiers"]["weak"]["completion_tokens"] == 5 - - -async def test_snapshot_includes_generic_tier_rollup(): - stats = StatsAccumulator() - await stats.record_success(model="plugin/model-a", tier="plugin") - await stats.record_usage( - model="plugin/model-a", - prompt_tokens=2, - completion_tokens=3, - tier="plugin", - ) - await stats.record_success(model="plugin/model-b", tier="plugin") - await stats.record_usage( - model="plugin/model-b", - prompt_tokens=5, - completion_tokens=7, - tier="plugin", - ) - - snapshot = await stats.snapshot() - - assert snapshot["models"]["plugin/model-a"]["tier"] == "plugin" - assert snapshot["models"]["plugin/model-b"]["tier"] == "plugin" - assert snapshot["tiers"]["plugin"]["model"] == "plugin/model-a" - assert snapshot["tiers"]["plugin"]["calls"] == 2 - assert snapshot["tiers"]["plugin"]["prompt_tokens"] == 7 - assert snapshot["tiers"]["plugin"]["completion_tokens"] == 10 - - -async def test_same_model_can_contribute_to_distinct_tier_rollups(): - stats = StatsAccumulator() - await stats.record_success(model="shared/model", tier="weak") - await stats.record_usage( - model="shared/model", - prompt_tokens=2, - completion_tokens=3, - tier="weak", - ) - await stats.record_success(model="shared/model", tier="primary") - await stats.record_usage( - model="shared/model", - prompt_tokens=5, - completion_tokens=7, - tier="primary", - ) - - snapshot = await stats.snapshot() - - assert snapshot["models"]["shared/model"]["calls"] == 2 - assert snapshot["models"]["shared/model"]["prompt_tokens"] == 7 - assert snapshot["models"]["shared/model"]["completion_tokens"] == 10 - assert snapshot["tiers"]["weak"]["calls"] == 1 - assert snapshot["tiers"]["weak"]["prompt_tokens"] == 2 - assert snapshot["tiers"]["weak"]["completion_tokens"] == 3 - assert snapshot["tiers"]["primary"]["calls"] == 1 - assert snapshot["tiers"]["primary"]["prompt_tokens"] == 5 - assert snapshot["tiers"]["primary"]["completion_tokens"] == 7 - - -async def test_usage_can_attach_explicit_untiered_success_to_tier(): - stats = StatsAccumulator() - await stats.record_success(model="shared/model") - await stats.record_usage( - model="shared/model", - prompt_tokens=2, - completion_tokens=3, - tier="weak", - success_was_untiered=True, - ) - - snapshot = await stats.snapshot() - - assert snapshot["models"]["shared/model"]["calls"] == 1 - assert snapshot["tiers"]["weak"]["calls"] == 1 - assert snapshot["tiers"]["weak"]["prompt_tokens"] == 2 - assert snapshot["tiers"]["weak"]["completion_tokens"] == 3 - - -async def test_legacy_untiered_success_then_tiered_usage_counts_tier_call(): - stats = StatsAccumulator() - await stats.record_success(model="shared/model") - await stats.record_usage( - model="shared/model", - prompt_tokens=2, - completion_tokens=3, - tier="weak", - ) - - snapshot = await stats.snapshot() - - assert snapshot["models"]["shared/model"]["calls"] == 1 - assert snapshot["tiers"]["weak"]["calls"] == 1 - assert snapshot["tiers"]["weak"]["prompt_tokens"] == 2 - assert snapshot["tiers"]["weak"]["completion_tokens"] == 3 - - -async def test_reset_sync_clears_async_recorded_stats(): - stats = StatsAccumulator() - await stats.record_success(model="model") - await stats.record_usage(model="model", prompt_tokens=10, completion_tokens=5) - - stats.reset_sync() - - snapshot = await stats.snapshot() - assert snapshot["total_requests"] == 0 - assert snapshot["total_tokens"]["total"] == 0 - assert snapshot["models"] == {} - - -async def test_classifier_usage_recorded_into_separate_bucket(): - """Classifier calls don't leak into the routed-backend ``models`` block. - - Default TB-lite config has classifier-model == weak-model - (Nemotron-3-Super-v3). Without the separate bucket the two would - accumulate into the same entry and the per-classifier breakdown - would be lost. - """ - stats = StatsAccumulator() - # Same model name on both sides — exactly the collision case. - await stats.record_success(model="nvidia/nemotron-3-super-v3", tier="weak") - await stats.record_usage( - model="nvidia/nemotron-3-super-v3", - prompt_tokens=1_000, - completion_tokens=200, - tier="weak", - ) - await stats.record_classifier_usage( - model="nvidia/nemotron-3-super-v3", - prompt_tokens=300, - completion_tokens=50, - latency_ms=42.0, - ) - - snapshot = await stats.snapshot() - - # Backend bucket counts the routed call only. - backend = snapshot["models"]["nvidia/nemotron-3-super-v3"] - assert backend["prompt_tokens"] == 1_000 - assert backend["completion_tokens"] == 200 - assert backend["calls"] == 1 - # Classifier bucket counts the classifier call only. - classifier = snapshot["classifier"]["models"]["nvidia/nemotron-3-super-v3"] - assert classifier["prompt_tokens"] == 300 - assert classifier["completion_tokens"] == 50 - assert classifier["calls"] == 1 - assert classifier["model_call_latency"]["count"] == 1 - - -async def test_cost_estimate_total_includes_classifier_overhead(): - """Headline ``cost_estimate.total_cost`` is backend + classifier. - - Existing consumers (baseline manifests, dashboards) read - ``total_cost`` and don't know about the new classifier bucket; the - accumulator must roll the two together so those readers reflect - true spend. - """ - stats = StatsAccumulator() - # Use a model with known pricing so we get non-zero numbers. - await stats.record_success(model="nvidia/nvidia/nemotron-3-super-v3", tier="weak") - await stats.record_usage( - model="nvidia/nvidia/nemotron-3-super-v3", - prompt_tokens=1_000_000, - completion_tokens=0, - tier="weak", - ) - await stats.record_classifier_usage( - model="nvidia/nvidia/nemotron-3-super-v3", - prompt_tokens=1_000_000, - completion_tokens=0, - ) - - snapshot = await stats.snapshot() - cost = snapshot["cost_estimate"] - # Nemotron-3 Super input is $0.10/Mtok — backend 1M tokens = $0.10, - # classifier 1M tokens = $0.10, total = $0.20. - assert cost["backend_cost"] == pytest.approx(0.10, rel=0.01) - assert cost["classifier_cost"] == pytest.approx(0.10, rel=0.01) - assert cost["total_cost"] == pytest.approx(0.20, rel=0.01) - - -async def test_reset_clears_classifier_bucket(): - stats = StatsAccumulator() - await stats.record_classifier_usage( - model="router/clf", - prompt_tokens=100, - completion_tokens=20, - ) - - stats.reset_sync() - - snapshot = await stats.snapshot() - assert snapshot["classifier"]["total_requests"] == 0 - assert snapshot["classifier"]["models"] == {} - assert snapshot["cost_estimate"]["classifier_cost"] == 0.0 - - -# Per-target attribution after evict-and-reroute is covered by -# `evict_and_reroute_attributes_error_to_weak_and_success_to_strong` in -# crates/switchyard-components/tests/stats_processors.rs — that integration -# test drives the chain executor end-to-end. Re-asserting the same shape here -# by hand-recording into the accumulator would only verify the accumulator's -# incrementers (already covered above), not the eviction code path. diff --git a/tests/test_stream_close_chain.py b/tests/test_stream_close_chain.py deleted file mode 100644 index 81b6e085e..000000000 --- a/tests/test_stream_close_chain.py +++ /dev/null @@ -1,212 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Close ownership survives the Python -> Rust-core -> Python round trip. - -The production path is: a Python backend returns -``ChatResponse.openai_stream(ResponseStream(sdk_stream))``; the backend adapter -converts it to a Rust-core stream via ``take_core``; response processors run in -core; the terminal translator rebuilds a Python ``ChatResponseStream`` via -``from_core``; the endpoint feeds that into an SSE helper. The rebuilt stream no -longer references the original SDK ``AsyncStream``, so unless close ownership is -preserved across the conversion, closing it on client disconnect never releases -the upstream connection — the OOM connection-pool leak. - -These tests exercise the *real* chain (``Switchyard`` + ``TranslationEngine``), -not just the SSE helper in isolation, and assert the fake SDK stream's close -hook runs after the response stream is closed mid-flight — for both same-format -(OpenAI in / OpenAI backend) and translated (Anthropic in / OpenAI backend) -streaming. -""" - -from __future__ import annotations - -import asyncio -from typing import Any - -from switchyard.lib.chat_response.openai_chat import ResponseStream -from switchyard.lib.endpoints.sse_helpers import ( - iter_anthropic_sse, - iter_chat_completion_sse, -) -from switchyard.lib.proxy_context import ProxyContext -from switchyard.lib.roles import LLMBackend -from switchyard.lib.switchyard import Switchyard -from switchyard_rust.core import ChatRequest, ChatRequestType, ChatResponse -from switchyard_rust.translation import TranslationEngine - - -class _FakeSdkStream: - """Mimics the OpenAI SDK ``AsyncStream``: async-iterable with async ``close``. - - Records whether ``close()`` ran so a test can assert the upstream response - (and its pooled connection) would have been released. - """ - - def __init__(self, chunks: list[dict[str, Any]]) -> None: - self._chunks = list(chunks) - self._index = 0 - self.closed = False - - def __aiter__(self) -> _FakeSdkStream: - return self - - async def __anext__(self) -> dict[str, Any]: - if self._index >= len(self._chunks): - raise StopAsyncIteration - chunk = self._chunks[self._index] - self._index += 1 - return chunk - - async def close(self) -> None: - self.closed = True - - -def _chunk(content: str | None, *, role: str | None = None, finish: str | None = None) -> dict: - delta: dict[str, Any] = {} - if role is not None: - delta["role"] = role - if content is not None: - delta["content"] = content - return { - "id": "chatcmpl-close", - "object": "chat.completion.chunk", - "created": 1700000000, - "model": "close-model", - "choices": [{"index": 0, "delta": delta, "finish_reason": finish}], - } - - -def _chunks() -> list[dict]: - return [ - _chunk("", role="assistant"), - _chunk("hello"), - _chunk(" world"), - _chunk(None, finish="stop"), - ] - - -class _StreamingSdkBackend(LLMBackend): - """Backend returning a streaming response wrapping a fake SDK stream. - - Holds the fake so a test can assert it was closed. Supports both OpenAI Chat - and Anthropic inbound so the same backend serves the same-format and the - translated paths (the response is always an OpenAI chat stream). - """ - - def __init__(self, fake: _FakeSdkStream) -> None: - self._fake = fake - - @property - def supported_request_types(self) -> list[ChatRequestType]: - return [ChatRequestType.OPENAI_CHAT, ChatRequestType.ANTHROPIC] - - async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: - _ = ctx, request - return ChatResponse.openai_stream(ResponseStream(self._fake)) - - -async def _wait_closed(fake: _FakeSdkStream, timeout: float = 2.0) -> bool: - """Wait for the best-effort, runtime-scheduled source close to land. - - ``ChatResponseStream``'s drop-time close runs as a fire-and-forget task on - the Rust tokio runtime (a separate thread), so it may not have completed the - instant ``aclose`` returns to the asyncio loop. Poll briefly for it. - """ - loop = asyncio.get_event_loop() - deadline = loop.time() + timeout - while not fake.closed and loop.time() < deadline: - await asyncio.sleep(0.005) - return fake.closed - - -def _openai_request() -> ChatRequest: - return ChatRequest.openai_chat({ - "model": "close-model", - "messages": [{"role": "user", "content": "hi"}], - "stream": True, - }) - - -def _anthropic_request() -> ChatRequest: - return ChatRequest.anthropic({ - "model": "close-model", - "max_tokens": 16, - "messages": [{"role": "user", "content": "hi"}], - "stream": True, - }) - - -async def test_same_format_stream_close_releases_sdk_source_through_chain() -> None: - """OpenAI in / OpenAI backend: closing the SSE stream after one frame closes - the SDK stream the backend wrapped — across take_core/from_core.""" - fake = _FakeSdkStream(_chunks()) - chain = Switchyard(backend=_StreamingSdkBackend(fake), translator=TranslationEngine()) - - result = await chain.call(_openai_request()) - sse = iter_chat_completion_sse(result) - first = await sse.__anext__() - assert first.startswith("data:") - - await sse.aclose() # client disconnect mid-stream - - assert await _wait_closed(fake), "backend SDK stream was not closed after teardown" - - -async def test_translated_stream_close_releases_sdk_source_through_chain() -> None: - """Anthropic in / OpenAI backend: the response is translated through - ``translate_stream``; closing the SSE stream still closes the SDK source.""" - fake = _FakeSdkStream(_chunks()) - chain = Switchyard(backend=_StreamingSdkBackend(fake), translator=TranslationEngine()) - - result = await chain.call(_anthropic_request()) - sse = iter_anthropic_sse(result) - first = await sse.__anext__() - assert first.startswith("event:") - - await sse.aclose() # client disconnect mid-stream - - assert await _wait_closed(fake), "backend SDK stream was not closed after translated teardown" - - -class _FakeTranslatableInput: - """Async-iterable openai chunk source with an async ``aclose`` it records.""" - - def __init__(self, chunks: list[dict[str, Any]]) -> None: - self._chunks = list(chunks) - self._index = 0 - self.aclosed = False - - def __aiter__(self) -> _FakeTranslatableInput: - return self - - async def __anext__(self) -> dict[str, Any]: - if self._index >= len(self._chunks): - raise StopAsyncIteration - chunk = self._chunks[self._index] - self._index += 1 - return chunk - - async def aclose(self) -> None: - self.aclosed = True - - -async def test_translate_stream_closes_input_on_disconnect() -> None: - """``translate_stream``'s ``finally`` closes its input when the consuming - SSE generator is ``aclose``-ed mid-stream (the client-disconnect path).""" - fake = _FakeTranslatableInput(_chunks()) - gen = TranslationEngine().translate_stream("openai_chat", "anthropic_messages", fake) - - await gen.__anext__() - await gen.aclose() - - assert fake.aclosed, "translate_stream did not close its input stream" - - -async def test_translate_stream_closes_input_on_normal_completion() -> None: - fake = _FakeTranslatableInput(_chunks()) - gen = TranslationEngine().translate_stream("openai_chat", "anthropic_messages", fake) - - _ = [frame async for frame in gen] - - assert fake.aclosed, "translate_stream did not close its input on completion" diff --git a/tests/test_stream_leak_repro.py b/tests/test_stream_leak_repro.py deleted file mode 100644 index 22c1f571c..000000000 --- a/tests/test_stream_leak_repro.py +++ /dev/null @@ -1,261 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Empirical repro: unclosed streaming responses exhaust the httpx pool, and our -SSE-helper close path releases the connection. - -Uses the REAL OpenAI ``AsyncStream`` against a loopback SSE server, with the -httpx pool capped at a single connection so exhaustion is deterministic. The -stream is driven through the real ``iter_chat_completion_sse`` helper, so the -``finally: await _aclose_stream(...)`` we added is the thing under test. - -Marked ``integration`` (real sockets + a pool timeout) so the default unit gate -can deselect it with ``-m "not integration"``; it is the canonical regression -for the OOM connection-pool leak. -""" - -from __future__ import annotations - -import json -import threading -import time -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer - -import httpx -import openai -import pytest -from openai import AsyncOpenAI - -from switchyard.lib.chat_response.openai_chat import ResponseStream -from switchyard.lib.endpoints.sse_helpers import iter_chat_completion_sse -from switchyard.lib.proxy_context import ProxyContext -from switchyard.lib.roles import LLMBackend -from switchyard.lib.switchyard import Switchyard -from switchyard_rust.core import ChatRequest, ChatRequestType, ChatResponse -from switchyard_rust.translation import TranslationEngine - -pytestmark = pytest.mark.integration - - -def _chunk(i: int) -> dict: - return { - "id": "chatcmpl-leak", - "object": "chat.completion.chunk", - "created": 1700000000, - "model": "leak-model", - "choices": [{"index": 0, "delta": {"content": str(i)}, "finish_reason": None}], - } - - -class _SlowSSEStub: - """Loopback OpenAI-compatible server that streams SSE chunks slowly. - - Streams many chunks with a delay so a client that reads one frame and - abandons the response leaves the connection checked out — the production - client-disconnect scenario. - """ - - def __init__(self, n_chunks: int = 500, delay_s: float = 0.02) -> None: - self._n = n_chunks - self._delay = delay_s - self._server: ThreadingHTTPServer | None = None - self._thread: threading.Thread | None = None - - def __enter__(self) -> _SlowSSEStub: - n, delay = self._n, self._delay - - class Handler(BaseHTTPRequestHandler): - protocol_version = "HTTP/1.1" - - def do_POST(self) -> None: - length = int(self.headers.get("content-length", "0")) - if length: - self.rfile.read(length) - self.send_response(200) - self.send_header("Content-Type", "text/event-stream") - self.send_header("Transfer-Encoding", "chunked") - self.end_headers() - - def send(data: bytes) -> None: - self.wfile.write(f"{len(data):X}\r\n".encode()) - self.wfile.write(data) - self.wfile.write(b"\r\n") - self.wfile.flush() - - try: - for i in range(n): - send(f"data: {json.dumps(_chunk(i))}\n\n".encode()) - time.sleep(delay) - send(b"data: [DONE]\n\n") - self.wfile.write(b"0\r\n\r\n") - self.wfile.flush() - except (BrokenPipeError, ConnectionResetError, OSError): - return # client disconnected — expected in these tests - - def log_message(self, *_args: object) -> None: - return - - self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) - self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) - self._thread.start() - return self - - def __exit__(self, *_args: object) -> None: - if self._server is not None: - self._server.shutdown() - self._server.server_close() - if self._thread is not None: - self._thread.join(timeout=2) - - @property - def base_url(self) -> str: - assert self._server is not None - host, port = self._server.server_address - return f"http://{host}:{port}/v1" - - -def _client(base_url: str) -> AsyncOpenAI: - # max_connections=1 makes exhaustion deterministic: one leaked stream takes - # the only slot. pool=1.5s fails the next acquire fast; max_retries=0 keeps - # the SDK from retrying the pool timeout (which would only slow the test). - return AsyncOpenAI( - base_url=base_url, - api_key="test", - max_retries=0, - http_client=httpx.AsyncClient( - limits=httpx.Limits(max_connections=1, max_keepalive_connections=1), - timeout=httpx.Timeout(5.0, pool=1.5), - ), - ) - - -def _pool_conns(client: AsyncOpenAI) -> int: - """Best-effort live connection count for evidence logging (version-dependent).""" - try: - return len(client._client._transport._pool.connections) # type: ignore[attr-defined] - except Exception: - return -1 - - -async def _open_and_read_one(client: AsyncOpenAI, *, close: bool): - """Open a streaming completion, drive it through the real SSE helper for one - frame, then either close it (our fix) or abandon it (the leak).""" - stream = await client.chat.completions.create( - model="leak-model", - messages=[{"role": "user", "content": "hi"}], - stream=True, - ) - sse = iter_chat_completion_sse(stream) - first = await sse.__anext__() - assert first.startswith("data:"), first - assert "error" not in first, f"stream errored on first frame: {first}" - if close: - await sse.aclose() # our fix: finally -> _aclose_stream -> AsyncStream.close() - return None - return sse, stream # hold refs so the connection stays checked out - - -async def test_unclosed_stream_exhausts_pool() -> None: - """An abandoned stream pins its pooled connection; the next request starves.""" - with _SlowSSEStub() as stub: - client = _client(stub.base_url) - try: - held = await _open_and_read_one(client, close=False) # leak: never closed - assert held is not None - assert _pool_conns(client) == 1 - # The only pool slot is occupied by the abandoned stream; a new - # streaming request cannot acquire a connection and must fail. - with pytest.raises( - ( - openai.APITimeoutError, - openai.APIConnectionError, - httpx.PoolTimeout, - httpx.TimeoutException, - ) - ): - await _open_and_read_one(client, close=False) - finally: - await client.close() - - -async def test_our_close_releases_pool_connection() -> None: - """Closing via the SSE helper frees the slot, so a sequence of streams runs.""" - with _SlowSSEStub() as stub: - client = _client(stub.base_url) - try: - # Our fix closes each stream, freeing the single pool slot, so a - # whole sequence of streaming requests succeeds — no exhaustion. - for _ in range(5): - await _open_and_read_one(client, close=True) - assert _pool_conns(client) <= 1 - finally: - await client.close() - - -class _RealSdkStreamingBackend(LLMBackend): - """Backend that issues a real streaming SDK call and wraps the AsyncStream. - - Mirrors the production shape: ``call`` returns - ``ChatResponse.openai_stream(ResponseStream(sdk_stream))`` where - ``sdk_stream`` is a genuine OpenAI ``AsyncStream`` holding a pooled httpx - connection. Running this through ``Switchyard`` exercises the real - ``take_core`` -> Rust-core -> ``from_core`` round trip that drops the - source — the exact path the in-process fakes cannot reach. - """ - - def __init__(self, client: AsyncOpenAI) -> None: - self._client = client - - @property - def supported_request_types(self) -> list[ChatRequestType]: - return [ChatRequestType.OPENAI_CHAT] - - async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: - _ = ctx, request - sdk_stream = await self._client.chat.completions.create( - model="leak-model", - messages=[{"role": "user", "content": "hi"}], - stream=True, - ) - return ChatResponse.openai_stream(ResponseStream(sdk_stream)) - - -def _chain_request() -> ChatRequest: - return ChatRequest.openai_chat({ - "model": "leak-model", - "messages": [{"role": "user", "content": "hi"}], - "stream": True, - }) - - -async def test_chain_round_trip_releases_pool_connection() -> None: - """End-to-end through the production chain: a real SDK stream survives - ``take_core``/``from_core`` and is released on client disconnect. - - With a single pool slot, each iteration acquires the only connection, reads - one frame, and abandons the stream. The next iteration's upstream call must - re-acquire that slot, which only succeeds if the prior stream's connection - was actually returned to the pool. If close ownership were lost across the - Rust-core conversion, the second iteration would starve and raise - ``PoolTimeout``; the loop completing proves the fix holds through the real - chain (backend -> take_core -> core -> from_core -> translator -> SSE). - """ - with _SlowSSEStub() as stub: - client = _client(stub.base_url) - chain = Switchyard( - backend=_RealSdkStreamingBackend(client), - translator=TranslationEngine(), - ) - try: - for _ in range(5): - result = await chain.call(_chain_request()) - sse = iter_chat_completion_sse(result) - first = await sse.__anext__() - assert first.startswith("data:"), first - assert "error" not in first, f"stream errored on first frame: {first}" - # Disconnect mid-stream. The next iteration's acquire blocks on - # the pool until the runtime-scheduled close returns this slot. - await sse.aclose() - assert _pool_conns(client) <= 1 - finally: - await client.close() diff --git a/tests/test_switchyard.py b/tests/test_switchyard.py deleted file mode 100644 index 34e794688..000000000 --- a/tests/test_switchyard.py +++ /dev/null @@ -1,517 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for Switchyard executor.""" - -import asyncio - -import pytest -from openai.types.chat import ChatCompletion -from openai.types.chat.chat_completion import Choice -from openai.types.chat.chat_completion_message import ChatCompletionMessage -from openai.types.completion_usage import CompletionUsage - -from switchyard.lib.proxy_context import ProxyContext -from switchyard.lib.roles import LLMBackend -from switchyard.lib.switchyard import Switchyard -from switchyard_rust.components import ( - StatsRequestProcessor, -) -from switchyard_rust.core import ( - ChatRequest, - ChatRequestType, - ChatResponse, - ChatResponseType, - SwitchyardContextWindowExceededError, - response_type_matches, -) -from switchyard_rust.translation import TranslationEngine - -# --------------------------------------------------------------------------- -# Fixtures -# --------------------------------------------------------------------------- - - -def make_completion(*, model: str = "gpt-4o", content: str = "hello") -> ChatCompletion: - return ChatCompletion( - id="chatcmpl-test", - object="chat.completion", - created=1700000000, - model=model, - choices=[ - Choice( - index=0, - message=ChatCompletionMessage(role="assistant", content=content), - finish_reason="stop", - ) - ], - usage=CompletionUsage(prompt_tokens=10, completion_tokens=5, total_tokens=15), - ) - - -def make_request(*, model: str = "gpt-4o") -> ChatRequest: - return ChatRequest.openai_chat({ - "model": model, - "messages": [{"role": "user", "content": "hi"}], - }) - - -# --------------------------------------------------------------------------- -# Test helpers — concrete processors for testing -# --------------------------------------------------------------------------- - - -class ModelOverrideProcessor: - """Mutates the model field on the request body.""" - - def __init__(self, new_model: str) -> None: - self._new_model = new_model - - async def process(self, ctx, request): - request.set_model(self._new_model) - return request - - -class MetadataTagProcessor: - """Writes a tag into ctx.metadata to prove it ran.""" - - def __init__(self, tag: str) -> None: - self._tag = tag - - async def process(self, ctx, request): - ctx.metadata[self._tag] = True - return request - - -class MetadataAssertResponseProcessor: - """Asserts request metadata remains visible after Rust-native components run.""" - - def __init__(self, expected: dict[str, object]) -> None: - self._expected = expected - - async def process(self, ctx, response): - for key, expected in self._expected.items(): - assert ctx.metadata[key] == expected - return response - - -class ModelMetadataProcessor: - async def process(self, ctx, request): - ctx.metadata["seen_model"] = request.model - return request - - -class ModelMetadataAssertProcessor: - async def process(self, ctx, response): - assert ctx.metadata["seen_model"] == response.body["model"] - return response - - -class FailingRequestProcessor: - async def process(self, ctx, request): - ctx.metadata["request_started"] = request.model - raise RuntimeError("request processor exploded") - - -class InvalidResponseProcessor: - async def process(self, ctx, response): - ctx.metadata["response_started"] = True - return {"not": "a ChatResponse"} - - -class ContextReadingStreamTapProcessor: - def __init__(self) -> None: - self.seen_models: list[str | None] = [] - - async def process(self, ctx, response): - ctx.selected_model = "stream-selected" - - async def tap(_event): - self.seen_models.append(ctx.selected_model) - - response.stream.tap(tap) - return response - - -class ContentUpperProcessor: - """Uppercases the first choice's content (non-streaming only).""" - - async def process(self, ctx, response): - if response_type_matches(response, ChatResponseType.OPENAI_COMPLETION): - body = response.body - body["choices"][0]["message"]["content"] = ( - body["choices"][0]["message"].get("content") or "" - ).upper() - response.replace_body(body) - return response - - -class RecordingBackend(LLMBackend): - """Returns a canned OpenAI response and records processed requests.""" - - def __init__(self, *, content: str = "base") -> None: - self._content = content - self.requests: list[ChatRequest] = [] - - @property - def supported_request_types(self) -> list[ChatRequestType]: - return [ChatRequestType.OPENAI_CHAT] - - async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: - self.requests.append(request) - if request.model: - ctx.selected_model = request.model - return ChatResponse.openai_completion( - make_completion(model=request.model or "test-model", content=self._content) - ) - - -class LegacyBackendWithoutSupportedTypes(LLMBackend): - """Compatibility backend that relies on the legacy Python executor surface.""" - - def __init__(self) -> None: - self.requests: list[ChatRequest] = [] - - async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: - self.requests.append(request) - if request.model: - ctx.selected_model = request.model - return ChatResponse.openai_completion( - make_completion(model=request.model or "test-model", content="legacy") - ) - - -class FailingBackend(LLMBackend): - """Raises after mutating context so restoration-on-error is observable.""" - - @property - def supported_request_types(self) -> list[ChatRequestType]: - return [ChatRequestType.OPENAI_CHAT] - - async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: - ctx.metadata["backend_started"] = request.model - raise RuntimeError("backend exploded") - - -class RecordingTranslator: - """Records the request the executor hands to the terminal translator.""" - - def __init__(self) -> None: - self.request_model: str | None = None - - async def translate( - self, - ctx: ProxyContext, - request: ChatRequest, - response: ChatResponse, - ): - _ = ctx - self.request_model = request.model - return response.body - - -async def _single_chunk_stream(): - yield { - "id": "chatcmpl-test", - "object": "chat.completion.chunk", - "model": "gpt-4o", - "choices": [ - { - "index": 0, - "delta": {"content": "hello"}, - "finish_reason": None, - } - ], - } - - -class StreamingBackend(LLMBackend): - @property - def supported_request_types(self) -> list[ChatRequestType]: - return [ChatRequestType.OPENAI_CHAT] - - async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: - _ = ctx, request - return ChatResponse.openai_stream(_single_chunk_stream()) - - -class PickWeakProcessor: - """Routes the first attempt to the weak target.""" - - async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: - ctx.selected_target = "weak" - return request - - -class OverflowWeakBackend(LLMBackend): - """Overflows on weak and succeeds once the compatibility chain rewrites to strong.""" - - def __init__(self) -> None: - self.calls: list[str | None] = [] - - @property - def supported_request_types(self) -> list[ChatRequestType]: - return [ChatRequestType.OPENAI_CHAT] - - async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: - self.calls.append(ctx.selected_target) - if ctx.selected_target == "weak": - error = SwitchyardContextWindowExceededError("weak target overflowed") - error.target_id = "weak" - error.model = "weak-model" - raise error - ctx.selected_model = "strong-model" - return ChatResponse.openai_completion( - make_completion(model=request.model or "strong-model", content="fallback") - ) - - -class ExceptionOnlyOverflowBackend(LLMBackend): - """Overflows with only an exception target id, then succeeds on fallback.""" - - def __init__(self) -> None: - self.calls: list[str | None] = [] - - @property - def supported_request_types(self) -> list[ChatRequestType]: - return [ChatRequestType.OPENAI_CHAT] - - async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: - self.calls.append(ctx.selected_target) - if len(self.calls) == 1: - error = SwitchyardContextWindowExceededError("weak target overflowed") - error.target_id = "weak" - error.model = "weak-model" - raise error - ctx.selected_model = "strong-model" - return ChatResponse.openai_completion( - make_completion(model=request.model or "strong-model", content="fallback") - ) - - -# --------------------------------------------------------------------------- -# Switchyard executor -# --------------------------------------------------------------------------- - - -class TestSwitchyard: - def _make_chain(self, **overrides): - defaults = { - "backend": RecordingBackend(content="base"), - "translator": TranslationEngine(), - } - defaults.update(overrides) - return Switchyard(**defaults) - - def test_public_chain_classes_are_switchyard_rust_compatibility_exports(self): - from switchyard_rust import core as rust_core - - assert Switchyard is rust_core.Switchyard - - async def test_minimal_chain(self): - """Backend + translator only, no processors.""" - chain = self._make_chain() - result = await chain.call(make_request()) - assert result["choices"][0]["message"]["content"] == "base" - - async def test_python_backend_without_supported_request_types_keeps_old_switchyard_behavior( - self, - ): - backend = LegacyBackendWithoutSupportedTypes() - chain = self._make_chain(backend=backend) - - result = await chain.call(make_request(model="gpt-4o")) - - assert result["choices"][0]["message"]["content"] == "legacy" - assert backend.requests[-1].request_type == ChatRequestType.OPENAI_CHAT - - async def test_python_backend_without_supported_request_types_accepts_legacy_anthropic_request( - self, - ): - """The compatibility fallback is intentionally all formats, matching old Python.""" - backend = LegacyBackendWithoutSupportedTypes() - chain = self._make_chain( - backend=backend, - translator=RecordingTranslator(), - ) - request = ChatRequest.anthropic({ - "model": "claude-test", - "messages": [{"role": "user", "content": "hi"}], - }) - - result = await chain.call(request) - - assert result["choices"][0]["message"]["content"] == "legacy" - assert backend.requests[-1].request_type == ChatRequestType.ANTHROPIC - - async def test_request_processor_runs(self): - chain = self._make_chain( - request_processors=[ModelOverrideProcessor("gpt-4o-mini")], - ) - await chain.call(make_request(model="gpt-4o")) - backend = chain._backend - assert isinstance(backend, RecordingBackend) - assert backend.requests[-1].model == "gpt-4o-mini" - - async def test_multiple_request_processors_chain(self): - chain = self._make_chain( - request_processors=[ - MetadataTagProcessor("first"), - MetadataTagProcessor("second"), - ], - ) - await chain.call(make_request()) - - async def test_response_processor_runs(self): - chain = self._make_chain( - response_processors=[ContentUpperProcessor()], - ) - result = await chain.call(make_request()) - assert result["choices"][0]["message"]["content"] == "BASE" - - async def test_full_chain(self): - """Request processors → backend → response processors → translator.""" - chain = self._make_chain( - request_processors=[ - MetadataTagProcessor("tagged"), - ModelOverrideProcessor("gpt-4o-mini"), - ], - response_processors=[ContentUpperProcessor()], - ) - result = await chain.call(make_request()) - assert result["choices"][0]["message"]["content"] == "BASE" - - async def test_translator_receives_processed_request_from_compatibility_executor(self): - """Terminal translation uses the post-request-pipeline request.""" - translator = RecordingTranslator() - chain = self._make_chain( - request_processors=[ModelOverrideProcessor("gpt-4o-mini")], - translator=translator, - ) - - await chain.call(make_request(model="gpt-4o")) - - assert translator.request_model == "gpt-4o-mini" - - async def test_python_metadata_survives_across_mixed_rust_and_python_components(self): - """Python metadata is carried through Rust context across mixed components.""" - chain = self._make_chain( - request_processors=[ - MetadataTagProcessor("before_native"), - StatsRequestProcessor(), - MetadataTagProcessor("after_native"), - ], - response_processors=[ - MetadataAssertResponseProcessor( - { - "before_native": True, - "after_native": True, - } - ) - ], - ) - - result = await chain.call(make_request()) - - assert result["choices"][0]["message"]["content"] == "base" - - async def test_backend_adapter_restores_context_after_error(self): - """Context mutations made before Python backend failure are not lost.""" - chain = self._make_chain(backend=FailingBackend()) - ctx = ProxyContext() - - with pytest.raises(RuntimeError, match="backend exploded"): - await chain.call(make_request(model="gpt-4o"), ctx=ctx) - - assert ctx.metadata["backend_started"] == "gpt-4o" - - async def test_request_processor_adapter_restores_context_after_error(self): - chain = self._make_chain(request_processors=[FailingRequestProcessor()]) - ctx = ProxyContext() - - with pytest.raises(RuntimeError, match="request processor exploded"): - await chain.call(make_request(model="gpt-4o"), ctx=ctx) - - assert ctx.metadata["request_started"] == "gpt-4o" - - async def test_response_processor_invalid_return_restores_context(self): - chain = self._make_chain(response_processors=[InvalidResponseProcessor()]) - ctx = ProxyContext() - - with pytest.raises(RuntimeError, match="ChatResponse"): - await chain.call(make_request(model="gpt-4o"), ctx=ctx) - - assert ctx.metadata["response_started"] is True - - async def test_concurrent_calls_do_not_share_context_metadata(self): - chain = self._make_chain( - request_processors=[ModelMetadataProcessor()], - response_processors=[ModelMetadataAssertProcessor()], - ) - - results = await asyncio.gather( - *(chain.call(make_request(model=f"model-{index}")) for index in range(20)) - ) - - assert {result["model"] for result in results} == { - f"model-{index}" for index in range(20) - } - - async def test_streaming_response_survives_compatibility_executor_boundary(self): - chain = self._make_chain(backend=StreamingBackend()) - - stream = await chain.call(make_request()) - events = [event async for event in stream] - - assert events[0]["choices"][0]["delta"]["content"] == "hello" - - async def test_streaming_callbacks_can_read_context_after_executor_returns(self): - processor = ContextReadingStreamTapProcessor() - chain = self._make_chain( - backend=StreamingBackend(), - response_processors=[processor], - ) - - stream = await chain.call(make_request()) - _ = [event async for event in stream] - - assert processor.seen_models == ["stream-selected"] - - async def test_context_overflow_evicts_target_and_retries_fallback(self): - backend = OverflowWeakBackend() - chain = self._make_chain( - request_processors=[PickWeakProcessor()], - backend=backend, - fallback_target_on_evict="strong", - ) - ctx = ProxyContext() - - result = await chain.call(make_request(model="client-model"), ctx=ctx) - - assert result["choices"][0]["message"]["content"] == "fallback" - assert backend.calls == ["weak", "strong"] - assert ctx.selected_target == "strong" - assert ctx.evicted_targets == ["weak"] - - async def test_context_overflow_exception_target_retries_fallback_when_context_unset(self): - backend = ExceptionOnlyOverflowBackend() - chain = self._make_chain( - backend=backend, - fallback_target_on_evict="strong", - ) - ctx = ProxyContext() - - result = await chain.call(make_request(model="client-model"), ctx=ctx) - - assert result["choices"][0]["message"]["content"] == "fallback" - assert backend.calls == [None, "strong"] - assert ctx.selected_target == "strong" - assert ctx.evicted_targets == ["weak"] - - async def test_empty_processors_ok(self): - chain = self._make_chain( - request_processors=[], - response_processors=[], - ) - result = await chain.call(make_request()) - assert result["choices"][0]["message"]["content"] == "base" diff --git a/tests/test_switchyard_app_factory.py b/tests/test_switchyard_app_factory.py deleted file mode 100644 index e77d1e244..000000000 --- a/tests/test_switchyard_app_factory.py +++ /dev/null @@ -1,88 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for the FastAPI app factory wiring.""" - -from __future__ import annotations - -from typing import Protocol - -from fastapi import FastAPI -from fastapi.testclient import TestClient - -from switchyard.lib.endpoints.base import Endpoint -from switchyard.server.switchyard_app import build_switchyard_app - - -class _RequestWithBody(Protocol): - body: dict[str, object] - - -class _RecordingSwitchyard: - def __init__(self) -> None: - self.requests: list[_RequestWithBody] = [] - - async def call( - self, - request: _RequestWithBody, - *, - ctx: object | None = None, - ) -> dict[str, object]: - self.requests.append(request) - return { - "id": "resp-test", - "object": "response", - "model": request.body["model"], - "output": [], - } - - -class _MarkerEndpoint(Endpoint): - def register(self, app: FastAPI) -> None: - @app.get("/marker") - async def marker() -> dict[str, str]: - return {"status": "ok"} - - -class _EndpointContributor: - def get_endpoint(self) -> Endpoint: - return _MarkerEndpoint() - - -class _SwitchyardWithComponents(_RecordingSwitchyard): - def iter_components(self) -> list[_EndpointContributor]: - return [_EndpointContributor()] - - -def test_app_exposes_switchyard_under_endpoint_state_key() -> None: - switchyard = _RecordingSwitchyard() - - app = build_switchyard_app(switchyard) # type: ignore[arg-type] - - assert app.state.switchyard is switchyard - assert app.state.switchyard is switchyard - - -def test_responses_endpoint_uses_app_factory_switchyard() -> None: - switchyard = _RecordingSwitchyard() - app = build_switchyard_app(switchyard) # type: ignore[arg-type] - - with TestClient(app, raise_server_exceptions=False) as client: - response = client.post( - "/v1/responses", - json={"model": "test-model", "input": "ping"}, - ) - - assert response.status_code == 200 - assert response.json()["model"] == "test-model" - assert len(switchyard.requests) == 1 - - -def test_app_registers_component_contributed_endpoints() -> None: - app = build_switchyard_app(_SwitchyardWithComponents()) # type: ignore[arg-type] - - with TestClient(app, raise_server_exceptions=False) as client: - response = client.get("/marker") - - assert response.status_code == 200 - assert response.json() == {"status": "ok"} diff --git a/tests/test_switchyard_app_lifecycle.py b/tests/test_switchyard_app_lifecycle.py deleted file mode 100644 index df73ad5f5..000000000 --- a/tests/test_switchyard_app_lifecycle.py +++ /dev/null @@ -1,49 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for app-level component lifecycle wiring.""" - -from __future__ import annotations - -from fastapi.testclient import TestClient - -from switchyard.server.switchyard_app import build_switchyard_app - - -class _LifecycleSwitchyard: - state_key = "switchyard" - - def __init__(self) -> None: - self.events: list[str] = [] - - def iter_components(self) -> list[object]: - return [_AsyncLifecycleComponent(self.events), _SyncLifecycleComponent(self.events)] - - -class _AsyncLifecycleComponent: - def __init__(self, events: list[str]) -> None: - self._events = events - - async def startup(self) -> None: - self._events.append("async-startup") - - async def shutdown(self) -> None: - self._events.append("async-shutdown") - - -class _SyncLifecycleComponent: - def __init__(self, events: list[str]) -> None: - self._events = events - - def shutdown(self) -> None: - self._events.append("sync-shutdown") - - -def test_build_switchyard_app_runs_component_lifecycle() -> None: - switchyard = _LifecycleSwitchyard() - app = build_switchyard_app(switchyard) # type: ignore[arg-type] - - with TestClient(app): - assert switchyard.events == ["async-startup"] - - assert switchyard.events == ["async-startup", "sync-shutdown", "async-shutdown"] diff --git a/tests/test_switchyard_rust_component_bindings.py b/tests/test_switchyard_rust_component_bindings.py deleted file mode 100644 index b85dcec4d..000000000 --- a/tests/test_switchyard_rust_component_bindings.py +++ /dev/null @@ -1,233 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for direct Rust component bindings.""" - -from __future__ import annotations - -import math - -import pytest - -from switchyard_rust import ( - AnthropicNativeBackend, - BackendFormat, - ChatRequest, - ChatRequestType, - ChatResponse, - EndpointConfig, - LLMBackend, - LlmTarget, - LlmTargetBackend, - MultiLlmBackend, - OpenAiNativeBackend, - ProxyContext, - RandomRoutingProcessorConfig, - StatsAccumulator, - StatsLlmBackend, - StatsRequestProcessor, - StatsResponseProcessor, -) - - -def _target( - target_id: str, - model: str, - *, - format: object = BackendFormat.OPENAI, -) -> LlmTarget: - return LlmTarget( - target_id, - model, - format=format, - endpoint=EndpointConfig(base_url="https://example.test/v1", api_key="test-key"), - ) - - -def test_component_exports_are_callable_processors_and_native_backends() -> None: - openai_target = _target("openai", "gpt-test", format=BackendFormat.OPENAI) - anthropic_target = _target("anthropic", "claude-test", format=BackendFormat.ANTHROPIC) - - assert callable(StatsRequestProcessor().process) - assert callable(StatsResponseProcessor(StatsAccumulator()).process) - assert isinstance(OpenAiNativeBackend(openai_target), LLMBackend) - assert isinstance(AnthropicNativeBackend(anthropic_target), LLMBackend) - - -def test_config_bindings_validate_and_own_values() -> None: - endpoint = EndpointConfig(base_url="https://example.test/v1", api_key="secret", timeout_secs=3.5) - target = LlmTarget( - "target-a", - "model-a", - format="openai", - endpoint=endpoint, - extra_body={"chat_template_kwargs": {"enable_thinking": False}}, - extra_headers={"X-Inference-Priority": "batch"}, - ) - - assert BackendFormat("openai") == BackendFormat.OPENAI - assert BackendFormat.OPENAI == "openai" - assert BackendFormat("responses") == BackendFormat.RESPONSES - assert BackendFormat.RESPONSES == "responses" - assert target.id == "target-a" - assert target.model == "model-a" - assert target.format == BackendFormat.OPENAI - assert target.endpoint.to_dict() == { - "api_key": "secret", - "base_url": "https://example.test/v1", - "timeout_secs": 3.5, - } - assert target.extra_body == {"chat_template_kwargs": {"enable_thinking": False}} - assert target.extra_headers == {"X-Inference-Priority": "batch"} - with pytest.raises(ValueError, match="Unknown backend format"): - BackendFormat("bedrock") - with pytest.raises(ValueError, match="must not be empty"): - LlmTarget(" ", "model-a") - with pytest.raises(ValueError, match="requires a model string"): - LlmTarget(id="target-a") - with pytest.raises(RuntimeError, match="strong_probability"): - RandomRoutingProcessorConfig( - _target("strong", "strong-model"), - _target("weak", "weak-model"), - strong_probability=math.nan, - ) - - -async def test_stats_processors_share_rust_accumulator() -> None: - stats = StatsAccumulator() - request_processor = StatsRequestProcessor() - response_processor = StatsResponseProcessor(stats) - ctx = ProxyContext() - request = ChatRequest.openai_chat({"model": "client-model", "messages": []}) - - processed = await request_processor.process(ctx, request) - ctx.selected_model = "served-model" - response = await response_processor.process( - ctx, - ChatResponse.openai_completion({ - "model": "served-model", - "usage": { - "prompt_tokens": 11, - "completion_tokens": 7, - "prompt_tokens_details": { - "cached_tokens": 3, - "cache_creation_tokens": 2, - }, - "completion_tokens_details": {"reasoning_tokens": 5}, - }, - }), - ) - - assert processed.model == "client-model" - assert response.body["model"] == "served-model" - snapshot = stats.snapshot_sync() - model_stats = snapshot["models"]["served-model"] - assert model_stats["prompt_tokens"] == 11 - assert model_stats["completion_tokens"] == 7 - assert model_stats["cached_tokens"] == 3 - assert model_stats["cache_creation_tokens"] == 2 - assert model_stats["reasoning_tokens"] == 5 - assert model_stats["total_latency"]["count"] == 1 - - -async def test_stream_callbacks_survive_handoff_to_rust_response_processor() -> None: - async def source(): - yield {"choices": [{"delta": {"content": "hi"}}]} - yield { - "choices": [{"delta": {}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 1, "completion_tokens": 2}, - } - - tapped: list[dict[str, object]] = [] - mapped: list[dict[str, object]] = [] - completed = False - - async def tap(event: dict[str, object]) -> None: - tapped.append(dict(event)) - - async def map_event(event: dict[str, object]) -> dict[str, object]: - mapped.append(dict(event)) - return {**event, "mapped": True} - - async def on_complete() -> None: - nonlocal completed - completed = True - - stats = StatsAccumulator() - ctx = ProxyContext() - ctx.selected_model = "served-model" - response = ChatResponse.openai_stream(source()) - response.stream.tap(tap).map(map_event).on_complete(on_complete) - - processed = await StatsResponseProcessor(stats).process(ctx, response) - events = [event async for event in processed.stream] - - assert [event["mapped"] for event in events] == [True, True] - assert len(tapped) == 2 - assert len(mapped) == 2 - assert completed is True - assert stats.snapshot_sync()["models"]["served-model"]["prompt_tokens"] == 1 - - -def test_backend_bindings_construct_without_provider_sdks_or_network() -> None: - openai_target = _target("openai", "gpt-test", format=BackendFormat.OPENAI) - responses_target = _target("responses", "gpt-responses", format=BackendFormat.RESPONSES) - anthropic_target = _target("anthropic", "claude-test", format=BackendFormat.ANTHROPIC) - openai = OpenAiNativeBackend(openai_target) - responses = OpenAiNativeBackend(responses_target) - anthropic = AnthropicNativeBackend(anthropic_target) - stats = StatsAccumulator() - - multi = MultiLlmBackend([ - LlmTargetBackend(openai_target, openai), - (anthropic_target, anthropic), - ], default_target_id="openai") - stats_backend = StatsLlmBackend(openai, stats) - - assert [request_type.value for request_type in openai.supported_request_types] == [ - "openai_chat" - ] - assert [request_type.value for request_type in responses.supported_request_types] == [ - "openai_responses" - ] - assert [request_type.value for request_type in anthropic.supported_request_types] == [ - "anthropic" - ] - assert set(multi.target_ids()) == {"openai", "anthropic"} - assert multi.default_target_id() == "openai" - assert stats_backend.supported_request_types == openai.supported_request_types - - -def test_backend_bindings_reject_invalid_native_composition() -> None: - openai_target = _target("openai", "gpt-test", format=BackendFormat.OPENAI) - openai = OpenAiNativeBackend(openai_target) - - with pytest.raises(RuntimeError, match="requires a target with resolved OpenAI format"): - OpenAiNativeBackend(_target("bad", "claude-test", format=BackendFormat.ANTHROPIC)) - with pytest.raises(RuntimeError, match="requires a target with resolved Anthropic format"): - AnthropicNativeBackend(_target("bad", "gpt-test", format=BackendFormat.OPENAI)) - with pytest.raises(RuntimeError, match="duplicate LLM target id"): - MultiLlmBackend([ - LlmTargetBackend(openai_target, openai), - LlmTargetBackend(openai_target, openai), - ]) - with pytest.raises(RuntimeError, match="default target missing is not configured"): - MultiLlmBackend( - [LlmTargetBackend(openai_target, openai)], - default_target_id="missing", - ) - with pytest.raises(RuntimeError, match="at least one request type"): - MultiLlmBackend([LlmTargetBackend(openai_target, openai)], supported_request_types=[]) - - -def test_wrappers_require_rust_native_backend_instances() -> None: - class PythonOnlyBackend(LLMBackend): - @property - def supported_request_types(self) -> list[ChatRequestType]: - return [ChatRequestType.OPENAI_CHAT] - - async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: - return ChatResponse.openai_completion({"model": request.model}) - - with pytest.raises(TypeError): - StatsLlmBackend(PythonOnlyBackend(), StatsAccumulator()) diff --git a/tests/test_switchyard_rust_core_bindings.py b/tests/test_switchyard_rust_core_bindings.py deleted file mode 100644 index 3ad6f1a72..000000000 --- a/tests/test_switchyard_rust_core_bindings.py +++ /dev/null @@ -1,273 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for Python core values and their native component interop.""" - -from __future__ import annotations - -import pytest - -from switchyard.lib.chat_response import ( - AnthropicResponseStream, - ResponsesApiStream, - ResponseStream, -) -from switchyard_rust import ( - ChatRequest, - ChatRequestType, - ChatResponse, - ChatResponseStream, - ChatResponseType, - LLMBackend, - ProxyContext, - ProxyMetadata, -) - - -def test_openai_chat_request_owns_body_and_exposes_model() -> None: - body = { - "model": "gpt-4o", - "messages": [{"role": "user", "content": "hello"}], - } - - request = ChatRequest.openai_chat(body) - - assert request.request_type == ChatRequestType.OPENAI_CHAT - assert request.request_type is ChatRequestType.OPENAI_CHAT - assert hash(request.request_type) == hash(ChatRequestType.OPENAI_CHAT) - assert request.request_type.value == "openai_chat" - assert request.model == "gpt-4o" - assert request.body == body - assert request.body is not body - - body["model"] = "mutated-after-construction" - assert request.model == "gpt-4o" - - -def test_request_constructors_preserve_wire_format() -> None: - responses = ChatRequest.openai_responses({"model": "gpt-4o", "input": "hello"}) - anthropic = ChatRequest.anthropic({"model": "claude-sonnet-4.5", "messages": []}) - - assert responses.request_type == ChatRequestType.OPENAI_RESPONSES - assert responses.request_type.value == "openai_responses" - assert responses.model == "gpt-4o" - assert anthropic.request_type == ChatRequestType.ANTHROPIC - assert anthropic.request_type.value == "anthropic" - assert anthropic.model == "claude-sonnet-4.5" - - -def test_set_model_mutates_owned_body() -> None: - request = ChatRequest.openai_chat({"model": "old", "messages": []}) - - request.set_model("new") - - assert request.model == "new" - assert request.to_body() == {"model": "new", "messages": []} - - -def test_set_model_recovers_malformed_non_object_body() -> None: - request = ChatRequest.anthropic(["not", "an", "object"]) - - request.set_model("claude-sonnet-4.5") - - assert request.model == "claude-sonnet-4.5" - assert request.body == {"model": "claude-sonnet-4.5"} - - -def test_replace_body_preserves_request_type() -> None: - request = ChatRequest.openai_responses({"model": "old", "input": "hello"}) - - request.replace_body({"model": "new", "input": "replacement"}) - - assert request.request_type == ChatRequestType.OPENAI_RESPONSES - assert request.request_type.value == "openai_responses" - assert request.model == "new" - assert request.body == {"model": "new", "input": "replacement"} - - -def test_non_json_body_is_rejected() -> None: - class NonJsonable: - pass - - with pytest.raises(ValueError): - ChatRequest.openai_chat(NonJsonable()) - - -def test_openai_completion_response_owns_body() -> None: - body = { - "id": "chatcmpl-test", - "model": "gpt-4o", - "choices": [{"message": {"role": "assistant", "content": "hello"}}], - } - - response = ChatResponse.openai_completion(body) - - assert response.response_type == ChatResponseType.OPENAI_COMPLETION - assert response.response_type is ChatResponseType.OPENAI_COMPLETION - assert hash(response.response_type) == hash(ChatResponseType.OPENAI_COMPLETION) - assert response.response_type.value == "openai_completion" - assert response.body == body - assert response.body is not body - - body["model"] = "mutated-after-construction" - assert response.body["model"] == "gpt-4o" - - -def test_buffered_response_preserves_json_null_body() -> None: - response = ChatResponse.openai_completion(None) - - assert response.body is None - assert response.to_body() is None - - -def test_response_constructors_preserve_wire_shape() -> None: - responses = ChatResponse.openai_responses_completion({ - "id": "resp-test", - "model": "gpt-4o", - "output": [], - }) - anthropic = ChatResponse.anthropic_completion({ - "id": "msg-test", - "model": "claude-sonnet-4.5", - "content": [], - }) - - assert responses.response_type == ChatResponseType.OPENAI_RESPONSES_COMPLETION - assert responses.response_type.value == "openai_responses_completion" - assert anthropic.response_type == ChatResponseType.ANTHROPIC_COMPLETION - assert anthropic.response_type.value == "anthropic_completion" - - -async def test_stream_response_uses_owned_async_stream_and_rejects_body_access() -> None: - async def source(): - yield {"delta": "hello"} - - response = ChatResponse.openai_stream(source()) - - assert response.response_type == ChatResponseType.OPENAI_STREAM - assert [event async for event in response.stream] == [{"delta": "hello"}] - with pytest.raises(AttributeError): - _ = response.body - with pytest.raises(ValueError): - response.replace_body({"not": "allowed"}) - - -def test_stream_replace_body_rejects_before_serializing_body() -> None: - class ExplodingBody: - def model_dump(self, **kwargs: object) -> object: - raise RuntimeError("should not serialize streaming replacement") - - response = ChatResponse.openai_stream(object()) - - with pytest.raises(ValueError, match="streaming ChatResponse"): - response.replace_body(ExplodingBody()) - - -async def test_stream_response_supports_taps_maps_and_completion_callbacks() -> None: - async def source(): - yield {"index": 0} - yield {"index": 1} - - tapped: list[dict[str, int]] = [] - completed = False - - async def tap(event: dict[str, int]) -> None: - tapped.append(dict(event)) - - async def map_event(event: dict[str, int]) -> dict[str, int]: - return {"index": event["index"] + 10} - - async def on_complete() -> None: - nonlocal completed - completed = True - - response = ChatResponse.openai_stream(source()) - stream = response.stream.tap(tap).map(map_event).on_complete(on_complete) - - assert [event async for event in stream] == [{"index": 10}, {"index": 11}] - assert tapped == [{"index": 0}, {"index": 1}] - assert completed is True - - with pytest.raises(RuntimeError, match="already been consumed"): - _ = [event async for event in stream] - - -def test_proxy_context_uses_python_wrappers_with_shared_native_state() -> None: - metadata = {"request_id": "client-visible", "nested": {"value": 1}} - - ctx = ProxyContext(metadata=metadata, request_id="rust-request") - - assert ctx.request_id == "rust-request" - assert isinstance(ctx.metadata, ProxyMetadata) - assert ctx.metadata is ctx.metadata - assert ctx.metadata == metadata - assert ctx.metadata is not metadata - - ctx.metadata["new"] = "value" - ctx.metadata.setdefault("order", []).append("first") - ctx.metadata.update({"updated": True}) - ctx.selected_model = "model-a" - ctx.selected_target = "target-a" - ctx.inbound_format = ChatRequestType.OPENAI_CHAT - ctx.backend_call_latency_ms = 42.5 - - assert ctx.metadata["new"] == "value" - assert ctx.metadata["order"] == ["first"] - assert ctx.metadata.get("missing", "fallback") == "fallback" - assert ctx.metadata.copy()["updated"] is True - assert sorted(ctx.metadata.keys()) == ["nested", "new", "order", "request_id", "updated"] - del ctx.metadata["updated"] - assert "updated" not in ctx.metadata - assert ctx.selected_model == "model-a" - assert ctx.selected_target == "target-a" - assert ctx.inbound_format == ChatRequestType.OPENAI_CHAT - assert ctx.backend_call_latency_ms == 42.5 - - ctx.backend_call_latency_ms = None - assert ctx.backend_call_latency_ms is None - - -def test_proxy_context_evicted_targets_are_mutable_from_python() -> None: - ctx = ProxyContext() - - assert ctx.evicted_targets is None - ctx.evicted_targets = ["weak", "strong"] - - assert ctx.evicted_targets == ["strong", "weak"] - - ctx.evicted_targets = None - assert ctx.evicted_targets is None - with pytest.raises(ValueError, match="invalid evicted target"): - ctx.evicted_targets = [" "] - - -def test_provider_stream_adapters_are_rust_chat_response_stream_aliases() -> None: - assert ResponseStream is ChatResponseStream - assert ResponsesApiStream is ChatResponseStream - assert AnthropicResponseStream is ChatResponseStream - - -def test_backend_role_class_is_the_public_python_export() -> None: - from switchyard.lib.roles import LLMBackend as PublicLLMBackend - - assert PublicLLMBackend is LLMBackend - - -async def test_request_response_components_are_method_based() -> None: - class Passthrough: - async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: - return request - - processor = Passthrough() - request = ChatRequest.openai_chat({"model": "gpt-4o", "messages": []}) - - assert await processor.process(ProxyContext(), request) is request - - -def test_replace_body_preserves_response_type() -> None: - response = ChatResponse.anthropic_completion({"model": "old"}) - - response.replace_body({"model": "new", "content": []}) - - assert response.response_type == ChatResponseType.ANTHROPIC_COMPLETION - assert response.body == {"model": "new", "content": []} diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py deleted file mode 100644 index d2238af79..000000000 --- a/tests/test_telemetry.py +++ /dev/null @@ -1,123 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for outbound telemetry version headers.""" - -from __future__ import annotations - -import importlib.metadata -from typing import Any -from unittest.mock import patch - -import pytest - -from switchyard.telemetry import ( - HEADER_NAME, - LEGACY_OPT_OUT_ENVVAR, - OPT_OUT_ENVVAR, - _get_version, - get_telemetry_headers, -) - - -@pytest.fixture(autouse=True) -def _reset_telemetry_env(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv(OPT_OUT_ENVVAR, raising=False) - monkeypatch.delenv(LEGACY_OPT_OUT_ENVVAR, raising=False) - _get_version.cache_clear() - yield - _get_version.cache_clear() - - -def test_get_telemetry_headers_uses_distribution_package_version( - monkeypatch: pytest.MonkeyPatch, -) -> None: - package_names: list[str] = [] - - def fake_version(package_name: str) -> str: - package_names.append(package_name) - return "1.2.3" - - monkeypatch.setattr(importlib.metadata, "version", fake_version) - - assert get_telemetry_headers() == {HEADER_NAME: "1.2.3"} - assert package_names == ["nemo-switchyard"] - - -@pytest.mark.parametrize("envvar", [OPT_OUT_ENVVAR, LEGACY_OPT_OUT_ENVVAR]) -@pytest.mark.parametrize("value", ["1", "true", "yes", "TRUE", "on"]) -def test_get_telemetry_headers_respects_opt_out_envvars( - monkeypatch: pytest.MonkeyPatch, - envvar: str, - value: str, -) -> None: - monkeypatch.setenv(envvar, value) - - assert get_telemetry_headers() == {} - - -@pytest.mark.parametrize("value", ["", "0", "false", "no", " FALSE "]) -def test_get_telemetry_headers_ignores_falsey_opt_out_values( - monkeypatch: pytest.MonkeyPatch, - value: str, -) -> None: - monkeypatch.setattr(importlib.metadata, "version", lambda _name: "9.8.7") - monkeypatch.setenv(OPT_OUT_ENVVAR, value) - - assert get_telemetry_headers() == {HEADER_NAME: "9.8.7"} - - -def test_get_telemetry_headers_falls_back_to_unknown( - monkeypatch: pytest.MonkeyPatch, -) -> None: - def raise_package_not_found(_package_name: str) -> str: - raise importlib.metadata.PackageNotFoundError - - monkeypatch.setattr(importlib.metadata, "version", raise_package_not_found) - - assert get_telemetry_headers() == {HEADER_NAME: "unknown"} - - -def test_openai_llm_client_passes_default_headers( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(importlib.metadata, "version", lambda _name: "1.2.3") - captured: list[dict[str, Any]] = [] - - class FakeAsyncOpenAI: - def __init__(self, **kwargs: Any) -> None: - captured.append(kwargs) - - with patch("switchyard.lib.llm_client.AsyncOpenAI", FakeAsyncOpenAI): - from switchyard.lib.llm_client import OpenAILLMClient - - OpenAILLMClient(api_key="sk-test", base_url="https://llm.test/v1", timeout=3.0) - - assert captured == [ - { - "api_key": "sk-test", - "base_url": "https://llm.test/v1", - "timeout": 3.0, - "default_headers": {HEADER_NAME: "1.2.3"}, - }, - ] - - -def test_openai_llm_client_passes_empty_default_headers_when_opted_out( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv(OPT_OUT_ENVVAR, "1") - captured: list[dict[str, Any]] = [] - - class FakeAsyncOpenAI: - def __init__(self, **kwargs: Any) -> None: - captured.append(kwargs) - - with patch("switchyard.lib.llm_client.AsyncOpenAI", FakeAsyncOpenAI): - from switchyard.lib.llm_client import OpenAILLMClient - - OpenAILLMClient(api_key="sk-test") - - assert captured == [ - {"api_key": "sk-test", "default_headers": {}}, - ] diff --git a/tests/test_tool_result_signal_collector.py b/tests/test_tool_result_signal_collector.py deleted file mode 100644 index 5c5846478..000000000 --- a/tests/test_tool_result_signal_collector.py +++ /dev/null @@ -1,192 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for tool-result signal extraction now built into DimensionCollector.""" - -from __future__ import annotations - -import pytest - -from switchyard_rust.components import DimensionCollector, get_tool_result_signal -from switchyard_rust.core import ChatRequest, ProxyContext - -# ─── unit: classify_text (via Rust) ────────────────────────────────────────── -# DimensionCollector is Rust; test the logic through the full processor path. - - -async def _run_collector(body: dict, fmt: str = "openai_chat") -> ProxyContext: - """Run DimensionCollector.process() and return the populated context.""" - collector = DimensionCollector() - if fmt == "anthropic": - request = ChatRequest.anthropic(body) - elif fmt == "openai_responses": - request = ChatRequest.openai_responses(body) - else: - request = ChatRequest.openai_chat(body) - ctx = ProxyContext() - await collector.process(ctx, request) - return ctx - - -# ─── severity tests via DimensionCollector ──────────────────────────────────── - - -async def test_traceback_stamps_hard_severity(): - ctx = await _run_collector({ - "messages": [ - {"role": "user", "content": "do something"}, - {"role": "tool", "tool_call_id": "1", - "content": "Traceback (most recent call last):\n ValueError"}, - ] - }) - signal = get_tool_result_signal(ctx) - assert signal is not None - assert signal.severity == pytest.approx(0.7) - - -async def test_oom_stamps_critical_severity(): - ctx = await _run_collector({ - "messages": [ - {"role": "tool", "tool_call_id": "1", "content": "Out of memory: kill process"}, - ] - }) - signal = get_tool_result_signal(ctx) - assert signal is not None - assert signal.severity == pytest.approx(1.0) - - -async def test_clean_result_has_zero_severity(): - ctx = await _run_collector({ - "messages": [ - {"role": "tool", "tool_call_id": "1", "content": "file written successfully"}, - ] - }) - signal = get_tool_result_signal(ctx) - assert signal is not None - assert signal.severity == pytest.approx(0.0) - - -async def test_no_tool_results_has_zero_severity(): - ctx = await _run_collector({ - "messages": [{"role": "user", "content": "hello"}] - }) - signal = get_tool_result_signal(ctx) - assert signal is not None - assert signal.severity == pytest.approx(0.0) - - -# ─── conversation metrics ───────────────────────────────────────────────────── - - -async def test_edit_and_write_counts(): - ctx = await _run_collector({ - "messages": [ - {"role": "assistant", "tool_calls": [ - {"function": {"name": "Edit", "arguments": "{}"}}, - {"function": {"name": "Edit", "arguments": "{}"}}, - {"function": {"name": "Write", "arguments": "{}"}}, - ]}, - {"role": "tool", "tool_call_id": "1", "content": "ok"}, - ] - }) - signal = get_tool_result_signal(ctx) - assert signal is not None - assert signal.edit_count == 2 - assert signal.write_count == 1 - - -async def test_no_error_streak_all_clean(): - ctx = await _run_collector({ - "messages": [ - {"role": "tool", "tool_call_id": "1", "content": "ok"}, - {"role": "tool", "tool_call_id": "2", "content": "also ok"}, - ] - }) - signal = get_tool_result_signal(ctx) - assert signal is not None - assert signal.no_error_streak == 2 - - -async def test_no_error_streak_stops_at_error(): - ctx = await _run_collector({ - "messages": [ - {"role": "tool", "tool_call_id": "1", - "content": "Traceback (most recent call last):\n ValueError"}, - {"role": "tool", "tool_call_id": "2", "content": "ok"}, - {"role": "tool", "tool_call_id": "3", "content": "ok"}, - ] - }) - signal = get_tool_result_signal(ctx) - assert signal is not None - assert signal.no_error_streak == 2 - - -async def test_tests_passed_detection(): - ctx = await _run_collector({ - "messages": [ - {"role": "tool", "tool_call_id": "1", - "content": "====== 5 passed in 0.3s ======"}, - ] - }) - signal = get_tool_result_signal(ctx) - assert signal is not None - assert signal.tests_passed is True - - -async def test_tests_passed_false_when_failures_present(): - ctx = await _run_collector({ - "messages": [ - {"role": "tool", "tool_call_id": "1", - "content": "2 failed, 3 passed in 0.5s"}, - ] - }) - signal = get_tool_result_signal(ctx) - assert signal is not None - assert signal.tests_passed is False - - -async def test_turn_depth_matches_message_count(): - ctx = await _run_collector({ - "messages": [ - {"role": "user", "content": "step 1"}, - {"role": "assistant", "content": "ok"}, - {"role": "tool", "tool_call_id": "1", "content": "done"}, - ] - }) - signal = get_tool_result_signal(ctx) - assert signal is not None - assert signal.turn_depth == 3 - - -# ─── Anthropic format ───────────────────────────────────────────────────────── - - -async def test_anthropic_tool_result_extracted(): - ctx = await _run_collector({ - "messages": [ - {"role": "user", "content": [ - {"type": "tool_result", "tool_use_id": "1", - "content": "Traceback (most recent call last):\n ImportError:"} - ]} - ] - }, fmt="anthropic") - signal = get_tool_result_signal(ctx) - assert signal is not None - assert signal.severity == pytest.approx(0.7) - - -# ─── OpenAI Responses format ────────────────────────────────────────────────── - - -async def test_responses_api_tool_output_extracted(): - ctx = await _run_collector({ - "input": [ - {"type": "function_call", "name": "Write"}, - {"type": "function_call_output", "call_id": "1", - "output": "file created"}, - ] - }, fmt="openai_responses") - signal = get_tool_result_signal(ctx) - assert signal is not None - assert signal.severity == pytest.approx(0.0) - assert signal.write_count == 1 diff --git a/tests/test_tracing.py b/tests/test_tracing.py deleted file mode 100644 index bb7f8a72d..000000000 --- a/tests/test_tracing.py +++ /dev/null @@ -1,59 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Unit tests for the optional ddtrace wrapper in :mod:`switchyard.lib.tracing`.""" - -from __future__ import annotations - -from typing import Any - -import pytest - -from switchyard.lib import tracing - - -class _FakeSpan: - def __init__(self) -> None: - self.tags: dict[str, Any] = {} - - def set_tag(self, key: str, value: Any) -> None: - self.tags[key] = value - - def __enter__(self) -> _FakeSpan: - return self - - def __exit__(self, *_args: object) -> bool: - return False - - -class _FakeTracer: - def __init__(self) -> None: - self.started: list[tuple[str, _FakeSpan]] = [] - - def trace(self, name: str) -> _FakeSpan: - span = _FakeSpan() - self.started.append((name, span)) - return span - - -def test_routing_span_noops_without_tracer(monkeypatch: pytest.MonkeyPatch) -> None: - """With no tracer, the context manager still yields a usable no-op span.""" - monkeypatch.setattr(tracing, "_dd_tracer", None) - with tracing.routing_span("switchyard.route_decision") as span: - span.set_tag("switchyard.model", "m") # must not raise - - -def test_routing_span_uses_tracer_when_present(monkeypatch: pytest.MonkeyPatch) -> None: - tracer = _FakeTracer() - monkeypatch.setattr(tracing, "_dd_tracer", tracer) - with tracing.routing_span("switchyard.upstream_attempt") as span: - span.set_tag("switchyard.selected_endpoint", "model-A") - assert tracer.started[0][0] == "switchyard.upstream_attempt" - assert tracer.started[0][1].tags == {"switchyard.selected_endpoint": "model-A"} - - -def test_set_tags_skips_none() -> None: - span = _FakeSpan() - tracing.set_tags(span, {"a": 1, "b": None, "c": "x", "d": False}) - # ``None`` is dropped; falsey-but-meaningful values (0, False, "") are kept. - assert span.tags == {"a": 1, "c": "x", "d": False} diff --git a/tests/test_translation_engine_chaos.py b/tests/test_translation_engine_chaos.py deleted file mode 100644 index dfd133a36..000000000 --- a/tests/test_translation_engine_chaos.py +++ /dev/null @@ -1,2064 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Adversarial / edge-case tests for ChatRequest and ChatResponse translation engines. - -Every test targets a specific boundary condition, failure mode, or surprising -interaction that the happy-path tests in test_request_translation_engine.py and -test_response_translation_engine.py do not cover. Nothing here is hypothetical --- each case was verified to exercise real code paths. -""" - -from __future__ import annotations - -import asyncio -import copy -import json -from dataclasses import dataclass -from typing import Any -from unittest.mock import MagicMock - -import pytest - -from switchyard.lib.chat_response.openai_chat import ResponseStream -from switchyard_rust.core import ChatRequest, ChatRequestType, ChatResponse, request_type_matches -from switchyard_rust.translation import TranslationEngine - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -@pytest.fixture -def req_engine(): - return TranslationEngine() - - -@pytest.fixture -def resp_engine(): - return TranslationEngine() - - -class _AlienChatRequest: - """A request-shaped object unknown to either engine. - - Used to verify that NotImplementedError is raised with a useful message - instead of silently passing through or crashing. - """ - - def __init__(self, body: dict[str, Any] | None = None): - self._body = body or {} - - @property - def request_type(self) -> str: - return "alien" - - @property - def body(self) -> dict[str, Any]: - return self._body - - -def _make_completion_dict( - content: str | None = "Hello!", - tool_calls: list[dict[str, Any]] | None = None, - finish_reason: str = "stop", - model: str = "gpt-4o", - prompt_tokens: int = 10, - completion_tokens: int = 5, -) -> dict[str, Any]: - """Build a plain-dict ChatCompletion-shaped payload.""" - message: dict[str, Any] = {"role": "assistant", "content": content} - if tool_calls is not None: - message["tool_calls"] = tool_calls - return { - "id": "chatcmpl-test", - "model": model, - "choices": [ - {"message": message, "finish_reason": finish_reason}, - ], - "usage": { - "prompt_tokens": prompt_tokens, - "completion_tokens": completion_tokens, - "total_tokens": prompt_tokens + completion_tokens, - }, - } - - -def _mock_completion( - content: str | None = "Hello!", - tool_calls_mock: list[Any] | None = None, - finish_reason: str = "stop", - model: str = "gpt-4o", - prompt_tokens: int = 10, - completion_tokens: int = 5, -) -> MagicMock: - """Build a MagicMock that quacks like ``ChatCompletion``.""" - completion = MagicMock() - completion.choices = [ - MagicMock( - message=MagicMock( - content=content, - tool_calls=tool_calls_mock, - refusal=None, - ), - finish_reason=finish_reason, - ) - ] - completion.usage = MagicMock( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=prompt_tokens + completion_tokens, - ) - completion.model = model - completion.id = "chatcmpl-test" - - tc_dicts = None - if tool_calls_mock: - tc_dicts = [] - for tc in tool_calls_mock: - tc_dicts.append({ - "id": tc.id, - "function": { - "name": tc.function.name, - "arguments": tc.function.arguments, - }, - }) - - completion.model_dump = lambda **kwargs: _make_completion_dict( - content=content, - tool_calls=tc_dicts, - finish_reason=finish_reason, - model=model, - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - ) - return completion - - -def _mock_tool_call( - call_id: str = "call_abc", - name: str = "get_weather", - arguments: str = '{"location":"SF"}', -) -> MagicMock: - tc = MagicMock() - tc.id = call_id - tc.function = MagicMock(name=name, arguments=arguments) - # MagicMock special-cases .name, so set it explicitly - tc.function.name = name - tc.function.arguments = arguments - return tc - - -# Streaming helpers --------------------------------------------------------- - -@dataclass -class MockDelta: - content: str | None = None - tool_calls: Any = None - reasoning: str | None = None - reasoning_content: str | None = None - - -@dataclass -class MockToolCallDelta: - index: int = 0 - id: str | None = None - function: Any = None - - -@dataclass -class MockFunctionDelta: - name: str | None = None - arguments: str | None = None - - -@dataclass -class MockChoice: - delta: MockDelta | None = None - finish_reason: str | None = None - - -@dataclass -class MockChunk: - choices: list[MockChoice] | None = None - usage: Any = None - - -@dataclass -class MockUsage: - prompt_tokens: int = 0 - completion_tokens: int = 0 - total_tokens: int = 0 - - -async def _async_chunks(*chunks: Any): - for c in chunks: - yield c - - -# ========================================================================= -# REQUEST ENGINE EDGE CASES -# ========================================================================= - - -class TestRequestEngineEdgeCases: - """Edge cases that exercise the TranslationEngine.""" - - # -- Empty / minimal bodies ------------------------------------------ - - def test_anthropic_empty_messages(self, req_engine): - """An Anthropic request with an empty messages list should produce - a valid OpenAI request with no messages (except possibly system). - """ - body = {"model": "claude-sonnet-4-20250514", "messages": [], "max_tokens": 100} - req = ChatRequest.anthropic(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - assert request_type_matches(result, ChatRequestType.OPENAI_CHAT) - assert result.body["messages"] == [] - - def test_anthropic_empty_model_string(self, req_engine): - """model="" is falsy -- verify it does NOT get forwarded.""" - body = {"model": "", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 100} - req = ChatRequest.anthropic(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - assert "model" not in result.body - - def test_anthropic_content_none(self, req_engine): - """Content=None in a message should not crash; it maps to empty string.""" - body = { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": None}], - "max_tokens": 100, - } - req = ChatRequest.anthropic(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - msgs = result.body["messages"] - assert len(msgs) == 1 - assert msgs[0]["content"] == "" - - def test_anthropic_content_integer(self, req_engine): - """Content as a non-string, non-list value (int) should be coerced to str.""" - body = { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": 42}], - "max_tokens": 100, - } - req = ChatRequest.anthropic(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - msgs = result.body["messages"] - assert msgs[0]["content"] == "42" - - def test_anthropic_content_boolean(self, req_engine): - """Content as a boolean -- a type confusion that can happen with - malformed requests. Should not crash. - """ - body = { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": True}], - "max_tokens": 100, - } - req = ChatRequest.anthropic(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - msgs = result.body["messages"] - # bool is truthy, not a list, not a str => str(True) = "True" - assert msgs[0]["content"] == "True" - - # -- System prompt variants ------------------------------------------ - - def test_anthropic_structured_system_blocks(self, req_engine): - """Anthropic system as a list of {type:text} blocks should be - concatenated into a single system message while preserving block - boundaries. - """ - body = { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 100, - "system": [ - {"type": "text", "text": "You are helpful."}, - {"type": "text", "text": "Be concise."}, - ], - } - req = ChatRequest.anthropic(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - system_msgs = [m for m in result.body["messages"] if m["role"] == "system"] - assert len(system_msgs) == 1 - assert system_msgs[0]["content"] == "You are helpful.\n\nBe concise." - - def test_anthropic_empty_system_string(self, req_engine): - """system="" is falsy -- should NOT produce a system message.""" - body = { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 100, - "system": "", - } - req = ChatRequest.anthropic(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - system_msgs = [m for m in result.body["messages"] if m["role"] == "system"] - assert len(system_msgs) == 0 - - def test_anthropic_whitespace_only_system(self, req_engine): - """system=" " is truthy -- it WILL produce a system message - (arguably a bug, but this test documents the current behavior). - """ - body = { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 100, - "system": " ", - } - req = ChatRequest.anthropic(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - system_msgs = [m for m in result.body["messages"] if m["role"] == "system"] - assert len(system_msgs) == 1 - assert system_msgs[0]["content"] == " " - - def test_anthropic_system_blocks_with_non_text(self, req_engine): - """Structured system with a non-text block type should silently skip it.""" - body = { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 100, - "system": [ - {"type": "text", "text": "Be helpful."}, - {"type": "image", "source": {"data": "base64..."}}, - ], - } - req = ChatRequest.anthropic(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - system_msgs = [m for m in result.body["messages"] if m["role"] == "system"] - assert len(system_msgs) == 1 - assert system_msgs[0]["content"] == "Be helpful." - - # -- Deeply nested content blocks ------------------------------------ - - def test_anthropic_mixed_text_tool_use_in_one_message(self, req_engine): - """A single assistant message with both text and tool_use content - blocks should produce ONE assistant message with content + tool_calls. - """ - body = { - "model": "claude-sonnet-4-20250514", - "messages": [ - { - "role": "assistant", - "content": [ - {"type": "text", "text": "Let me look that up."}, - { - "type": "tool_use", - "id": "toolu_1", - "name": "search", - "input": {"query": "weather"}, - }, - ], - }, - ], - "max_tokens": 1024, - } - req = ChatRequest.anthropic(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - msgs = result.body["messages"] - assert len(msgs) == 1 - assert msgs[0]["role"] == "assistant" - assert msgs[0]["content"] == "Let me look that up." - assert len(msgs[0]["tool_calls"]) == 1 - assert msgs[0]["tool_calls"][0]["function"]["name"] == "search" - - def test_anthropic_tool_result_with_structured_content(self, req_engine): - """tool_result where content is a list of blocks (text + image) - should flatten text parts and JSON-serialize non-text blocks. - """ - body = { - "model": "claude-sonnet-4-20250514", - "messages": [ - { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "toolu_1", - "content": [ - {"type": "text", "text": "Temperature: 72F"}, - { - "type": "image", - "source": { - "type": "base64", - "data": "iVBORw...", - }, - }, - ], - }, - ], - }, - ], - "max_tokens": 1024, - } - req = ChatRequest.anthropic(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - tool_msgs = [m for m in result.body["messages"] if m["role"] == "tool"] - assert len(tool_msgs) == 1 - # Text is preserved - assert "Temperature: 72F" in tool_msgs[0]["content"] - # Non-text block is JSON-serialized, not dropped - assert "image" in tool_msgs[0]["content"] - - def test_anthropic_content_list_with_non_dict_items(self, req_engine): - """Content blocks that are not dicts (e.g. raw strings in the list) - should be silently skipped, not crash. - """ - body = { - "model": "claude-sonnet-4-20250514", - "messages": [ - { - "role": "user", - "content": ["just a string", 123, {"type": "text", "text": "real block"}], - }, - ], - "max_tokens": 1024, - } - req = ChatRequest.anthropic(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - msgs = result.body["messages"] - assert len(msgs) == 1 - assert msgs[0]["content"] == "real block" - - def test_anthropic_empty_content_list(self, req_engine): - """Content as an empty list should produce a message with empty content.""" - body = { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": []}], - "max_tokens": 100, - } - req = ChatRequest.anthropic(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - msgs = result.body["messages"] - assert len(msgs) == 1 - assert msgs[0]["content"] == "" - - # -- Multi-turn conversations ---------------------------------------- - - def test_twenty_message_conversation(self, req_engine): - """A realistic 20-message conversation with alternating roles, - including system, tool_use, and tool_result, should translate - without loss. - """ - messages = [] - for i in range(10): - messages.append({"role": "user", "content": f"Question {i}"}) - messages.append({"role": "assistant", "content": f"Answer {i}"}) - body = { - "model": "claude-sonnet-4-20250514", - "messages": messages, - "max_tokens": 1024, - "system": "You are a test assistant.", - } - req = ChatRequest.anthropic(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - # 1 system + 20 conversation messages - assert len(result.body["messages"]) == 21 - assert result.body["messages"][0]["role"] == "system" - # Verify message ordering preserved - for i in range(10): - assert result.body["messages"][1 + 2 * i]["content"] == f"Question {i}" - assert result.body["messages"][2 + 2 * i]["content"] == f"Answer {i}" - - def test_multi_turn_with_tool_roundtrip(self, req_engine): - """A conversation with tool_use -> tool_result -> assistant follow-up - should map cleanly to OpenAI assistant(tool_calls) -> tool -> assistant. - """ - messages = [ - {"role": "user", "content": "What's the weather?"}, - { - "role": "assistant", - "content": [ - {"type": "text", "text": "Let me check."}, - { - "type": "tool_use", - "id": "toolu_1", - "name": "get_weather", - "input": {"location": "SF"}, - }, - ], - }, - { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "toolu_1", - "content": "72F and sunny", - }, - ], - }, - {"role": "assistant", "content": "It's 72F and sunny in SF!"}, - ] - body = { - "model": "claude-sonnet-4-20250514", - "messages": messages, - "max_tokens": 1024, - } - req = ChatRequest.anthropic(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - roles = [m["role"] for m in result.body["messages"]] - assert roles == ["user", "assistant", "tool", "assistant"] - assert result.body["messages"][1].get("tool_calls") is not None - - # -- Tool edge cases ------------------------------------------------- - - def test_anthropic_tool_with_empty_input_schema(self, req_engine): - """A tool with input_schema={} should produce parameters={}.""" - body = { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 100, - "tools": [ - {"name": "noop", "description": "Does nothing", "input_schema": {}}, - ], - } - req = ChatRequest.anthropic(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - assert result.body["tools"][0]["function"]["parameters"] == {} - - def test_anthropic_tool_with_no_name(self, req_engine): - """A tool definition missing the 'name' key should be dropped.""" - body = { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 100, - "tools": [{"description": "mystery tool", "input_schema": {}}], - } - req = ChatRequest.anthropic(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - assert "tools" not in result.body - - def test_anthropic_tool_with_unicode_name(self, req_engine): - """Tool names with unicode should pass through without mangling.""" - body = { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 100, - "tools": [ - { - "name": "recherche_meteo", - "description": "Recherche meteo", - "input_schema": {"type": "object", "properties": {}}, - }, - ], - } - req = ChatRequest.anthropic(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - assert result.body["tools"][0]["function"]["name"] == "recherche_meteo" - - def test_anthropic_tool_with_deeply_nested_schema(self, req_engine): - """A deeply nested JSON schema in input_schema should pass through.""" - deep_schema = { - "type": "object", - "properties": { - "config": { - "type": "object", - "properties": { - "nested": { - "type": "object", - "properties": { - "deep": {"type": "string"}, - }, - }, - }, - }, - }, - } - body = { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 100, - "tools": [ - {"name": "deep_tool", "description": "d", "input_schema": deep_schema}, - ], - } - req = ChatRequest.anthropic(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - params = result.body["tools"][0]["function"]["parameters"] - assert params["properties"]["config"]["properties"]["nested"]["properties"]["deep"]["type"] == "string" - - def test_anthropic_tool_use_with_string_input(self, req_engine): - """tool_use block where input is a pre-serialized JSON string - should be used directly without double-encoding. - """ - body = { - "model": "claude-sonnet-4-20250514", - "messages": [ - { - "role": "assistant", - "content": [ - { - "type": "tool_use", - "id": "toolu_1", - "name": "run_cmd", - "input": '{"cmd": "ls -la"}', - }, - ], - }, - ], - "max_tokens": 1024, - } - req = ChatRequest.anthropic(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - args = result.body["messages"][0]["tool_calls"][0]["function"]["arguments"] - # Should be the string itself, NOT json.dumps(string) - assert args == '{"cmd": "ls -la"}' - # Verify it's valid JSON - parsed = json.loads(args) - assert parsed["cmd"] == "ls -la" - - # -- Extra kwargs / pass-through fields ------------------------------ - - def test_anthropic_extra_kwargs_filtered_by_openai_whitelist(self, req_engine): - """Only fields that exist on OpenAI Chat Completions survive the - conversion — Anthropic-only fields (``thinking``, ``cache_control``, - etc.) must be dropped because the OpenAI SDK raises TypeError on - unknown kwargs. Fields that happen to exist on both APIs - (``metadata``) pass through. - """ - body = { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 100, - "metadata": {"user_id": "u123"}, # exists on OpenAI → survives - "thinking": {"type": "enabled", "budget_tokens": 5000}, # Anthropic-only → dropped - } - req = ChatRequest.anthropic(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - assert result.body["metadata"] == {"user_id": "u123"} - assert "thinking" not in result.body - - def test_anthropic_stream_flag_preserved(self, req_engine): - """stream=True in the Anthropic body should appear in the OpenAI body.""" - body = { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 100, - "stream": True, - } - req = ChatRequest.anthropic(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - assert result.body["stream"] is True - - def test_anthropic_stop_sequences_to_stop(self, req_engine): - """stop_sequences should become stop in the OpenAI body.""" - body = { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 100, - "stop_sequences": ["\n\nHuman:", "END"], - } - req = ChatRequest.anthropic(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - assert result.body["stop"] == ["\n\nHuman:", "END"] - - # -- Mutation safety ------------------------------------------------- - - def test_anthropic_passthrough_metadata_does_not_share_nested_objects(self, req_engine): - """Native translation serializes passthrough metadata instead of - exposing source object identity in the translated request. - """ - inner_tags = ["tag1", "tag2"] - body = { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 100, - "metadata": {"tags": inner_tags}, - } - req = ChatRequest.anthropic(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - - result_meta = result.body.get("metadata") - if result_meta is not None: - assert result_meta["tags"] == inner_tags - assert result_meta["tags"] is not inner_tags - - def test_responses_shallow_copy_does_not_mutate_body(self, req_engine): - """Verify the Responses path also shallow-copies.""" - body = {"model": "gpt-4o", "input": "hello", "instructions": "Be nice."} - original = copy.deepcopy(body) - req = ChatRequest.openai_responses(body) - req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - assert req.body == original - - # -- Responses API input variants ------------------------------------ - - def test_responses_empty_input_string(self, req_engine): - """input="" should produce a user message with empty content.""" - body = {"model": "gpt-4o", "input": ""} - req = ChatRequest.openai_responses(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - user_msgs = [m for m in result.body["messages"] if m["role"] == "user"] - assert len(user_msgs) == 1 - assert user_msgs[0]["content"] == "" - - def test_responses_empty_input_list(self, req_engine): - """input=[] should produce an empty messages list.""" - body = {"model": "gpt-4o", "input": []} - req = ChatRequest.openai_responses(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - assert result.body["messages"] == [] - - def test_responses_unknown_input_item_type_is_preserved(self, req_engine): - """Unknown input items should survive as valid Chat text content.""" - body = { - "model": "gpt-4o", - "input": [ - {"type": "message", "role": "user", "content": "hi"}, - {"type": "audio_clip", "data": "base64..."}, # unknown - {"type": "message", "role": "assistant", "content": "hello"}, - ], - } - req = ChatRequest.openai_responses(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - msgs = result.body["messages"] - assert len(msgs) == 3 - assert msgs[0]["content"] == "hi" - assert msgs[1]["content"][0]["type"] == "text" - assert json.loads(msgs[1]["content"][0]["text"]) == { - "type": "audio_clip", - "data": "base64...", - } - assert msgs[2]["content"] == "hello" - - def test_responses_orphan_function_call_output(self, req_engine): - """A function_call_output without a preceding function_call should - not produce an invalid Chat ``tool`` message. - """ - body = { - "model": "gpt-4o", - "input": [ - { - "type": "function_call_output", - "call_id": "call_orphan", - "output": "result", - }, - ], - } - req = ChatRequest.openai_responses(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - assert result.body["messages"] == [ - {"role": "user", "content": "Tool result call_orphan: result"} - ] - - def test_responses_max_output_tokens_to_max_completion_tokens(self, req_engine): - """max_output_tokens should map to Chat's current max token cap.""" - body = {"model": "gpt-4o", "input": "hi", "max_output_tokens": 4096} - req = ChatRequest.openai_responses(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - assert result.body["max_completion_tokens"] == 4096 - assert "max_tokens" not in result.body - assert "max_output_tokens" not in result.body - - def test_responses_tool_with_codex_format(self, req_engine): - """Tools in Codex CLI format (id + inputSchema.jsonSchema) should - be converted correctly. - """ - body = { - "model": "gpt-4o", - "input": "hi", - "tools": [ - { - "type": "function", - "id": "codex_tool", - "inputSchema": { - "jsonSchema": { - "type": "object", - "properties": {"path": {"type": "string"}}, - } - }, - } - ], - } - req = ChatRequest.openai_responses(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - tools = result.body["tools"] - assert len(tools) == 1 - assert tools[0]["function"]["name"] == "codex_tool" - assert tools[0]["function"]["parameters"]["properties"]["path"]["type"] == "string" - - def test_responses_tool_with_empty_name_skipped(self, req_engine): - """Tools with no name AND no id should be skipped.""" - body = { - "model": "gpt-4o", - "input": "hi", - "tools": [ - {"type": "function", "description": "ghost", "parameters": {}}, - {"type": "function", "name": "real_tool", "parameters": {}}, - ], - } - req = ChatRequest.openai_responses(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - tools = result.body["tools"] - assert len(tools) == 1 - assert tools[0]["function"]["name"] == "real_tool" - - def test_responses_multi_turn_tool_roundtrip(self, req_engine): - """function_call + function_call_output sequences should be merged - into assistant(tool_calls) + tool messages. - """ - body = { - "model": "gpt-4o", - "input": [ - {"type": "message", "role": "user", "content": "Search for X"}, - { - "type": "function_call", - "name": "search", - "call_id": "call_1", - "arguments": '{"q":"X"}', - }, - { - "type": "function_call_output", - "call_id": "call_1", - "output": "Found X", - }, - {"type": "message", "role": "assistant", "content": "I found X!"}, - ], - } - req = ChatRequest.openai_responses(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - msgs = result.body["messages"] - roles = [m["role"] for m in msgs] - assert roles == ["user", "assistant", "tool", "assistant"] - assert msgs[1]["tool_calls"][0]["function"]["name"] == "search" - - def test_responses_parallel_tool_calls_merged(self, req_engine): - """Multiple consecutive function_call items (before any output) - should be merged into a single assistant message. - """ - body = { - "model": "gpt-4o", - "input": [ - {"type": "message", "role": "user", "content": "Do two things"}, - { - "type": "function_call", - "name": "tool_a", - "call_id": "call_a", - "arguments": "{}", - }, - { - "type": "function_call", - "name": "tool_b", - "call_id": "call_b", - "arguments": "{}", - }, - { - "type": "function_call_output", - "call_id": "call_a", - "output": "A done", - }, - { - "type": "function_call_output", - "call_id": "call_b", - "output": "B done", - }, - ], - } - req = ChatRequest.openai_responses(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - msgs = result.body["messages"] - # user, assistant(2 tool_calls), tool(a), tool(b) - assert msgs[0]["role"] == "user" - assert msgs[1]["role"] == "assistant" - assert len(msgs[1]["tool_calls"]) == 2 - assert msgs[2]["role"] == "tool" - assert msgs[3]["role"] == "tool" - - def test_responses_intervening_message_deferred_until_after_tool_output(self, req_engine): - """Messages between a function_call and matching output must not - separate Chat tool_calls from their tool result. - """ - body = { - "model": "gpt-4o", - "input": [ - {"type": "message", "role": "user", "content": "Search for X"}, - { - "type": "function_call", - "name": "search", - "call_id": "call_1", - "arguments": "{}", - }, - { - "type": "message", - "role": "assistant", - "content": "I will summarize after the tool.", - }, - { - "type": "function_call_output", - "call_id": "call_1", - "output": "Found X", - }, - ], - } - req = ChatRequest.openai_responses(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - msgs = result.body["messages"] - assert [m["role"] for m in msgs] == ["user", "assistant", "tool", "assistant"] - assert msgs[1]["tool_calls"][0]["id"] == "call_1" - assert msgs[2]["tool_call_id"] == "call_1" - assert msgs[3]["content"] == "I will summarize after the tool." - - def test_responses_chat_compatible_fields_survive_native_translation(self, req_engine): - body = { - "model": "gpt-4o", - "input": "hi", - "metadata": {"trace": "abc"}, - "parallel_tool_calls": False, - "prompt_cache_key": "session-1", - "prompt_cache_retention": "24h", - "safety_identifier": "safe-1", - "service_tier": "flex", - "store": False, - "stream_options": {"include_usage": True}, - "top_logprobs": 2, - "user": "u-123", - } - req = ChatRequest.openai_responses(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - assert result.body["metadata"] == {"trace": "abc"} - assert result.body["parallel_tool_calls"] is False - assert result.body["prompt_cache_key"] == "session-1" - assert result.body["prompt_cache_retention"] == "24h" - assert result.body["safety_identifier"] == "safe-1" - assert result.body["service_tier"] == "flex" - assert result.body["store"] is False - assert result.body["stream_options"] == {"include_usage": True} - assert result.body["top_logprobs"] == 2 - assert result.body["user"] == "u-123" - - def test_responses_json_schema_text_format_maps_to_chat_response_format(self, req_engine): - body = { - "model": "gpt-4o", - "input": "Return JSON", - "text": { - "format": { - "type": "json_schema", - "name": "answer", - "schema": {"type": "object"}, - "strict": True, - } - }, - } - req = ChatRequest.openai_responses(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - assert result.body["response_format"] == { - "type": "json_schema", - "json_schema": { - "name": "answer", - "schema": {"type": "object"}, - "strict": True, - }, - } - - # -- NotImplementedError paths --------------------------------------- - - def test_unknown_request_type_to_openai(self, req_engine): - """An unknown ChatRequest subclass should raise NotImplementedError - with a message that includes the class name. - """ - req = _AlienChatRequest({"model": "x"}) - with pytest.raises(NotImplementedError, match="_AlienChatRequest"): - req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - - def test_unknown_request_type_to_anthropic(self, req_engine): - """An unknown subclass going to_anthropic should mention the class name.""" - req = _AlienChatRequest() - with pytest.raises(NotImplementedError, match="_AlienChatRequest"): - req_engine.request_to(ChatRequestType.ANTHROPIC, req) - - def test_unknown_request_type_to_responses(self, req_engine): - """An unknown subclass going to_responses should raise.""" - req = _AlienChatRequest() - with pytest.raises(NotImplementedError, match="_AlienChatRequest"): - req_engine.request_to(ChatRequestType.OPENAI_RESPONSES, req) - - # -- OpenAI -> Anthropic edge cases ---------------------------------- - - def test_openai_to_anthropic_default_max_tokens(self, req_engine): - """When ``max_tokens`` is omitted, Anthropic requires a default. - - The translation layer injects ``64_000`` to provide room for long - coding outputs without defaulting every request to the model's - absolute output ceiling. - """ - body = { - "model": "gpt-4o", - "messages": [{"role": "user", "content": "hi"}], - } - req = ChatRequest.openai_chat(body) - result = req_engine.request_to(ChatRequestType.ANTHROPIC, req) - assert result.body["max_tokens"] == 64_000 - - def test_openai_to_anthropic_system_extraction(self, req_engine): - """System messages should be extracted into the Anthropic system param.""" - body = { - "model": "gpt-4o", - "messages": [ - {"role": "system", "content": "Be helpful."}, - {"role": "user", "content": "hi"}, - ], - "max_tokens": 100, - } - req = ChatRequest.openai_chat(body) - result = req_engine.request_to(ChatRequestType.ANTHROPIC, req) - assert result.body["system"] == "Be helpful." - # System should NOT appear in messages - for m in result.body["messages"]: - assert m["role"] != "system" - - def test_openai_to_anthropic_tool_calls_in_messages(self, req_engine): - """OpenAI assistant messages with tool_calls should become Anthropic - assistant messages with tool_use content blocks. - """ - body = { - "model": "gpt-4o", - "messages": [ - {"role": "user", "content": "weather?"}, - { - "role": "assistant", - "content": "Checking...", - "tool_calls": [ - { - "id": "call_1", - "function": { - "name": "get_weather", - "arguments": '{"loc": "SF"}', - }, - }, - ], - }, - { - "role": "tool", - "tool_call_id": "call_1", - "content": "72F", - }, - ], - "max_tokens": 100, - } - req = ChatRequest.openai_chat(body) - result = req_engine.request_to(ChatRequestType.ANTHROPIC, req) - msgs = result.body["messages"] - # assistant with tool_use blocks - asst = msgs[1] - assert asst["role"] == "assistant" - assert any(b["type"] == "tool_use" for b in asst["content"]) - # tool result - tool_result = msgs[2] - assert tool_result["role"] == "user" - assert tool_result["content"][0]["type"] == "tool_result" - - def test_openai_to_anthropic_stop_string_to_list(self, req_engine): - """stop as a single string should become stop_sequences=[string].""" - body = { - "model": "gpt-4o", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 100, - "stop": "END", - } - req = ChatRequest.openai_chat(body) - result = req_engine.request_to(ChatRequestType.ANTHROPIC, req) - assert result.body["stop_sequences"] == ["END"] - - def test_openai_to_anthropic_multimodal_content(self, req_engine): - """OpenAI messages with list content (text + image_url) should be - converted to Anthropic content blocks. - """ - body = { - "model": "gpt-4o", - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What's in this image?"}, - {"type": "image_url", "image_url": {"url": "http://example.com/img.png"}}, - ], - }, - ], - "max_tokens": 100, - } - req = ChatRequest.openai_chat(body) - result = req_engine.request_to(ChatRequestType.ANTHROPIC, req) - msg = result.body["messages"][0] - assert isinstance(msg["content"], list) - text_blocks = [b for b in msg["content"] if b.get("type") == "text"] - assert len(text_blocks) == 1 - - -# ========================================================================= -# RESPONSE ENGINE EDGE CASES -# ========================================================================= - - -class TestResponseEngineEdgeCases: - """Edge cases for TranslationEngine.translate().""" - - # -- Empty / degenerate responses ------------------------------------ - - def test_anthropic_empty_choices(self, resp_engine): - """ChatCompletion with choices=[] should produce a valid Anthropic - response with empty text content, not crash. - """ - completion = MagicMock() - completion.model_dump.return_value = { - "id": "chatcmpl-test", - "model": "gpt-4o", - "choices": [], - "usage": {"prompt_tokens": 0, "completion_tokens": 0}, - } - resp = ChatResponse.openai_completion(completion) - req = ChatRequest.anthropic({ - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 1024, - }) - result = resp_engine.response_for_request(req, resp) - assert result["type"] == "message" - assert result["content"][0]["type"] == "text" - assert result["content"][0]["text"] == "" - - def test_responses_empty_choices(self, resp_engine): - """ChatCompletion with choices=[] translated for Responses API - should produce a response with empty output. - """ - completion = MagicMock() - completion.model_dump.return_value = { - "id": "chatcmpl-test", - "model": "gpt-4o", - "choices": [], - "usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}, - } - resp = ChatResponse.openai_completion(completion) - req = ChatRequest.openai_responses({"model": "gpt-4o", "input": "hi"}) - result = resp_engine.response_for_request(req, resp) - assert result["status"] == "completed" - assert result["output"] == [] - - def test_anthropic_multiple_choices_uses_first(self, resp_engine): - """When the completion has multiple choices, only the first one - should be used for the Anthropic response. - """ - completion = MagicMock() - completion.model_dump.return_value = { - "id": "chatcmpl-test", - "model": "gpt-4o", - "choices": [ - { - "message": {"role": "assistant", "content": "first"}, - "finish_reason": "stop", - }, - { - "message": {"role": "assistant", "content": "second"}, - "finish_reason": "stop", - }, - ], - "usage": {"prompt_tokens": 10, "completion_tokens": 5}, - } - resp = ChatResponse.openai_completion(completion) - req = ChatRequest.anthropic({ - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 1024, - }) - result = resp_engine.response_for_request(req, resp) - text = [b for b in result["content"] if b["type"] == "text"] - assert text[0]["text"] == "first" - - def test_anthropic_null_content_no_tool_calls(self, resp_engine): - """content=None and no tool_calls should produce an empty text block.""" - completion = _mock_completion(content=None, tool_calls_mock=None) - resp = ChatResponse.openai_completion(completion) - req = ChatRequest.anthropic({ - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 1024, - }) - result = resp_engine.response_for_request(req, resp) - assert result["type"] == "message" - # Should have at least one content block - assert len(result["content"]) >= 1 - - def test_anthropic_content_and_tool_calls_together(self, resp_engine): - """A response with both text content AND tool_calls should produce - both text and tool_use content blocks. - """ - tc = _mock_tool_call(call_id="call_1", name="search", arguments='{"q":"test"}') - completion = _mock_completion( - content="Let me search for that.", - tool_calls_mock=[tc], - finish_reason="tool_calls", - ) - resp = ChatResponse.openai_completion(completion) - req = ChatRequest.anthropic({ - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "search"}], - "max_tokens": 1024, - }) - result = resp_engine.response_for_request(req, resp) - text_blocks = [b for b in result["content"] if b["type"] == "text"] - tool_blocks = [b for b in result["content"] if b["type"] == "tool_use"] - assert len(text_blocks) == 1 - assert text_blocks[0]["text"] == "Let me search for that." - assert len(tool_blocks) == 1 - assert tool_blocks[0]["name"] == "search" - - def test_anthropic_tool_call_ids_are_sanitized(self, resp_engine): - """OpenAI tool call IDs must become valid Anthropic ``tool_use.id``s.""" - tc = _mock_tool_call( - call_id="call.bad:id/with space", - name="search", - arguments='{"q":"test"}', - ) - completion = _mock_completion( - content=None, - tool_calls_mock=[tc], - finish_reason="tool_calls", - ) - resp = ChatResponse.openai_completion(completion) - req = ChatRequest.anthropic({ - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "search"}], - "max_tokens": 1024, - }) - - result = resp_engine.response_for_request(req, resp) - - tool_blocks = [b for b in result["content"] if b["type"] == "tool_use"] - assert len(tool_blocks) == 1 - assert tool_blocks[0]["id"] == "call_bad_id_with_space" - - def test_anthropic_model_fallback_chain(self, resp_engine): - """When the Anthropic request body has no 'model' key, the response - should use the model from the ChatCompletion response or 'unknown'. - """ - completion = MagicMock() - completion.model_dump.return_value = { - "id": "chatcmpl-test", - "model": "llama-3.1-70b", - "choices": [ - { - "message": {"role": "assistant", "content": "hi"}, - "finish_reason": "stop", - }, - ], - "usage": {"prompt_tokens": 5, "completion_tokens": 1}, - } - resp = ChatResponse.openai_completion(completion) - req = ChatRequest.anthropic({ - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 100, - }) - result = resp_engine.response_for_request(req, resp) - # model=None from request => falls through to response model - assert result["model"] == "llama-3.1-70b" - - def test_anthropic_finish_reason_mapping(self, resp_engine): - """All finish_reason values should map correctly to Anthropic stop_reason.""" - mappings = { - "stop": "end_turn", - "length": "max_tokens", - "tool_calls": "tool_use", - "content_filter": "end_turn", - } - for oai_reason, expected_reason in mappings.items(): - completion = MagicMock() - completion.model_dump.return_value = { - "id": "chatcmpl-test", - "model": "gpt-4o", - "choices": [ - { - "message": {"role": "assistant", "content": "x"}, - "finish_reason": oai_reason, - }, - ], - "usage": {"prompt_tokens": 1, "completion_tokens": 1}, - } - resp = ChatResponse.openai_completion(completion) - req = ChatRequest.anthropic({ - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 100, - }) - result = resp_engine.response_for_request(req, resp) - assert result["stop_reason"] == expected_reason, ( - f"finish_reason={oai_reason!r} should map to {expected_reason!r}" - ) - - def test_responses_content_and_tool_calls(self, resp_engine): - """Responses translation with both content and tool_calls should - produce both message and function_call output items. - """ - completion = MagicMock() - completion.model_dump.return_value = { - "id": "chatcmpl-test", - "model": "gpt-4o", - "choices": [ - { - "message": { - "role": "assistant", - "content": "Let me check.", - "tool_calls": [ - { - "id": "call_1", - "function": { - "name": "search", - "arguments": '{"q":"test"}', - }, - }, - ], - }, - "finish_reason": "tool_calls", - }, - ], - "usage": {"prompt_tokens": 10, "completion_tokens": 15, "total_tokens": 25}, - } - resp = ChatResponse.openai_completion(completion) - req = ChatRequest.openai_responses({"model": "gpt-4o", "input": "search"}) - result = resp_engine.response_for_request(req, resp) - msg_items = [o for o in result["output"] if o["type"] == "message"] - fc_items = [o for o in result["output"] if o["type"] == "function_call"] - assert len(msg_items) == 1 - assert len(fc_items) == 1 - - def test_model_dump_failure_propagates(self, resp_engine): - """If the response's model_dump() raises, the error should propagate - (not be silently swallowed). - """ - class BadCompletion: - def model_dump(self, **kwargs): - del kwargs - raise RuntimeError("serialization failed") - - with pytest.raises(RuntimeError, match="serialization failed"): - ChatResponse.openai_completion(BadCompletion()) - - def test_unknown_request_type_for_response(self, resp_engine): - """An unknown ChatRequest subclass should raise NotImplementedError.""" - completion = _mock_completion() - resp = ChatResponse.openai_completion(completion) - req = _AlienChatRequest() - with pytest.raises(NotImplementedError, match="_AlienChatRequest"): - resp_engine.response_for_request(req, resp) - - def test_unknown_request_type_for_stream(self, resp_engine): - """An unknown ChatRequest subclass in translate_stream should raise.""" - stream = ResponseStream(_async_chunks()) - resp = ChatResponse.openai_stream(stream) - req = _AlienChatRequest() - with pytest.raises(NotImplementedError, match="_AlienChatRequest"): - resp_engine.stream_for_request(req, resp) - - def test_anthropic_tool_call_with_invalid_json_arguments(self, resp_engine): - """Tool call arguments that are not valid JSON should be wrapped - in {"raw": ...} rather than crashing. - """ - tc = _mock_tool_call( - call_id="call_bad", - name="broken_tool", - arguments="this is not json{{{", - ) - completion = _mock_completion( - content=None, - tool_calls_mock=[tc], - finish_reason="tool_calls", - ) - resp = ChatResponse.openai_completion(completion) - req = ChatRequest.anthropic({ - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 100, - }) - result = resp_engine.response_for_request(req, resp) - tool_blocks = [b for b in result["content"] if b["type"] == "tool_use"] - assert len(tool_blocks) == 1 - assert tool_blocks[0]["input"] == {"raw": "this is not json{{{"} - - def test_anthropic_usage_as_dict(self, resp_engine): - """When the response dict has usage as a plain dict (no attributes), - it should still be extracted correctly. - """ - completion = MagicMock() - completion.model_dump.return_value = { - "id": "chatcmpl-test", - "model": "gpt-4o", - "choices": [ - { - "message": {"role": "assistant", "content": "hi"}, - "finish_reason": "stop", - }, - ], - "usage": {"prompt_tokens": 42, "completion_tokens": 7}, - } - resp = ChatResponse.openai_completion(completion) - req = ChatRequest.anthropic({ - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 100, - }) - result = resp_engine.response_for_request(req, resp) - assert result["usage"]["input_tokens"] == 42 - assert result["usage"]["output_tokens"] == 7 - - -# ========================================================================= -# STREAMING EDGE CASES -# ========================================================================= - - -class TestStreamingEdgeCases: - """Edge cases in translate_stream for both Anthropic and Responses paths.""" - - async def test_anthropic_stream_empty_chunks(self, resp_engine): - """Chunks with choices=[] or choices=None should be skipped - without crashing, and the stream should still emit valid - lifecycle events. - """ - chunks = [ - MockChunk(choices=[]), - MockChunk(choices=None), - MockChunk(choices=[MockChoice(delta=MockDelta(content="Hi"))]), - MockChunk(choices=[MockChoice(delta=MockDelta(), finish_reason="stop")]), - ] - stream = ResponseStream(_async_chunks(*chunks)) - resp = ChatResponse.openai_stream(stream) - req = ChatRequest.anthropic({ - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 100, - }) - result = resp_engine.stream_for_request(req, resp) - events = [e async for e in result] - types = [e["type"] for e in events] - assert "message_start" in types - assert "message_stop" in types - # The text "Hi" should be in a content_block_delta - deltas = [e for e in events if e["type"] == "content_block_delta"] - assert any(d["delta"]["text"] == "Hi" for d in deltas) - - async def test_openai_reasoning_stream_deltas_do_not_become_anthropic_text( - self, - resp_engine, - ): - """OpenAI-compatible reasoning deltas are not visible assistant text.""" - chunks = [ - MockChunk(choices=[MockChoice(delta=MockDelta( - reasoning="private reasoning", - reasoning_content="private reasoning content", - ))]), - MockChunk(choices=[MockChoice(delta=MockDelta(content="Visible"))]), - MockChunk(choices=[MockChoice(delta=MockDelta(), finish_reason="stop")]), - ] - stream = ResponseStream(_async_chunks(*chunks)) - resp = ChatResponse.openai_stream(stream) - req = ChatRequest.anthropic({ - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 100, - }) - result = resp_engine.stream_for_request(req, resp) - events = [e async for e in result] - serialized = json.dumps(events) - - assert "Visible" in serialized - assert "private reasoning" in serialized - assert "private reasoning content" in serialized - assert any( - event.get("type") == "content_block_delta" - and event.get("delta", {}).get("type") == "thinking_delta" - for event in events - ) - assert not any( - event.get("type") == "content_block_delta" - and event.get("delta", {}).get("type") == "text_delta" - and "private reasoning" in event.get("delta", {}).get("text", "") - for event in events - ) - - from anthropic.types import ( # noqa: PLC0415 - RawContentBlockDeltaEvent, - RawContentBlockStartEvent, - ) - - for event in events: - if event.get("type") == "content_block_start": - RawContentBlockStartEvent.model_validate(event) - if event.get("type") == "content_block_delta": - RawContentBlockDeltaEvent.model_validate(event) - - async def test_anthropic_stream_only_usage_chunk(self, resp_engine): - """A stream where the only chunk has no choices but has usage - should produce valid message_start, empty content block, message_stop. - """ - chunks = [ - MockChunk(choices=None, usage=MockUsage(prompt_tokens=5, completion_tokens=0)), - ] - stream = ResponseStream(_async_chunks(*chunks)) - resp = ChatResponse.openai_stream(stream) - req = ChatRequest.anthropic({ - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 100, - }) - result = resp_engine.stream_for_request(req, resp) - events = [e async for e in result] - types = [e["type"] for e in events] - assert types[0] == "message_start" - assert types[-1] == "message_stop" - # Should have emitted a minimal empty text block - assert "content_block_start" in types - assert "content_block_stop" in types - - async def test_anthropic_stream_completely_empty(self, resp_engine): - """An empty async iterator (no chunks at all) should still - emit the full Anthropic lifecycle: message_start, empty text block, - message_delta, message_stop. - """ - stream = ResponseStream(_async_chunks()) - resp = ChatResponse.openai_stream(stream) - req = ChatRequest.anthropic({ - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 100, - }) - result = resp_engine.stream_for_request(req, resp) - events = [e async for e in result] - types = [e["type"] for e in events] - assert types[0] == "message_start" - assert types[-1] == "message_stop" - # Empty stream produces a minimal text block - assert "content_block_start" in types - - async def test_anthropic_stream_interleaved_tool_calls(self, resp_engine): - """Tool call chunks that arrive with name in one chunk and arguments - split across multiple chunks should be correctly assembled. - """ - chunks = [ - # First chunk: tool call with name - MockChunk(choices=[MockChoice(delta=MockDelta( - tool_calls=[MockToolCallDelta( - index=0, - id="call_1", - function=MockFunctionDelta(name="get_weather", arguments='{"loc'), - )], - ))]), - # Second chunk: more arguments - MockChunk(choices=[MockChoice(delta=MockDelta( - tool_calls=[MockToolCallDelta( - index=0, - function=MockFunctionDelta(arguments='ation":"SF"}'), - )], - ))]), - # Finish - MockChunk(choices=[MockChoice( - delta=MockDelta(), - finish_reason="tool_calls", - )]), - ] - stream = ResponseStream(_async_chunks(*chunks)) - resp = ChatResponse.openai_stream(stream) - req = ChatRequest.anthropic({ - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "weather?"}], - "max_tokens": 100, - }) - result = resp_engine.stream_for_request(req, resp) - events = [e async for e in result] - - # Should have content_block_start for tool_use - starts = [e for e in events if e["type"] == "content_block_start"] - tool_starts = [s for s in starts if s["content_block"]["type"] == "tool_use"] - assert len(tool_starts) == 1 - assert tool_starts[0]["content_block"]["name"] == "get_weather" - - # Should have input_json_delta events - json_deltas = [ - e for e in events - if e["type"] == "content_block_delta" - and e["delta"].get("type") == "input_json_delta" - ] - assert len(json_deltas) >= 1 - - # Message delta should have stop_reason = tool_use - msg_delta = [e for e in events if e["type"] == "message_delta"] - assert msg_delta[0]["delta"]["stop_reason"] == "tool_use" - - async def test_anthropic_stream_text_then_tool(self, resp_engine): - """A stream with text content followed by a tool call should - close the text block before starting the tool_use block. - """ - chunks = [ - MockChunk(choices=[MockChoice(delta=MockDelta(content="Checking..."))]), - MockChunk(choices=[MockChoice(delta=MockDelta( - tool_calls=[MockToolCallDelta( - index=0, - id="call_1", - function=MockFunctionDelta(name="search", arguments='{"q":"x"}'), - )], - ))]), - MockChunk(choices=[MockChoice( - delta=MockDelta(), finish_reason="tool_calls", - )]), - ] - stream = ResponseStream(_async_chunks(*chunks)) - resp = ChatResponse.openai_stream(stream) - req = ChatRequest.anthropic({ - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "search"}], - "max_tokens": 100, - }) - result = resp_engine.stream_for_request(req, resp) - events = [e async for e in result] - types = [e["type"] for e in events] - - # Text block should be started and stopped before tool block starts - text_stop_idx = types.index("content_block_stop") - # Find tool_use content_block_start - tool_start_idx = None - for i, e in enumerate(events): - if ( - e["type"] == "content_block_start" - and e.get("content_block", {}).get("type") == "tool_use" - ): - tool_start_idx = i - break - assert tool_start_idx is not None - assert text_stop_idx < tool_start_idx - - async def test_responses_stream_empty_chunks(self, resp_engine): - """Responses stream with empty/null choice chunks should still - produce valid lifecycle SSE events. - """ - chunks = [ - MockChunk(choices=[]), - MockChunk(choices=[MockChoice(delta=MockDelta(content="Hi"))]), - MockChunk(choices=[MockChoice(delta=MockDelta(), finish_reason="stop")]), - ] - stream = ResponseStream(_async_chunks(*chunks)) - resp = ChatResponse.openai_stream(stream) - req = ChatRequest.openai_responses({"model": "gpt-4o", "input": "hi"}) - result = resp_engine.stream_for_request(req, resp) - events = [e async for e in result] - all_text = "".join(events) - assert "response.created" in all_text - assert "response.completed" in all_text - assert "Hi" in all_text - - async def test_responses_stream_captures_usage_from_final_chunk(self, resp_engine): - """The usage-only final chunk should be captured in the - response.completed event. - """ - chunks = [ - MockChunk(choices=[MockChoice(delta=MockDelta(content="Hello"))]), - MockChunk( - choices=[MockChoice(delta=MockDelta(), finish_reason="stop")], - ), - # Usage-only final chunk (no choices) - MockChunk( - choices=None, - usage=MockUsage(prompt_tokens=10, completion_tokens=5, total_tokens=15), - ), - ] - stream = ResponseStream(_async_chunks(*chunks)) - resp = ChatResponse.openai_stream(stream) - req = ChatRequest.openai_responses({"model": "gpt-4o", "input": "hi"}) - result = resp_engine.stream_for_request(req, resp) - events = [e async for e in result] - # Find the response.completed event and parse it - for event_str in events: - if "response.completed" in event_str: - # Extract JSON data - for line in event_str.split("\n"): - if line.startswith("data: "): - data = json.loads(line[6:]) - assert data["response"]["usage"]["input_tokens"] == 10 - assert data["response"]["usage"]["output_tokens"] == 5 - break - - async def test_responses_stream_tool_calls(self, resp_engine): - """Responses streaming with tool calls should emit function_call - lifecycle events. - """ - chunks = [ - MockChunk(choices=[MockChoice(delta=MockDelta( - tool_calls=[MockToolCallDelta( - index=0, - id="call_1", - function=MockFunctionDelta(name="search", arguments='{"q":"x"}'), - )], - ))]), - MockChunk(choices=[MockChoice( - delta=MockDelta(), finish_reason="tool_calls", - )]), - ] - stream = ResponseStream(_async_chunks(*chunks)) - resp = ChatResponse.openai_stream(stream) - req = ChatRequest.openai_responses({"model": "gpt-4o", "input": "search"}) - result = resp_engine.stream_for_request(req, resp) - events = [e async for e in result] - all_text = "".join(events) - assert "response.output_item.added" in all_text - assert "function_call" in all_text - assert "response.function_call_arguments.delta" in all_text - - async def test_response_stream_double_consume_raises(self, resp_engine): - """ResponseStream should raise RuntimeError on second iteration.""" - chunks = [ - MockChunk(choices=[MockChoice(delta=MockDelta(content="Hi"))]), - ] - stream = ResponseStream(_async_chunks(*chunks)) - resp = ChatResponse.openai_stream(stream) - req = ChatRequest.anthropic({ - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 100, - }) - result = resp_engine.stream_for_request(req, resp) - _ = [e async for e in result] - # Second consume of the same stream - with pytest.raises(RuntimeError, match="already been consumed"): - _ = [e async for e in resp.stream] - - async def test_anthropic_stream_multiple_tool_calls(self, resp_engine): - """Multiple parallel tool calls should each get their own - content_block_start/stop events with correct block indices. - """ - chunks = [ - # Tool 0 name - MockChunk(choices=[MockChoice(delta=MockDelta( - tool_calls=[MockToolCallDelta( - index=0, - id="call_1", - function=MockFunctionDelta(name="search", arguments=None), - )], - ))]), - # Tool 1 name - MockChunk(choices=[MockChoice(delta=MockDelta( - tool_calls=[MockToolCallDelta( - index=1, - id="call_2", - function=MockFunctionDelta(name="fetch", arguments=None), - )], - ))]), - # Tool 0 args - MockChunk(choices=[MockChoice(delta=MockDelta( - tool_calls=[MockToolCallDelta( - index=0, - function=MockFunctionDelta(arguments='{"q":"test"}'), - )], - ))]), - # Tool 1 args - MockChunk(choices=[MockChoice(delta=MockDelta( - tool_calls=[MockToolCallDelta( - index=1, - function=MockFunctionDelta(arguments='{"url":"http://x"}'), - )], - ))]), - # Finish - MockChunk(choices=[MockChoice( - delta=MockDelta(), finish_reason="tool_calls", - )]), - ] - stream = ResponseStream(_async_chunks(*chunks)) - resp = ChatResponse.openai_stream(stream) - req = ChatRequest.anthropic({ - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "parallel"}], - "max_tokens": 100, - }) - result = resp_engine.stream_for_request(req, resp) - events = [e async for e in result] - - # Two tool_use block starts - tool_starts = [ - e for e in events - if e["type"] == "content_block_start" - and e.get("content_block", {}).get("type") == "tool_use" - ] - assert len(tool_starts) == 2 - names = {s["content_block"]["name"] for s in tool_starts} - assert names == {"search", "fetch"} - # Block indices should be distinct - indices = {s["index"] for s in tool_starts} - assert len(indices) == 2 - - -# ========================================================================= -# CROSS-CUTTING CONCERNS -# ========================================================================= - - -class TestCrossCutting: - """Tests that verify properties across the entire translation pipeline.""" - - def test_idempotency_request_engine(self, req_engine): - """Translating the same request twice should produce equivalent results.""" - body = { - "model": "claude-sonnet-4-20250514", - "messages": [ - {"role": "user", "content": "Hello"}, - {"role": "assistant", "content": "Hi there!"}, - {"role": "user", "content": "How are you?"}, - ], - "max_tokens": 1024, - "system": "Be helpful.", - } - req = ChatRequest.anthropic(body) - r1 = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - r2 = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - assert r1.body["messages"] == r2.body["messages"] - assert r1.body.get("model") == r2.body.get("model") - assert r1.body.get("max_tokens") == r2.body.get("max_tokens") - - def test_idempotency_response_engine(self, resp_engine): - """Translating the same response twice should produce equivalent results - (modulo non-deterministic IDs). - """ - completion = _mock_completion(content="Hello!") - resp = ChatResponse.openai_completion(completion) - req = ChatRequest.anthropic({ - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 100, - }) - r1 = resp_engine.response_for_request(req, resp) - r2 = resp_engine.response_for_request(req, resp) - # IDs are randomly generated, so compare everything else - assert r1["type"] == r2["type"] - assert r1["role"] == r2["role"] - assert r1["content"] == r2["content"] - assert r1["model"] == r2["model"] - assert r1["stop_reason"] == r2["stop_reason"] - assert r1["usage"] == r2["usage"] - - def test_round_trip_openai_to_anthropic_to_openai(self, req_engine): - """OpenAI -> Anthropic -> OpenAI should preserve core semantics: - model, messages content, max_tokens. - """ - original_body = { - "model": "gpt-4o", - "messages": [ - {"role": "system", "content": "Be helpful."}, - {"role": "user", "content": "Hello!"}, - {"role": "assistant", "content": "Hi there!"}, - {"role": "user", "content": "How are you?"}, - ], - "max_tokens": 1024, - "temperature": 0.7, - } - openai_req = ChatRequest.openai_chat(original_body) - - # OpenAI -> Anthropic - anthropic_req = req_engine.request_to(ChatRequestType.ANTHROPIC, openai_req) - assert request_type_matches(anthropic_req, ChatRequestType.ANTHROPIC) - assert anthropic_req.body["model"] == "gpt-4o" - assert anthropic_req.body["system"] == "Be helpful." - - # Anthropic -> OpenAI - round_tripped = req_engine.request_to(ChatRequestType.OPENAI_CHAT, anthropic_req) - assert request_type_matches(round_tripped, ChatRequestType.OPENAI_CHAT) - - # Core semantics preserved - assert round_tripped.body["model"] == "gpt-4o" - rt_msgs = round_tripped.body["messages"] - # System should be first - assert rt_msgs[0]["role"] == "system" - assert rt_msgs[0]["content"] == "Be helpful." - # User messages preserved - user_msgs = [m for m in rt_msgs if m["role"] == "user"] - assert len(user_msgs) == 2 - assert user_msgs[0]["content"] == "Hello!" - assert user_msgs[1]["content"] == "How are you?" - - def test_round_trip_preserves_tools(self, req_engine): - """OpenAI -> Anthropic -> OpenAI should preserve tool definitions.""" - original_body = { - "model": "gpt-4o", - "messages": [{"role": "user", "content": "weather?"}], - "max_tokens": 100, - "tools": [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the weather", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string"}, - }, - "required": ["location"], - }, - }, - }, - ], - } - openai_req = ChatRequest.openai_chat(original_body) - anthropic_req = req_engine.request_to(ChatRequestType.ANTHROPIC, openai_req) - round_tripped = req_engine.request_to(ChatRequestType.OPENAI_CHAT, anthropic_req) - - rt_tools = round_tripped.body.get("tools", []) - assert len(rt_tools) == 1 - assert rt_tools[0]["function"]["name"] == "get_weather" - assert ( - rt_tools[0]["function"]["parameters"]["properties"]["location"]["type"] - == "string" - ) - - async def test_concurrent_translation_no_state_corruption(self, req_engine): - """The engine is documented as stateless. Verify that calling it - from multiple concurrent coroutines doesn't corrupt state. - """ - async def translate_one(i: int) -> ChatRequest: - body = { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": f"msg-{i}"}], - "max_tokens": 100, - "system": f"system-{i}", - } - req = ChatRequest.anthropic(body) - return req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - - results = await asyncio.gather(*(translate_one(i) for i in range(50))) - # Each result should have its own unique content - for i, r in enumerate(results): - msgs = r.body["messages"] - system_msg = [m for m in msgs if m["role"] == "system"][0] - user_msg = [m for m in msgs if m["role"] == "user"][0] - assert system_msg["content"] == f"system-{i}" - assert user_msg["content"] == f"msg-{i}" - - async def test_concurrent_response_translation(self, resp_engine): - """Verify concurrent response translation doesn't corrupt state.""" - async def translate_one(i: int) -> dict: - completion = MagicMock() - completion.model_dump.return_value = { - "id": f"chatcmpl-{i}", - "model": f"model-{i}", - "choices": [ - { - "message": {"role": "assistant", "content": f"response-{i}"}, - "finish_reason": "stop", - }, - ], - "usage": {"prompt_tokens": i, "completion_tokens": i * 2}, - } - resp = ChatResponse.openai_completion(completion) - req = ChatRequest.anthropic({ - "model": f"model-{i}", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 100, - }) - return resp_engine.response_for_request(req, resp) - - results = await asyncio.gather(*(translate_one(i) for i in range(50))) - for i, r in enumerate(results): - text = [b for b in r["content"] if b["type"] == "text"] - assert text[0]["text"] == f"response-{i}" - assert r["usage"]["input_tokens"] == i - assert r["usage"]["output_tokens"] == i * 2 - - def test_engine_instances_are_truly_stateless(self): - """Creating multiple engine instances and using them interleaved - should not cause any interference. - """ - e1 = TranslationEngine() - e2 = TranslationEngine() - body1 = { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "from e1"}], - "max_tokens": 100, - } - body2 = { - "model": "gpt-4o-mini", - "messages": [{"role": "user", "content": "from e2"}], - "max_tokens": 200, - } - r1 = e1.request_to(ChatRequestType.OPENAI_CHAT, ChatRequest.anthropic(body1)) - r2 = e2.request_to(ChatRequestType.OPENAI_CHAT, ChatRequest.anthropic(body2)) - assert r1.body["model"] == "claude-sonnet-4-20250514" - assert r2.body["model"] == "gpt-4o-mini" - - def test_unicode_content_survives_translation(self, req_engine): - """Unicode characters (emoji, CJK, RTL) should survive - Anthropic -> OpenAI translation without mangling. - """ - body = { - "model": "claude-sonnet-4-20250514", - "messages": [ - {"role": "user", "content": "Hello world"}, - {"role": "assistant", "content": "Hola mundo"}, - ], - "max_tokens": 100, - "system": "multilingual assistant", - } - req = ChatRequest.anthropic(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - msgs = result.body["messages"] - assert msgs[1]["content"] == "Hello world" - assert msgs[2]["content"] == "Hola mundo" - - def test_very_long_content_no_truncation(self, req_engine): - """A message with very long content (100K chars) should not be - truncated by the translation layer. - """ - long_text = "x" * 100_000 - body = { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": long_text}], - "max_tokens": 100, - } - req = ChatRequest.anthropic(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - assert len(result.body["messages"][0]["content"]) == 100_000 - - def test_responses_to_openai_passthrough_params(self, req_engine): - """temperature, top_p, and stream should pass through from - Responses API to Chat Completions. - """ - body = { - "model": "gpt-4o", - "input": "hi", - "temperature": 0.5, - "top_p": 0.9, - "stream": True, - } - req = ChatRequest.openai_responses(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - assert result.body["temperature"] == 0.5 - assert result.body["top_p"] == 0.9 - assert result.body["stream"] is True - - def test_anthropic_tool_choice_variants(self, req_engine): - """All Anthropic tool_choice variants should map correctly.""" - # String variants - for ant_choice, expected in [("auto", "auto"), ("any", "required"), ("none", "none")]: - body = { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 100, - "tools": [{"name": "t", "description": "d", "input_schema": {}}], - "tool_choice": ant_choice, - } - req = ChatRequest.anthropic(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - assert result.body["tool_choice"] == expected, ( - f"tool_choice={ant_choice!r} should map to {expected!r}" - ) - - def test_anthropic_tool_choice_specific_tool(self, req_engine): - """Anthropic tool_choice of type 'tool' with a name should map - to OpenAI's function-specific tool_choice. - """ - body = { - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 100, - "tools": [{"name": "search", "description": "d", "input_schema": {}}], - "tool_choice": {"type": "tool", "name": "search"}, - } - req = ChatRequest.anthropic(body) - result = req_engine.request_to(ChatRequestType.OPENAI_CHAT, req) - tc = result.body["tool_choice"] - assert tc["type"] == "function" - assert tc["function"]["name"] == "search" diff --git a/tests/test_upstream_error_log.py b/tests/test_upstream_error_log.py deleted file mode 100644 index 7929fe75a..000000000 --- a/tests/test_upstream_error_log.py +++ /dev/null @@ -1,132 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Unit tests for the structured per-attempt upstream-failure log.""" - -from __future__ import annotations - -import json -import logging -from datetime import datetime - -import pytest - -from switchyard.lib.endpoints import upstream_error_log -from switchyard.lib.endpoints.upstream_error_log import ( - EVENT_NAME, - log_upstream_attempt_failure, -) - -_LOGGER_NAME = "switchyard.upstream_errors" - - -def _emit_and_parse( - caplog: pytest.LogCaptureFixture, - *, - model: str, - attempt: int, - status_code: int | None, - error: BaseException, -) -> dict[str, object]: - """Call the logger and return the single emitted record parsed as JSON.""" - caplog.clear() - with caplog.at_level(logging.WARNING, logger=_LOGGER_NAME): - log_upstream_attempt_failure( - model=model, attempt=attempt, status_code=status_code, error=error - ) - records = [r for r in caplog.records if r.name == _LOGGER_NAME] - assert len(records) == 1, "expected exactly one structured record" - assert records[0].levelno == logging.WARNING - return json.loads(records[0].getMessage()) - - -class TestRecordShape: - def test_message_is_valid_json_with_expected_keys( - self, caplog: pytest.LogCaptureFixture - ) -> None: - rec = _emit_and_parse( - caplog, model="m", attempt=1, status_code=500, error=RuntimeError("boom") - ) - assert set(rec) == { - "event", - "timestamp", - "model", - "upstream_model", - "attempt", - "status_code", - "code", - "outcome", - "error_source", - "error_type", - "error", - } - assert rec["event"] == EVENT_NAME == "upstream_attempt_failed" - assert rec["model"] == "m" - # No upstream_model supplied → logged as null; source is constant. - assert rec["upstream_model"] is None - assert rec["error_source"] == "provider" - assert rec["attempt"] == 1 - assert rec["error_type"] == "RuntimeError" - assert rec["error"] == "boom" - - def test_timestamp_is_iso8601_and_utc( - self, caplog: pytest.LogCaptureFixture - ) -> None: - rec = _emit_and_parse( - caplog, model="m", attempt=1, status_code=429, error=ValueError("x") - ) - # Round-trips through fromisoformat and carries a UTC offset. - parsed = datetime.fromisoformat(str(rec["timestamp"])) - assert parsed.utcoffset() is not None - assert parsed.utcoffset().total_seconds() == 0 # type: ignore[union-attr] - - -class TestCodeAndOutcomeMirrorTheMetric: - def test_none_is_network_failure(self, caplog: pytest.LogCaptureFixture) -> None: - rec = _emit_and_parse( - caplog, model="m", attempt=2, status_code=None, - error=ConnectionError("reset"), - ) - assert rec["status_code"] is None - assert rec["code"] == "none" - assert rec["outcome"] == "retryable_error" - - @pytest.mark.parametrize("code", [429, 500, 504]) - def test_retryable_codes( - self, caplog: pytest.LogCaptureFixture, code: int - ) -> None: - rec = _emit_and_parse( - caplog, model="m", attempt=1, status_code=code, error=RuntimeError("e") - ) - assert rec["status_code"] == code - assert rec["code"] == str(code) - assert rec["outcome"] == "retryable_error" - - def test_other_error_code(self, caplog: pytest.LogCaptureFixture) -> None: - rec = _emit_and_parse( - caplog, model="m", attempt=1, status_code=401, error=RuntimeError("e") - ) - assert rec["status_code"] == 401 - assert rec["code"] == "401" - assert rec["outcome"] == "other_error" - - def test_unknown_code_keeps_raw_status_but_clamps_label( - self, caplog: pytest.LogCaptureFixture - ) -> None: - """The raw code is preserved for audit; the joinable label is clamped.""" - rec = _emit_and_parse( - caplog, model="m", attempt=1, status_code=418, error=RuntimeError("e") - ) - assert rec["status_code"] == 418 - assert rec["code"] == "4xx" - assert rec["outcome"] == "other_error" - - -class TestErrorTruncation: - def test_long_error_is_capped(self, caplog: pytest.LogCaptureFixture) -> None: - rec = _emit_and_parse( - caplog, model="m", attempt=1, status_code=500, - error=RuntimeError("x" * 5000), - ) - assert isinstance(rec["error"], str) - assert len(rec["error"]) == upstream_error_log._MAX_ERROR_CHARS diff --git a/tests/test_upstream_error_passthrough.py b/tests/test_upstream_error_passthrough.py deleted file mode 100644 index ac61dfe48..000000000 --- a/tests/test_upstream_error_passthrough.py +++ /dev/null @@ -1,317 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""End-to-end gate that upstream HTTP errors keep stable provider details. - -Before this fix, a 401 from the upstream LLM (typically a bad API key -or expired credential) became a generic 500 at the client because the -compatibility executor wrapped the Python ``openai.APIStatusError`` in a -``SwitchyardError::Backend(error.to_string())``, which surfaced as a -plain Python ``RuntimeError`` and FastAPI defaulted it to 500. - -Python backends stash upstream status/body on ``ProxyContext.metadata``. -Rust backends surface a typed upstream exception with ``status_code`` and -``body`` attributes. The endpoints recover either signal, preserve the -HTTP status plus stable provider fields, and return the normalized -Switchyard error envelope instead of FastAPI's default plain-text 500. -""" - -from __future__ import annotations - -import json - -from fastapi.testclient import TestClient - -from switchyard.cli.route_bundle import build_route_bundle_table -from switchyard.lib.endpoints.error_envelope import ERROR_SOURCE_HEADER -from switchyard.lib.endpoints.upstream_error import ( - internal_chain_error_response, - upstream_response_from_ctx, -) -from switchyard.lib.proxy_context import ( - CTX_UPSTREAM_HTTP_BODY, - CTX_UPSTREAM_HTTP_STATUS, - ProxyContext, -) -from switchyard.server.switchyard_app import build_switchyard_app -from tests._chain_test_helpers import _OpenAICompatStub - -# --------------------------------------------------------------------------- -# Helper unit tests -# --------------------------------------------------------------------------- - - -class TestUpstreamResponseFromCtx: - def test_returns_none_when_no_status_recorded(self) -> None: - ctx = ProxyContext() - assert upstream_response_from_ctx(ctx) is None - - def test_wraps_string_body_in_error_envelope(self) -> None: - ctx = ProxyContext() - ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] = 401 - ctx.metadata[CTX_UPSTREAM_HTTP_BODY] = "Unauthorized" - - response = upstream_response_from_ctx(ctx) - - assert response is not None - assert response.status_code == 401 - - def test_normalizes_dict_body_into_switchyard_envelope(self) -> None: - ctx = ProxyContext() - ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] = 429 - ctx.metadata[CTX_UPSTREAM_HTTP_BODY] = { - "error": {"message": "rate limited", "type": "rate_limit"}, - } - - response = upstream_response_from_ctx(ctx) - - assert response is not None - assert response.status_code == 429 - body = json.loads(response.body) - assert body == { - "error": { - "message": "rate limited", - "type": "rate_limit", - "code": "rate_limit", - } - } - - def test_preserves_provider_error_param_when_present(self) -> None: - ctx = ProxyContext() - ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] = 400 - ctx.metadata[CTX_UPSTREAM_HTTP_BODY] = { - "error": { - "message": "bad input", - "type": "invalid_request_error", - "code": "invalid_value", - "param": "messages.0.role", - }, - } - - response = upstream_response_from_ctx(ctx) - - assert response is not None - assert response.status_code == 400 - assert json.loads(response.body) == { - "error": { - "message": "bad input", - "type": "invalid_request_error", - "code": "invalid_value", - "param": "messages.0.role", - } - } - - def test_synthesizes_envelope_when_body_missing(self) -> None: - ctx = ProxyContext() - ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] = 503 - - response = upstream_response_from_ctx(ctx) - - assert response is not None - assert response.status_code == 503 - body = json.loads(response.body) - assert body == { - "error": { - "message": "upstream returned HTTP 503", - "type": "upstream_error", - "code": "upstream_error", - } - } - - def test_ignores_non_int_status(self) -> None: - """Defensive: a stray non-int value must not crash the error path.""" - ctx = ProxyContext() - ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] = "401" # wrong type — ignore - - assert upstream_response_from_ctx(ctx) is None - - def test_internal_chain_error_uses_openai_error_envelope(self) -> None: - response = internal_chain_error_response(RuntimeError("connection refused"), "openai") - - assert response.status_code == 500 - assert response.headers["content-type"].startswith("application/json") - body = json.loads(response.body) - assert "connection refused" in body["error"]["message"] - assert body["error"]["type"] == "internal_error" - assert body["error"]["code"] == "internal_chain_error" - - def test_internal_chain_error_uses_same_envelope_for_anthropic_inbound(self) -> None: - response = internal_chain_error_response(RuntimeError("connection refused"), "anthropic") - - assert response.status_code == 500 - assert response.headers["content-type"].startswith("application/json") - body = json.loads(response.body) - assert body["error"]["type"] == "internal_error" - assert body["error"]["code"] == "internal_chain_error" - assert "connection refused" in body["error"]["message"] - - def test_internal_chain_error_truncates_long_repr(self) -> None: - long_msg = "x" * 500 - response = internal_chain_error_response(RuntimeError(long_msg), "openai") - body = json.loads(response.body) - assert len(body["error"]["message"]) <= 200 - - -# --------------------------------------------------------------------------- -# Anthropic and Responses endpoints share the same helper -# --------------------------------------------------------------------------- - - -def test_rust_openai_route_upstream_401_returns_structured_openai_error() -> None: - """Rust OpenAI-native backend errors must not fall through to FastAPI's 500.""" - with _OpenAICompatStub() as upstream: - upstream.respond_json( - {"error": {"message": "bad key", "type": "invalid_api_key"}}, - status=401, - ) - table = build_route_bundle_table({ - "defaults": { - "api_key": "bad-key", - "base_url": upstream.base_url, - "format": "openai", - }, - "routes": { - "bad-key": { - "type": "passthrough", - "target": "nvidia/nvidia/nemotron-nano-9b-v2", - } - }, - }) - - with TestClient(build_switchyard_app(table), raise_server_exceptions=False) as client: - response = client.post( - "/v1/chat/completions", - json={ - "model": "bad-key", - "messages": [{"role": "user", "content": "ping"}], - }, - ) - - assert response.status_code == 401 - assert response.json() == { - "error": { - "message": "bad key", - "type": "invalid_api_key", - "code": "invalid_api_key", - }, - } - - -def test_rust_openai_route_upstream_401_returns_same_error_shape_for_anthropic_inbound() -> None: - """Anthropic inbound clients should receive the same HTTP error envelope.""" - with _OpenAICompatStub() as upstream: - upstream.respond_json( - {"error": {"message": "bad key", "type": "invalid_api_key"}}, - status=401, - ) - table = build_route_bundle_table({ - "defaults": { - "api_key": "bad-key", - "base_url": upstream.base_url, - "format": "openai", - }, - "routes": { - "bad-key": { - "type": "passthrough", - "target": "nvidia/nvidia/nemotron-nano-9b-v2", - } - }, - }) - - with TestClient(build_switchyard_app(table), raise_server_exceptions=False) as client: - response = client.post( - "/v1/messages", - json={ - "model": "bad-key", - "max_tokens": 16, - "messages": [{"role": "user", "content": "ping"}], - }, - ) - - assert response.status_code == 401 - assert response.json() == { - "error": { - "message": "bad key", - "type": "invalid_api_key", - "code": "invalid_api_key", - }, - } - - -def test_rust_openai_route_upstream_401_returns_same_error_shape_for_responses_inbound() -> None: - """Responses inbound clients should receive the same HTTP error envelope.""" - with _OpenAICompatStub() as upstream: - upstream.respond_json( - {"error": {"message": "bad key", "type": "invalid_api_key"}}, - status=401, - ) - table = build_route_bundle_table({ - "defaults": { - "api_key": "bad-key", - "base_url": upstream.base_url, - "format": "openai", - }, - "routes": { - "bad-key": { - "type": "passthrough", - "target": "nvidia/nvidia/nemotron-nano-9b-v2", - } - }, - }) - - with TestClient(build_switchyard_app(table), raise_server_exceptions=False) as client: - response = client.post( - "/v1/responses", - json={ - "model": "bad-key", - "input": "ping", - }, - ) - - assert response.status_code == 401 - assert response.json() == { - "error": { - "message": "bad key", - "type": "invalid_api_key", - "code": "invalid_api_key", - }, - } - - -# --------------------------------------------------------------------------- -# Failure-source headers on the wire -# --------------------------------------------------------------------------- -# The header tests above the endpoint layer call helpers directly; these prove -# the annotation actually survives the real FastAPI handlers + middleware to -# the HTTP response a client receives. - - -def test_model_not_found_404_header_labels_switchyard_on_the_wire() -> None: - """RouteTable dispatch 404s carry the switchyard source header.""" - with _OpenAICompatStub() as upstream: - table = build_route_bundle_table({ - "defaults": { - "api_key": "k", - "base_url": upstream.base_url, - "format": "openai", - }, - "routes": { - "registered": { - "type": "passthrough", - "target": "nvidia/nvidia/nemotron-nano-9b-v2", - } - }, - }) - - with TestClient(build_switchyard_app(table), raise_server_exceptions=False) as client: - response = client.post( - "/v1/chat/completions", - json={ - "model": "no-such-model", - "messages": [{"role": "user", "content": "hi"}], - }, - ) - - assert response.status_code == 404 - assert response.json()["error"]["code"] == "model_not_found" - assert response.headers[ERROR_SOURCE_HEADER] == "switchyard" diff --git a/tests/translation/__init__.py b/tests/translation/__init__.py deleted file mode 100644 index 52a7a9daf..000000000 --- a/tests/translation/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 diff --git a/tests/translation/test_format_fidelity_contract.py b/tests/translation/test_format_fidelity_contract.py deleted file mode 100644 index 15c403ebd..000000000 --- a/tests/translation/test_format_fidelity_contract.py +++ /dev/null @@ -1,148 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Format-fidelity contract: AUTO probes upstream in priority order. - -BackendFormat.AUTO probes: - 1. /v1/chat/completions → OPENAI - 2. /v1/messages → ANTHROPIC - 3. /v1/responses → RESPONSES - 4. fallback → OPENAI (Chat Completions assumed universal) - -The TranslationEngine converts any inbound format to any backend format -through a neutral IR, so all combinations are valid regardless of client. -""" - -import pytest - -from switchyard.lib.backends import backend_format_resolver as resolver_mod -from switchyard.lib.backends.llm_target import BackendFormat, LlmTarget -from switchyard.lib.backends.multi_llm_backend import resolve_llm_target - - -class TestAutoFormatResolution: - """BackendFormat.AUTO probes upstream capabilities and selects the best format.""" - - def test_auto_resolves_to_responses_for_openai_upstream( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - """Chat Completions fails, /v1/messages fails, /v1/responses succeeds → RESPONSES.""" - monkeypatch.setattr( - resolver_mod, "probe_openai_chat_completions_support_sync", - lambda *, base_url, api_key, **_kw: False, - ) - monkeypatch.setattr( - resolver_mod, "probe_anthropic_messages_support_sync", - lambda *, base_url, api_key, **_kw: False, - ) - monkeypatch.setattr( - resolver_mod, "probe_openai_responses_support_sync", - lambda *, base_url, api_key, **_kw: True, - ) - - target = LlmTarget( - model="openai/gpt-5.2", - format=BackendFormat.AUTO, - base_url="https://api.openai.com/v1", - api_key="sk-test", # pragma: allowlist secret - ) - resolved = resolve_llm_target(target) - - assert resolved.format is BackendFormat.RESPONSES - - def test_auto_falls_back_to_openai_for_nim_upstream( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - """NIM upstream: Chat Completions succeeds → OPENAI (first probe wins).""" - monkeypatch.setattr( - resolver_mod, "probe_openai_chat_completions_support_sync", - lambda *, base_url, api_key, **_kw: True, - ) - monkeypatch.setattr( - resolver_mod, "probe_anthropic_messages_support_sync", - lambda *, base_url, api_key, **_kw: pytest.fail("should not probe Anthropic"), - ) - monkeypatch.setattr( - resolver_mod, "probe_openai_responses_support_sync", - lambda *, base_url, api_key, **_kw: pytest.fail("should not probe Responses"), - ) - - target = LlmTarget( - model="nvidia/nvidia/nemotron-nano-9b-v2", - format=BackendFormat.AUTO, - base_url="https://integrate.api.nvidia.com/v1", - api_key="sk-test", # pragma: allowlist secret - ) - resolved = resolve_llm_target(target) - - assert resolved.format is BackendFormat.OPENAI - - def test_auto_resolves_to_anthropic_for_messages_upstream( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - """Chat Completions fails, /v1/messages succeeds → ANTHROPIC.""" - monkeypatch.setattr( - resolver_mod, "probe_openai_chat_completions_support_sync", - lambda *, base_url, api_key, **_kw: False, - ) - monkeypatch.setattr( - resolver_mod, "probe_anthropic_messages_support_sync", - lambda *, base_url, api_key, **_kw: True, - ) - - target = LlmTarget( - model="some-non-prefixed-model", - format=BackendFormat.AUTO, - base_url="https://api.anthropic.com/v1", - api_key="sk-test", # pragma: allowlist secret - ) - resolved = resolve_llm_target(target) - - assert resolved.format is BackendFormat.ANTHROPIC - - def test_auto_prefix_fast_path_skips_all_probes( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - """anthropic/ and claude prefixes → ANTHROPIC immediately, no probes fired.""" - monkeypatch.setattr( - resolver_mod, "probe_openai_chat_completions_support_sync", - lambda **_kw: pytest.fail("prefix fast-path must not probe"), - ) - monkeypatch.setattr( - resolver_mod, "probe_anthropic_messages_support_sync", - lambda **_kw: pytest.fail("prefix fast-path must not probe"), - ) - monkeypatch.setattr( - resolver_mod, "probe_openai_responses_support_sync", - lambda **_kw: pytest.fail("prefix fast-path must not probe"), - ) - - target = LlmTarget( - model="anthropic/claude-sonnet-4-5", - format=BackendFormat.AUTO, - base_url="https://api.anthropic.com/v1", - api_key="sk-test", # pragma: allowlist secret - ) - resolved = resolve_llm_target(target) - - assert resolved.format is BackendFormat.ANTHROPIC - - def test_explicit_formats_bypass_probe( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - """Explicit RESPONSES/OPENAI/ANTHROPIC are honored as-is without probing.""" - def fail_probe(**_kw: object) -> bool: - pytest.fail("explicit formats must not trigger a probe") - - monkeypatch.setattr(resolver_mod, "probe_openai_chat_completions_support_sync", fail_probe) - monkeypatch.setattr(resolver_mod, "probe_anthropic_messages_support_sync", fail_probe) - monkeypatch.setattr(resolver_mod, "probe_openai_responses_support_sync", fail_probe) - - for fmt in (BackendFormat.RESPONSES, BackendFormat.OPENAI, BackendFormat.ANTHROPIC): - target = LlmTarget( - model="some/model", - format=fmt, - base_url="https://api.openai.com/v1", - api_key="sk-test", # pragma: allowlist secret - ) - assert resolve_llm_target(target).format is fmt diff --git a/uv.lock b/uv.lock index 0c7d61866..8a1234ca6 100644 --- a/uv.lock +++ b/uv.lock @@ -141,25 +141,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, ] -[[package]] -name = "anthropic" -version = "0.99.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "distro" }, - { name = "docstring-parser" }, - { name = "httpx" }, - { name = "jiter" }, - { name = "pydantic" }, - { name = "sniffio" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0d/c9/e8a3a1caeab575e80551b30b084096b5a430abc52739a526a1daaadd038c/anthropic-0.99.0.tar.gz", hash = "sha256:16f41e00f215ed2d193b146be3dd567c4319c32ed3af6c8725d68ba875257c1c", size = 727239, upload-time = "2026-05-05T16:03:07.986Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/84/d0917506744e1707cf55659a57f1e3ff952eda5636df0ffffe3e884b7c61/anthropic-0.99.0-py3-none-any.whl", hash = "sha256:c44469b746ab2ef19a4c52dcbdb98e17bc95c60bebdd18ec40d76d2d23592b49", size = 700564, upload-time = "2026-05-05T16:03:06.059Z" }, -] - [[package]] name = "anyio" version = "4.13.0" @@ -204,15 +185,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3e/5c/fb93d3092640a24dfb7bd7727a24016d7c01774ca013e60efd3f683c8002/backrefs-7.0-py314-none-any.whl", hash = "sha256:a6448b28180e3ca01134c9cf09dcebafad8531072e09903c5451748a05f24bc9", size = 412349, upload-time = "2026-04-28T16:28:02.412Z" }, ] -[[package]] -name = "bytecode" -version = "0.18.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/27/8f/7d12c539869a5cbd801d550b86cc0f030ecaeb12f57f8b3ff19f2d2a184c/bytecode-0.18.1.tar.gz", hash = "sha256:d9564f1565fe1ae6a1173e544ef43a85f093e83997ef45af65d0d250eb48d7a1", size = 104631, upload-time = "2026-06-03T14:17:59.115Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/ef/6a629424ec08adc3819ddfd7ec0a710361eaa29d1e5fffb4e02f074be5c9/bytecode-0.18.1-py3-none-any.whl", hash = "sha256:9535bfdd665260b2888ec4121569e3ca5106965a7fedbb6de6ba1bafebc5c7d7", size = 42868, upload-time = "2026-06-03T14:17:57.654Z" }, -] - [[package]] name = "cachetools" version = "6.2.6" @@ -400,90 +372,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] -[[package]] -name = "coverage" -version = "7.13.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9d/e0/70553e3000e345daff267cec284ce4cbf3fc141b6da229ac52775b5428f1/coverage-7.13.5.tar.gz", hash = "sha256:c81f6515c4c40141f83f502b07bbfa5c240ba25bbe73da7b33f1e5b6120ff179", size = 915967, upload-time = "2026-03-17T10:33:18.341Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/c3/a396306ba7db865bf96fc1fb3b7fd29bcbf3d829df642e77b13555163cd6/coverage-7.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:460cf0114c5016fa841214ff5564aa4864f11948da9440bc97e21ad1f4ba1e01", size = 219554, upload-time = "2026-03-17T10:30:42.208Z" }, - { url = "https://files.pythonhosted.org/packages/a6/16/a68a19e5384e93f811dccc51034b1fd0b865841c390e3c931dcc4699e035/coverage-7.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e223ce4b4ed47f065bfb123687686512e37629be25cc63728557ae7db261422", size = 219908, upload-time = "2026-03-17T10:30:43.906Z" }, - { url = "https://files.pythonhosted.org/packages/29/72/20b917c6793af3a5ceb7fb9c50033f3ec7865f2911a1416b34a7cfa0813b/coverage-7.13.5-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6e3370441f4513c6252bf042b9c36d22491142385049243253c7e48398a15a9f", size = 251419, upload-time = "2026-03-17T10:30:45.545Z" }, - { url = "https://files.pythonhosted.org/packages/8c/49/cd14b789536ac6a4778c453c6a2338bc0a2fb60c5a5a41b4008328b9acc1/coverage-7.13.5-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:03ccc709a17a1de074fb1d11f217342fb0d2b1582ed544f554fc9fc3f07e95f5", size = 254159, upload-time = "2026-03-17T10:30:47.204Z" }, - { url = "https://files.pythonhosted.org/packages/9d/00/7b0edcfe64e2ed4c0340dac14a52ad0f4c9bd0b8b5e531af7d55b703db7c/coverage-7.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3f4818d065964db3c1c66dc0fbdac5ac692ecbc875555e13374fdbe7eedb4376", size = 255270, upload-time = "2026-03-17T10:30:48.812Z" }, - { url = "https://files.pythonhosted.org/packages/93/89/7ffc4ba0f5d0a55c1e84ea7cee39c9fc06af7b170513d83fbf3bbefce280/coverage-7.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:012d5319e66e9d5a218834642d6c35d265515a62f01157a45bcc036ecf947256", size = 257538, upload-time = "2026-03-17T10:30:50.77Z" }, - { url = "https://files.pythonhosted.org/packages/81/bd/73ddf85f93f7e6fa83e77ccecb6162d9415c79007b4bc124008a4995e4a7/coverage-7.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8dd02af98971bdb956363e4827d34425cb3df19ee550ef92855b0acb9c7ce51c", size = 251821, upload-time = "2026-03-17T10:30:52.5Z" }, - { url = "https://files.pythonhosted.org/packages/a0/81/278aff4e8dec4926a0bcb9486320752811f543a3ce5b602cc7a29978d073/coverage-7.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f08fd75c50a760c7eb068ae823777268daaf16a80b918fa58eea888f8e3919f5", size = 253191, upload-time = "2026-03-17T10:30:54.543Z" }, - { url = "https://files.pythonhosted.org/packages/70/ee/fe1621488e2e0a58d7e94c4800f0d96f79671553488d401a612bebae324b/coverage-7.13.5-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:843ea8643cf967d1ac7e8ecd4bb00c99135adf4816c0c0593fdcc47b597fcf09", size = 251337, upload-time = "2026-03-17T10:30:56.663Z" }, - { url = "https://files.pythonhosted.org/packages/37/a6/f79fb37aa104b562207cc23cb5711ab6793608e246cae1e93f26b2236ed9/coverage-7.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d44d7aa963820b1b971dbecd90bfe5fe8f81cff79787eb6cca15750bd2f79b9", size = 255404, upload-time = "2026-03-17T10:30:58.427Z" }, - { url = "https://files.pythonhosted.org/packages/75/f0/ed15262a58ec81ce457ceb717b7f78752a1713556b19081b76e90896e8d4/coverage-7.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7132bed4bd7b836200c591410ae7d97bf7ae8be6fc87d160b2bd881df929e7bf", size = 250903, upload-time = "2026-03-17T10:31:00.093Z" }, - { url = "https://files.pythonhosted.org/packages/0f/e9/9129958f20e7e9d4d56d51d42ccf708d15cac355ff4ac6e736e97a9393d2/coverage-7.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a698e363641b98843c517817db75373c83254781426e94ada3197cabbc2c919c", size = 252780, upload-time = "2026-03-17T10:31:01.916Z" }, - { url = "https://files.pythonhosted.org/packages/a4/d7/0ad9b15812d81272db94379fe4c6df8fd17781cc7671fdfa30c76ba5ff7b/coverage-7.13.5-cp312-cp312-win32.whl", hash = "sha256:bdba0a6b8812e8c7df002d908a9a2ea3c36e92611b5708633c50869e6d922fdf", size = 222093, upload-time = "2026-03-17T10:31:03.642Z" }, - { url = "https://files.pythonhosted.org/packages/29/3d/821a9a5799fac2556bcf0bd37a70d1d11fa9e49784b6d22e92e8b2f85f18/coverage-7.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:d2c87e0c473a10bffe991502eac389220533024c8082ec1ce849f4218dded810", size = 222900, upload-time = "2026-03-17T10:31:05.651Z" }, - { url = "https://files.pythonhosted.org/packages/d4/fa/2238c2ad08e35cf4f020ea721f717e09ec3152aea75d191a7faf3ef009a8/coverage-7.13.5-cp312-cp312-win_arm64.whl", hash = "sha256:bf69236a9a81bdca3bff53796237aab096cdbf8d78a66ad61e992d9dac7eb2de", size = 221515, upload-time = "2026-03-17T10:31:07.293Z" }, - { url = "https://files.pythonhosted.org/packages/74/8c/74fedc9663dcf168b0a059d4ea756ecae4da77a489048f94b5f512a8d0b3/coverage-7.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ec4af212df513e399cf11610cc27063f1586419e814755ab362e50a85ea69c1", size = 219576, upload-time = "2026-03-17T10:31:09.045Z" }, - { url = "https://files.pythonhosted.org/packages/0c/c9/44fb661c55062f0818a6ffd2685c67aa30816200d5f2817543717d4b92eb/coverage-7.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:941617e518602e2d64942c88ec8499f7fbd49d3f6c4327d3a71d43a1973032f3", size = 219942, upload-time = "2026-03-17T10:31:10.708Z" }, - { url = "https://files.pythonhosted.org/packages/5f/13/93419671cee82b780bab7ea96b67c8ef448f5f295f36bf5031154ec9a790/coverage-7.13.5-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:da305e9937617ee95c2e39d8ff9f040e0487cbf1ac174f777ed5eddd7a7c1f26", size = 250935, upload-time = "2026-03-17T10:31:12.392Z" }, - { url = "https://files.pythonhosted.org/packages/ac/68/1666e3a4462f8202d836920114fa7a5ee9275d1fa45366d336c551a162dd/coverage-7.13.5-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:78e696e1cc714e57e8b25760b33a8b1026b7048d270140d25dafe1b0a1ee05a3", size = 253541, upload-time = "2026-03-17T10:31:14.247Z" }, - { url = "https://files.pythonhosted.org/packages/4e/5e/3ee3b835647be646dcf3c65a7c6c18f87c27326a858f72ab22c12730773d/coverage-7.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02ca0eed225b2ff301c474aeeeae27d26e2537942aa0f87491d3e147e784a82b", size = 254780, upload-time = "2026-03-17T10:31:16.193Z" }, - { url = "https://files.pythonhosted.org/packages/44/b3/cb5bd1a04cfcc49ede6cd8409d80bee17661167686741e041abc7ee1b9a9/coverage-7.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:04690832cbea4e4663d9149e05dba142546ca05cb1848816760e7f58285c970a", size = 256912, upload-time = "2026-03-17T10:31:17.89Z" }, - { url = "https://files.pythonhosted.org/packages/1b/66/c1dceb7b9714473800b075f5c8a84f4588f887a90eb8645282031676e242/coverage-7.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0590e44dd2745c696a778f7bab6aa95256de2cbc8b8cff4f7db8ff09813d6969", size = 251165, upload-time = "2026-03-17T10:31:19.605Z" }, - { url = "https://files.pythonhosted.org/packages/b7/62/5502b73b97aa2e53ea22a39cf8649ff44827bef76d90bf638777daa27a9d/coverage-7.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d7cfad2d6d81dd298ab6b89fe72c3b7b05ec7544bdda3b707ddaecff8d25c161", size = 252908, upload-time = "2026-03-17T10:31:21.312Z" }, - { url = "https://files.pythonhosted.org/packages/7d/37/7792c2d69854397ca77a55c4646e5897c467928b0e27f2d235d83b5d08c6/coverage-7.13.5-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:e092b9499de38ae0fbfbc603a74660eb6ff3e869e507b50d85a13b6db9863e15", size = 250873, upload-time = "2026-03-17T10:31:23.565Z" }, - { url = "https://files.pythonhosted.org/packages/a3/23/bc866fb6163be52a8a9e5d708ba0d3b1283c12158cefca0a8bbb6e247a43/coverage-7.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:48c39bc4a04d983a54a705a6389512883d4a3b9862991b3617d547940e9f52b1", size = 255030, upload-time = "2026-03-17T10:31:25.58Z" }, - { url = "https://files.pythonhosted.org/packages/7d/8b/ef67e1c222ef49860701d346b8bbb70881bef283bd5f6cbba68a39a086c7/coverage-7.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2d3807015f138ffea1ed9afeeb8624fd781703f2858b62a8dd8da5a0994c57b6", size = 250694, upload-time = "2026-03-17T10:31:27.316Z" }, - { url = "https://files.pythonhosted.org/packages/46/0d/866d1f74f0acddbb906db212e096dee77a8e2158ca5e6bb44729f9d93298/coverage-7.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ee2aa19e03161671ec964004fb74b2257805d9710bf14a5c704558b9d8dbaf17", size = 252469, upload-time = "2026-03-17T10:31:29.472Z" }, - { url = "https://files.pythonhosted.org/packages/7a/f5/be742fec31118f02ce42b21c6af187ad6a344fed546b56ca60caacc6a9a0/coverage-7.13.5-cp313-cp313-win32.whl", hash = "sha256:ce1998c0483007608c8382f4ff50164bfc5bd07a2246dd272aa4043b75e61e85", size = 222112, upload-time = "2026-03-17T10:31:31.526Z" }, - { url = "https://files.pythonhosted.org/packages/66/40/7732d648ab9d069a46e686043241f01206348e2bbf128daea85be4d6414b/coverage-7.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:631efb83f01569670a5e866ceb80fe483e7c159fac6f167e6571522636104a0b", size = 222923, upload-time = "2026-03-17T10:31:33.633Z" }, - { url = "https://files.pythonhosted.org/packages/48/af/fea819c12a095781f6ccd504890aaddaf88b8fab263c4940e82c7b770124/coverage-7.13.5-cp313-cp313-win_arm64.whl", hash = "sha256:f4cd16206ad171cbc2470dbea9103cf9a7607d5fe8c242fdf1edf36174020664", size = 221540, upload-time = "2026-03-17T10:31:35.445Z" }, - { url = "https://files.pythonhosted.org/packages/23/d2/17879af479df7fbbd44bd528a31692a48f6b25055d16482fdf5cdb633805/coverage-7.13.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0428cbef5783ad91fe240f673cc1f76b25e74bbfe1a13115e4aa30d3f538162d", size = 220262, upload-time = "2026-03-17T10:31:37.184Z" }, - { url = "https://files.pythonhosted.org/packages/5b/4c/d20e554f988c8f91d6a02c5118f9abbbf73a8768a3048cb4962230d5743f/coverage-7.13.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e0b216a19534b2427cc201a26c25da4a48633f29a487c61258643e89d28200c0", size = 220617, upload-time = "2026-03-17T10:31:39.245Z" }, - { url = "https://files.pythonhosted.org/packages/29/9c/f9f5277b95184f764b24e7231e166dfdb5780a46d408a2ac665969416d61/coverage-7.13.5-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:972a9cd27894afe4bc2b1480107054e062df08e671df7c2f18c205e805ccd806", size = 261912, upload-time = "2026-03-17T10:31:41.324Z" }, - { url = "https://files.pythonhosted.org/packages/d5/f6/7f1ab39393eeb50cfe4747ae8ef0e4fc564b989225aa1152e13a180d74f8/coverage-7.13.5-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4b59148601efcd2bac8c4dbf1f0ad6391693ccf7a74b8205781751637076aee3", size = 263987, upload-time = "2026-03-17T10:31:43.724Z" }, - { url = "https://files.pythonhosted.org/packages/a0/d7/62c084fb489ed9c6fbdf57e006752e7c516ea46fd690e5ed8b8617c7d52e/coverage-7.13.5-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:505d7083c8b0c87a8fa8c07370c285847c1f77739b22e299ad75a6af6c32c5c9", size = 266416, upload-time = "2026-03-17T10:31:45.769Z" }, - { url = "https://files.pythonhosted.org/packages/a9/f6/df63d8660e1a0bff6125947afda112a0502736f470d62ca68b288ea762d8/coverage-7.13.5-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:60365289c3741e4db327e7baff2a4aaacf22f788e80fa4683393891b70a89fbd", size = 267558, upload-time = "2026-03-17T10:31:48.293Z" }, - { url = "https://files.pythonhosted.org/packages/5b/02/353ca81d36779bd108f6d384425f7139ac3c58c750dcfaafe5d0bee6436b/coverage-7.13.5-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1b88c69c8ef5d4b6fe7dea66d6636056a0f6a7527c440e890cf9259011f5e606", size = 261163, upload-time = "2026-03-17T10:31:50.125Z" }, - { url = "https://files.pythonhosted.org/packages/2c/16/2e79106d5749bcaf3aee6d309123548e3276517cd7851faa8da213bc61bf/coverage-7.13.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5b13955d31d1633cf9376908089b7cebe7d15ddad7aeaabcbe969a595a97e95e", size = 263981, upload-time = "2026-03-17T10:31:51.961Z" }, - { url = "https://files.pythonhosted.org/packages/29/c7/c29e0c59ffa6942030ae6f50b88ae49988e7e8da06de7ecdbf49c6d4feae/coverage-7.13.5-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:f70c9ab2595c56f81a89620e22899eea8b212a4041bd728ac6f4a28bf5d3ddd0", size = 261604, upload-time = "2026-03-17T10:31:53.872Z" }, - { url = "https://files.pythonhosted.org/packages/40/48/097cdc3db342f34006a308ab41c3a7c11c3f0d84750d340f45d88a782e00/coverage-7.13.5-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:084b84a8c63e8d6fc7e3931b316a9bcafca1458d753c539db82d31ed20091a87", size = 265321, upload-time = "2026-03-17T10:31:55.997Z" }, - { url = "https://files.pythonhosted.org/packages/bb/1f/4994af354689e14fd03a75f8ec85a9a68d94e0188bbdab3fc1516b55e512/coverage-7.13.5-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:ad14385487393e386e2ea988b09d62dd42c397662ac2dabc3832d71253eee479", size = 260502, upload-time = "2026-03-17T10:31:58.308Z" }, - { url = "https://files.pythonhosted.org/packages/22/c6/9bb9ef55903e628033560885f5c31aa227e46878118b63ab15dc7ba87797/coverage-7.13.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7f2c47b36fe7709a6e83bfadf4eefb90bd25fbe4014d715224c4316f808e59a2", size = 262688, upload-time = "2026-03-17T10:32:00.141Z" }, - { url = "https://files.pythonhosted.org/packages/14/4f/f5df9007e50b15e53e01edea486814783a7f019893733d9e4d6caad75557/coverage-7.13.5-cp313-cp313t-win32.whl", hash = "sha256:67e9bc5449801fad0e5dff329499fb090ba4c5800b86805c80617b4e29809b2a", size = 222788, upload-time = "2026-03-17T10:32:02.246Z" }, - { url = "https://files.pythonhosted.org/packages/e1/98/aa7fccaa97d0f3192bec013c4e6fd6d294a6ed44b640e6bb61f479e00ed5/coverage-7.13.5-cp313-cp313t-win_amd64.whl", hash = "sha256:da86cdcf10d2519e10cabb8ac2de03da1bcb6e4853790b7fbd48523332e3a819", size = 223851, upload-time = "2026-03-17T10:32:04.416Z" }, - { url = "https://files.pythonhosted.org/packages/3d/8b/e5c469f7352651e5f013198e9e21f97510b23de957dd06a84071683b4b60/coverage-7.13.5-cp313-cp313t-win_arm64.whl", hash = "sha256:0ecf12ecb326fe2c339d93fc131816f3a7367d223db37817208905c89bded911", size = 222104, upload-time = "2026-03-17T10:32:06.65Z" }, - { url = "https://files.pythonhosted.org/packages/8e/77/39703f0d1d4b478bfd30191d3c14f53caf596fac00efb3f8f6ee23646439/coverage-7.13.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fbabfaceaeb587e16f7008f7795cd80d20ec548dc7f94fbb0d4ec2e038ce563f", size = 219621, upload-time = "2026-03-17T10:32:08.589Z" }, - { url = "https://files.pythonhosted.org/packages/e2/3e/51dff36d99ae14639a133d9b164d63e628532e2974d8b1edb99dd1ebc733/coverage-7.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9bb2a28101a443669a423b665939381084412b81c3f8c0fcfbac57f4e30b5b8e", size = 219953, upload-time = "2026-03-17T10:32:10.507Z" }, - { url = "https://files.pythonhosted.org/packages/6a/6c/1f1917b01eb647c2f2adc9962bd66c79eb978951cab61bdc1acab3290c07/coverage-7.13.5-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bd3a2fbc1c6cccb3c5106140d87cc6a8715110373ef42b63cf5aea29df8c217a", size = 250992, upload-time = "2026-03-17T10:32:12.41Z" }, - { url = "https://files.pythonhosted.org/packages/22/e5/06b1f88f42a5a99df42ce61208bdec3bddb3d261412874280a19796fc09c/coverage-7.13.5-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6c36ddb64ed9d7e496028d1d00dfec3e428e0aabf4006583bb1839958d280510", size = 253503, upload-time = "2026-03-17T10:32:14.449Z" }, - { url = "https://files.pythonhosted.org/packages/80/28/2a148a51e5907e504fa7b85490277734e6771d8844ebcc48764a15e28155/coverage-7.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:380e8e9084d8eb38db3a9176a1a4f3c0082c3806fa0dc882d1d87abc3c789247", size = 254852, upload-time = "2026-03-17T10:32:16.56Z" }, - { url = "https://files.pythonhosted.org/packages/61/77/50e8d3d85cc0b7ebe09f30f151d670e302c7ff4a1bf6243f71dd8b0981fa/coverage-7.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e808af52a0513762df4d945ea164a24b37f2f518cbe97e03deaa0ee66139b4d6", size = 257161, upload-time = "2026-03-17T10:32:19.004Z" }, - { url = "https://files.pythonhosted.org/packages/3b/c4/b5fd1d4b7bf8d0e75d997afd3925c59ba629fc8616f1b3aae7605132e256/coverage-7.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e301d30dd7e95ae068671d746ba8c34e945a82682e62918e41b2679acd2051a0", size = 251021, upload-time = "2026-03-17T10:32:21.344Z" }, - { url = "https://files.pythonhosted.org/packages/f8/66/6ea21f910e92d69ef0b1c3346ea5922a51bad4446c9126db2ae96ee24c4c/coverage-7.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:800bc829053c80d240a687ceeb927a94fd108bbdc68dfbe505d0d75ab578a882", size = 252858, upload-time = "2026-03-17T10:32:23.506Z" }, - { url = "https://files.pythonhosted.org/packages/9e/ea/879c83cb5d61aa2a35fb80e72715e92672daef8191b84911a643f533840c/coverage-7.13.5-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:0b67af5492adb31940ee418a5a655c28e48165da5afab8c7fa6fd72a142f8740", size = 250823, upload-time = "2026-03-17T10:32:25.516Z" }, - { url = "https://files.pythonhosted.org/packages/8a/fb/616d95d3adb88b9803b275580bdeee8bd1b69a886d057652521f83d7322f/coverage-7.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c9136ff29c3a91e25b1d1552b5308e53a1e0653a23e53b6366d7c2dcbbaf8a16", size = 255099, upload-time = "2026-03-17T10:32:27.944Z" }, - { url = "https://files.pythonhosted.org/packages/1c/93/25e6917c90ec1c9a56b0b26f6cad6408e5f13bb6b35d484a0d75c9cf000d/coverage-7.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:cff784eef7f0b8f6cb28804fbddcfa99f89efe4cc35fb5627e3ac58f91ed3ac0", size = 250638, upload-time = "2026-03-17T10:32:29.914Z" }, - { url = "https://files.pythonhosted.org/packages/fc/7b/dc1776b0464145a929deed214aef9fb1493f159b59ff3c7eeeedf91eddd0/coverage-7.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:68a4953be99b17ac3c23b6efbc8a38330d99680c9458927491d18700ef23ded0", size = 252295, upload-time = "2026-03-17T10:32:31.981Z" }, - { url = "https://files.pythonhosted.org/packages/ea/fb/99cbbc56a26e07762a2740713f3c8f9f3f3106e3a3dd8cc4474954bccd34/coverage-7.13.5-cp314-cp314-win32.whl", hash = "sha256:35a31f2b1578185fbe6aa2e74cea1b1d0bbf4c552774247d9160d29b80ed56cc", size = 222360, upload-time = "2026-03-17T10:32:34.233Z" }, - { url = "https://files.pythonhosted.org/packages/8d/b7/4758d4f73fb536347cc5e4ad63662f9d60ba9118cb6785e9616b2ce5d7fa/coverage-7.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:2aa055ae1857258f9e0045be26a6d62bdb47a72448b62d7b55f4820f361a2633", size = 223174, upload-time = "2026-03-17T10:32:36.369Z" }, - { url = "https://files.pythonhosted.org/packages/2c/f2/24d84e1dfe70f8ac9fdf30d338239860d0d1d5da0bda528959d0ebc9da28/coverage-7.13.5-cp314-cp314-win_arm64.whl", hash = "sha256:1b11eef33edeae9d142f9b4358edb76273b3bfd30bc3df9a4f95d0e49caf94e8", size = 221739, upload-time = "2026-03-17T10:32:38.736Z" }, - { url = "https://files.pythonhosted.org/packages/60/5b/4a168591057b3668c2428bff25dd3ebc21b629d666d90bcdfa0217940e84/coverage-7.13.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10a0c37f0b646eaff7cce1874c31d1f1ccb297688d4c747291f4f4c70741cc8b", size = 220351, upload-time = "2026-03-17T10:32:41.196Z" }, - { url = "https://files.pythonhosted.org/packages/f5/21/1fd5c4dbfe4a58b6b99649125635df46decdfd4a784c3cd6d410d303e370/coverage-7.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b5db73ba3c41c7008037fa731ad5459fc3944cb7452fc0aa9f822ad3533c583c", size = 220612, upload-time = "2026-03-17T10:32:43.204Z" }, - { url = "https://files.pythonhosted.org/packages/d6/fe/2a924b3055a5e7e4512655a9d4609781b0d62334fa0140c3e742926834e2/coverage-7.13.5-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:750db93a81e3e5a9831b534be7b1229df848b2e125a604fe6651e48aa070e5f9", size = 261985, upload-time = "2026-03-17T10:32:45.514Z" }, - { url = "https://files.pythonhosted.org/packages/d7/0d/c8928f2bd518c45990fe1a2ab8db42e914ef9b726c975facc4282578c3eb/coverage-7.13.5-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9ddb4f4a5479f2539644be484da179b653273bca1a323947d48ab107b3ed1f29", size = 264107, upload-time = "2026-03-17T10:32:47.971Z" }, - { url = "https://files.pythonhosted.org/packages/ef/ae/4ae35bbd9a0af9d820362751f0766582833c211224b38665c0f8de3d487f/coverage-7.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8a7a2049c14f413163e2bdabd37e41179b1d1ccb10ffc6ccc4b7a718429c607", size = 266513, upload-time = "2026-03-17T10:32:50.1Z" }, - { url = "https://files.pythonhosted.org/packages/9c/20/d326174c55af36f74eac6ae781612d9492f060ce8244b570bb9d50d9d609/coverage-7.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1c85e0b6c05c592ea6d8768a66a254bfb3874b53774b12d4c89c481eb78cb90", size = 267650, upload-time = "2026-03-17T10:32:52.391Z" }, - { url = "https://files.pythonhosted.org/packages/7a/5e/31484d62cbd0eabd3412e30d74386ece4a0837d4f6c3040a653878bfc019/coverage-7.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:777c4d1eff1b67876139d24288aaf1817f6c03d6bae9c5cc8d27b83bcfe38fe3", size = 261089, upload-time = "2026-03-17T10:32:54.544Z" }, - { url = "https://files.pythonhosted.org/packages/e9/d8/49a72d6de146eebb0b7e48cc0f4bc2c0dd858e3d4790ab2b39a2872b62bd/coverage-7.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6697e29b93707167687543480a40f0db8f356e86d9f67ddf2e37e2dfd91a9dab", size = 263982, upload-time = "2026-03-17T10:32:56.803Z" }, - { url = "https://files.pythonhosted.org/packages/06/3b/0351f1bd566e6e4dd39e978efe7958bde1d32f879e85589de147654f57bb/coverage-7.13.5-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:8fdf453a942c3e4d99bd80088141c4c6960bb232c409d9c3558e2dbaa3998562", size = 261579, upload-time = "2026-03-17T10:32:59.466Z" }, - { url = "https://files.pythonhosted.org/packages/5d/ce/796a2a2f4017f554d7810f5c573449b35b1e46788424a548d4d19201b222/coverage-7.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:32ca0c0114c9834a43f045a87dcebd69d108d8ffb666957ea65aa132f50332e2", size = 265316, upload-time = "2026-03-17T10:33:01.847Z" }, - { url = "https://files.pythonhosted.org/packages/3d/16/d5ae91455541d1a78bc90abf495be600588aff8f6db5c8b0dae739fa39c9/coverage-7.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:8769751c10f339021e2638cd354e13adeac54004d1941119b2c96fe5276d45ea", size = 260427, upload-time = "2026-03-17T10:33:03.945Z" }, - { url = "https://files.pythonhosted.org/packages/48/11/07f413dba62db21fb3fad5d0de013a50e073cc4e2dc4306e770360f6dfc8/coverage-7.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cec2d83125531bd153175354055cdb7a09987af08a9430bd173c937c6d0fba2a", size = 262745, upload-time = "2026-03-17T10:33:06.285Z" }, - { url = "https://files.pythonhosted.org/packages/91/15/d792371332eb4663115becf4bad47e047d16234b1aff687b1b18c58d60ae/coverage-7.13.5-cp314-cp314t-win32.whl", hash = "sha256:0cd9ed7a8b181775459296e402ca4fb27db1279740a24e93b3b41942ebe4b215", size = 223146, upload-time = "2026-03-17T10:33:08.756Z" }, - { url = "https://files.pythonhosted.org/packages/db/51/37221f59a111dca5e85be7dbf09696323b5b9f13ff65e0641d535ed06ea8/coverage-7.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:301e3b7dfefecaca37c9f1aa6f0049b7d4ab8dd933742b607765d757aca77d43", size = 224254, upload-time = "2026-03-17T10:33:11.174Z" }, - { url = "https://files.pythonhosted.org/packages/54/83/6acacc889de8987441aa7d5adfbdbf33d288dad28704a67e574f1df9bcbb/coverage-7.13.5-cp314-cp314t-win_arm64.whl", hash = "sha256:9dacc2ad679b292709e0f5fc1ac74a6d4d5562e424058962c7bb0c658ad25e45", size = 222276, upload-time = "2026-03-17T10:33:13.466Z" }, - { url = "https://files.pythonhosted.org/packages/9e/ee/a4cf96b8ce1e566ed238f0659ac2d3f007ed1d14b181bcb684e19561a69a/coverage-7.13.5-py3-none-any.whl", hash = "sha256:34b02417cf070e173989b3db962f7ed56d2f644307b2cf9d5a0f258e13084a61", size = 211346, upload-time = "2026-03-17T10:33:15.691Z" }, -] - [[package]] name = "cryptography" version = "48.0.0" @@ -562,53 +450,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/65/99/00f3196036501b53032c4b1ab8337a0b978dee832ed276dae3815df4e8b5/datasets-4.8.5-py3-none-any.whl", hash = "sha256:5079900781719c0e063a8efdd2cd95a31ad0c63209178669cd23cf1b926149ff", size = 528973, upload-time = "2026-04-27T15:43:53.702Z" }, ] -[[package]] -name = "ddtrace" -version = "3.19.7" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "bytecode" }, - { name = "envier" }, - { name = "opentelemetry-api" }, - { name = "wrapt" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/19/0c/05ab6cf1db00b89fc4f86d7d4b262bc4a17e547ee4f09086c76916976211/ddtrace-3.19.7.tar.gz", hash = "sha256:e531a7c6370458d0e61b498714969471514033f6851a4603ac62f2cac1adc925", size = 7653384, upload-time = "2026-03-13T19:03:53.67Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/00/beadc842ac96cc872c744c9f55d940a1f9dc0cbbfe8663415f2b273db34c/ddtrace-3.19.7-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:2a1b4ccf33d82cc03334f6f54b8e37711f0e46fddaade56769ac42e6c6683d98", size = 6538107, upload-time = "2026-03-13T19:01:08.903Z" }, - { url = "https://files.pythonhosted.org/packages/92/0d/4dde240e6a70b132728399ff196aab33da07dd5be3191e53d7d9e7d8b2d0/ddtrace-3.19.7-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:b6037dba7f0b7fa59b8e1ab44a2750fccc3e619917d451a94cc1bc108b16d3cb", size = 6944628, upload-time = "2026-03-13T19:01:11.159Z" }, - { url = "https://files.pythonhosted.org/packages/76/78/770178a9f826b115e9c1ce8d053873d30e231db82856b307c1e87879b044/ddtrace-3.19.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:38b515c8b5100e6b3c513e58160eb268bddbd9783fa2cb6f7f65ee7c387936ee", size = 7531273, upload-time = "2026-03-13T19:01:13.771Z" }, - { url = "https://files.pythonhosted.org/packages/10/43/9601599f7988bd12b59f083dbaec321f1ce66f27eae06b8afb90841917d8/ddtrace-3.19.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:300807941a2f005aacf4ef3d81e94dd4034d9acf1bf3786e73c382659bd8b617", size = 7836056, upload-time = "2026-03-13T19:01:15.991Z" }, - { url = "https://files.pythonhosted.org/packages/b7/56/8e60e9a0f46f53595936576fac97c1689873498518e68382fa54fcd7d158/ddtrace-3.19.7-cp312-cp312-manylinux_2_28_i686.whl", hash = "sha256:c74524559e7d21beb29b8e13c5090644525b5df03a6a7450e9286997e31ac9c9", size = 5622525, upload-time = "2026-03-13T19:01:18.284Z" }, - { url = "https://files.pythonhosted.org/packages/6a/4a/3cf7dccd4011e5030f507fe765495b375a6e4d23851fe7811d72395f6e5a/ddtrace-3.19.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:817abdd18b24ec0f881843af4c6bae55c4ecd997dc0c3688c4a45b0a18a1368c", size = 8588806, upload-time = "2026-03-13T19:01:20.554Z" }, - { url = "https://files.pythonhosted.org/packages/f6/92/3d1c0bcb7412586c39c3ac5226cd87ae6819ebcb3a8edce2f6eed29b19f6/ddtrace-3.19.7-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c59faed5fa5330ffd5fe60284b1ec1a0525d7fc063787de227a418810851c2f3", size = 6677414, upload-time = "2026-03-13T19:01:23.07Z" }, - { url = "https://files.pythonhosted.org/packages/4f/0e/8a4d7211f1d35082956cab1437c2c81245158547677445017a2130782869/ddtrace-3.19.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:af3f1200c3e4c850a9c408047a6f83f5b87cbb0c641022a4107e566e5064c434", size = 8897744, upload-time = "2026-03-13T19:01:25.702Z" }, - { url = "https://files.pythonhosted.org/packages/d0/65/4733cf6cd78c00f34cbce8fdba86d26c5adbd6613e049d5f3430449300ae/ddtrace-3.19.7-cp312-cp312-win32.whl", hash = "sha256:6fd3409777c2549b5bf743abc3c1ab17ba7a1a399c3a580e2a4bac383a576e35", size = 5116495, upload-time = "2026-03-13T19:01:28.243Z" }, - { url = "https://files.pythonhosted.org/packages/dc/49/fa81f8084d43011ce4e80a3a52dcf144b5069ee7f8f191d41f0d1dcc7d6f/ddtrace-3.19.7-cp312-cp312-win_amd64.whl", hash = "sha256:69448c80f440aee4d1a3e19295e66c44a84ae014da6c05a52d846b4b7ac821c9", size = 5611097, upload-time = "2026-03-13T19:01:30.848Z" }, - { url = "https://files.pythonhosted.org/packages/48/d8/3c3a1e67df4c54b5d2585ad180d9eaf5a1d05a98e81cedf8b2b69930a0d6/ddtrace-3.19.7-cp312-cp312-win_arm64.whl", hash = "sha256:30827b6bbbbb75f883717c59339e43feef31a7a31da623f2f291ac6e9a044c6f", size = 5334126, upload-time = "2026-03-13T19:01:33.526Z" }, - { url = "https://files.pythonhosted.org/packages/de/30/fe91d89021ceeb7a41bcbc8a05f3c54fc619ea482fd14257f423e1e1aa0e/ddtrace-3.19.7-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:425bacf644a772aa3faca075234f3bc5d37972fa1f32932343b00653190cdab5", size = 6532341, upload-time = "2026-03-13T19:01:35.891Z" }, - { url = "https://files.pythonhosted.org/packages/4a/b1/99664152487bd8612169506cc33f9f3fabe988884bf2fdcb3877f169df8d/ddtrace-3.19.7-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:dec566649d660bfe179d2d97d8217304a466c0294ac6cec84ab9ee5bd2c20056", size = 6940282, upload-time = "2026-03-13T19:01:38.642Z" }, - { url = "https://files.pythonhosted.org/packages/8d/42/191d84881dcd99ba32577efa3cc4e4d3bd519629798421fa8a6165e679af/ddtrace-3.19.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:91e62d9d206150ddc360af2628615572f3bcc314ae420ab59041e6e493477993", size = 7520474, upload-time = "2026-03-13T19:01:41.157Z" }, - { url = "https://files.pythonhosted.org/packages/cb/4d/4413bacc2448629223effaa9cc14c8830872853bcd7e2673e186179c32cf/ddtrace-3.19.7-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:251607dec183356cfe1ec8f102f01ac556a38136216f74e31b5748b6fa994d97", size = 7825317, upload-time = "2026-03-13T19:01:43.87Z" }, - { url = "https://files.pythonhosted.org/packages/bc/d2/a7ed2c22b9069fe7cb0797f04b983f0dd17ce06ab581cd5e5433dd69ea3a/ddtrace-3.19.7-cp313-cp313-manylinux_2_28_i686.whl", hash = "sha256:6952e2e78f126dce55d4d988490bc76c24e439d52d2686ff618f8d7640534e61", size = 5611950, upload-time = "2026-03-13T19:01:46.47Z" }, - { url = "https://files.pythonhosted.org/packages/5d/bf/6caa894b8ef814bba6f5514d6249f0ab33e61173d795dd34a2114de39aa1/ddtrace-3.19.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:82b7fbfedac7986274d2c947d247affdb9d918edcb6d65587ab9c1d8d81af9bf", size = 8583032, upload-time = "2026-03-13T19:01:50.098Z" }, - { url = "https://files.pythonhosted.org/packages/5c/5b/761df8107c1689ab09feeb46c1791a7ebed5c248b4df620dcda2c6f6e722/ddtrace-3.19.7-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b96f689ab0225d2d8934c4aa42a7ce1001b91fbf1ff8e8158a6428e0f0faece6", size = 6668576, upload-time = "2026-03-13T19:01:53.456Z" }, - { url = "https://files.pythonhosted.org/packages/88/90/cc40e6ce5888782c972d2aaa81828648b0c24db2e84c34134274a8d50ce9/ddtrace-3.19.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9a0995ac17c3e43fae6f6c2ac0d9a90d035ea13d629342f84ba7596ac002ce87", size = 8889877, upload-time = "2026-03-13T19:01:56.557Z" }, - { url = "https://files.pythonhosted.org/packages/14/d3/0b396f24703521b102f6220869b4931d6388c55402c16e8ace8c155d6871/ddtrace-3.19.7-cp313-cp313-win32.whl", hash = "sha256:846b695964580cb4d91c4600907119f38b57b792d7ff1f21793d2140157b820b", size = 5113764, upload-time = "2026-03-13T19:01:59.688Z" }, - { url = "https://files.pythonhosted.org/packages/6e/67/12806c2523c22d394e072f0d696dafe794b1ac3f441d3cb0b3e37ed4b770/ddtrace-3.19.7-cp313-cp313-win_amd64.whl", hash = "sha256:de1aabc50c663f27c1ae341d16b31bdc3b8fb1c28c4b8a683e9000cbfd11fa44", size = 5607876, upload-time = "2026-03-13T19:02:02.588Z" }, - { url = "https://files.pythonhosted.org/packages/ba/c3/27bff2e1ba9bc8323516d6c6bf379f5de8c0e3dddd672f2fb82bc58fc99d/ddtrace-3.19.7-cp313-cp313-win_arm64.whl", hash = "sha256:2d41fd500ec915be8b3b95cd7a5535f87323acbdc7e9b2c02797d83b6985b451", size = 5331124, upload-time = "2026-03-13T19:02:05.267Z" }, - { url = "https://files.pythonhosted.org/packages/6a/01/34d3af11ee01870366af5a71aa5b1d1ce342fdad6f030a37538aa5d21961/ddtrace-3.19.7-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:528fb2565048e669cfe5ad0e24f4f39f73747977982c78339f9ece7e39430180", size = 6090587, upload-time = "2026-03-13T19:02:08.355Z" }, - { url = "https://files.pythonhosted.org/packages/81/3b/1dcf930c60e7e1393e86ce2fb9202b6e3b3eeb01aeccb92b2170768fdcd8/ddtrace-3.19.7-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:29202bdd4496098b27907dfdfd85000549901461ebea6f0f90d19eb4ebac2b76", size = 6433258, upload-time = "2026-03-13T19:02:11.192Z" }, - { url = "https://files.pythonhosted.org/packages/90/56/0c45a0aac2c728acb6500a87155bad3dc18bd9990909fb0388eac7d81f47/ddtrace-3.19.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:abb3b5b8cac3247b01d381fe71aeb56b20f8759275619701418e53f0306eb6a0", size = 7125123, upload-time = "2026-03-13T19:02:14.001Z" }, - { url = "https://files.pythonhosted.org/packages/b9/bd/5b6c42d8c23390e24210988a01d317732e72cbb063850220aa99827f86b4/ddtrace-3.19.7-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7d27d310e272aa7772fdf7087bd5056966448d8051586de75577076ad8c8e323", size = 7357163, upload-time = "2026-03-13T19:02:16.849Z" }, - { url = "https://files.pythonhosted.org/packages/f3/d2/832cfa6b2cd1d16038997853128fbf0989b197695b36b61974ac2b1c720b/ddtrace-3.19.7-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:aec1ec1fbcdafa76362c7f468aab240224d14060c4cda73e9a0676293be471c5", size = 5614026, upload-time = "2026-03-13T19:02:19.648Z" }, - { url = "https://files.pythonhosted.org/packages/dc/ca/edfa36f942d043e6b5cbf9312bcae3e5b22c785b9db894d37aff608e1895/ddtrace-3.19.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:26c300dbb6d8c29b6fcd4847da29010a988150a28426c4cc46e7e1f3816ea195", size = 8095787, upload-time = "2026-03-13T19:02:22.838Z" }, - { url = "https://files.pythonhosted.org/packages/fc/91/173203d14cdd5e6084290266f05c550fef96e38006bca486584f870f7fc4/ddtrace-3.19.7-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:8c3a74eda7907e5d4a5a36c62e02ee41fca10de83ae2f592e0a63530d8eda7aa", size = 6669581, upload-time = "2026-03-13T19:02:25.684Z" }, - { url = "https://files.pythonhosted.org/packages/a3/61/b33224286d3ee39686e4e1be85a9da2ecd58b4e8661875fb9fd113e73f34/ddtrace-3.19.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:79cdb4246da2a6e32b768a0c7f952890ddeb9c8bea9228adf2d82bcf66849177", size = 8438684, upload-time = "2026-03-13T19:02:28.648Z" }, - { url = "https://files.pythonhosted.org/packages/7d/d9/88b83c185fe26849aff109d80f9c93eca170a6365a2a1885756d90b3213d/ddtrace-3.19.7-cp314-cp314-win32.whl", hash = "sha256:759d49298575079e31da354780ca37220775c80a007bbe2ae7fb756a11210740", size = 5208198, upload-time = "2026-03-13T19:02:31.979Z" }, - { url = "https://files.pythonhosted.org/packages/45/a1/57d6266163c180272f2eb0191c93ec8d4bf6b67476817dab5f5749411aab/ddtrace-3.19.7-cp314-cp314-win_amd64.whl", hash = "sha256:054ebbdd1e02e90874cda803b31721f4e4ab6d0d3833bc8e82de8b78f1fc9d57", size = 5740240, upload-time = "2026-03-13T19:02:34.79Z" }, - { url = "https://files.pythonhosted.org/packages/a9/16/6c269f658c8b2e79083ecc8af1dd8cfb64ea031492eaf040a13a4ac86178/ddtrace-3.19.7-cp314-cp314-win_arm64.whl", hash = "sha256:ff953b2d7dc8a7d8e8d452aa0f9ac1e094e9deba371f3715bc73613bc1fa0069", size = 5475688, upload-time = "2026-03-13T19:02:37.758Z" }, -] - [[package]] name = "deprecation" version = "2.1.0" @@ -651,24 +492,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, ] -[[package]] -name = "docstring-parser" -version = "0.18.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e0/4d/f332313098c1de1b2d2ff91cf2674415cc7cddab2ca1b01ae29774bd5fdf/docstring_parser-0.18.0.tar.gz", hash = "sha256:292510982205c12b1248696f44959db3cdd1740237a968ea1e2e7a900eeb2015", size = 29341, upload-time = "2026-04-14T04:09:19.867Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl", hash = "sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b", size = 22484, upload-time = "2026-04-14T04:09:18.638Z" }, -] - -[[package]] -name = "envier" -version = "0.6.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/19/e7/4fe4d3f6e21213cea9bcddc36ba60e6ae4003035f9ce8055e6a9f0322ddb/envier-0.6.1.tar.gz", hash = "sha256:3309a01bb3d8850c9e7a31a5166d5a836846db2faecb79b9cb32654dd50ca9f9", size = 10063, upload-time = "2024-10-22T09:56:47.226Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/56/e9/30493b1cc967f7c07869de4b2ab3929151a58e6bb04495015554d24b61db/envier-0.6.1-py3-none-any.whl", hash = "sha256:73609040a76be48bbcb97074d9969666484aa0de706183a6e9ef773156a8a6a9", size = 10638, upload-time = "2024-10-22T09:56:45.968Z" }, -] - [[package]] name = "fastapi" version = "0.136.1" @@ -954,35 +777,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] -[[package]] -name = "httptools" -version = "0.7.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b5/46/120a669232c7bdedb9d52d4aeae7e6c7dfe151e99dc70802e2fc7a5e1993/httptools-0.7.1.tar.gz", hash = "sha256:abd72556974f8e7c74a259655924a717a2365b236c882c3f6f8a45fe94703ac9", size = 258961, upload-time = "2025-10-10T03:55:08.559Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/53/7f/403e5d787dc4942316e515e949b0c8a013d84078a915910e9f391ba9b3ed/httptools-0.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:38e0c83a2ea9746ebbd643bdfb521b9aa4a91703e2cd705c20443405d2fd16a5", size = 206280, upload-time = "2025-10-10T03:54:39.274Z" }, - { url = "https://files.pythonhosted.org/packages/2a/0d/7f3fd28e2ce311ccc998c388dd1c53b18120fda3b70ebb022b135dc9839b/httptools-0.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f25bbaf1235e27704f1a7b86cd3304eabc04f569c828101d94a0e605ef7205a5", size = 110004, upload-time = "2025-10-10T03:54:40.403Z" }, - { url = "https://files.pythonhosted.org/packages/84/a6/b3965e1e146ef5762870bbe76117876ceba51a201e18cc31f5703e454596/httptools-0.7.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2c15f37ef679ab9ecc06bfc4e6e8628c32a8e4b305459de7cf6785acd57e4d03", size = 517655, upload-time = "2025-10-10T03:54:41.347Z" }, - { url = "https://files.pythonhosted.org/packages/11/7d/71fee6f1844e6fa378f2eddde6c3e41ce3a1fb4b2d81118dd544e3441ec0/httptools-0.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7fe6e96090df46b36ccfaf746f03034e5ab723162bc51b0a4cf58305324036f2", size = 511440, upload-time = "2025-10-10T03:54:42.452Z" }, - { url = "https://files.pythonhosted.org/packages/22/a5/079d216712a4f3ffa24af4a0381b108aa9c45b7a5cc6eb141f81726b1823/httptools-0.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f72fdbae2dbc6e68b8239defb48e6a5937b12218e6ffc2c7846cc37befa84362", size = 495186, upload-time = "2025-10-10T03:54:43.937Z" }, - { url = "https://files.pythonhosted.org/packages/e9/9e/025ad7b65278745dee3bd0ebf9314934c4592560878308a6121f7f812084/httptools-0.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e99c7b90a29fd82fea9ef57943d501a16f3404d7b9ee81799d41639bdaae412c", size = 499192, upload-time = "2025-10-10T03:54:45.003Z" }, - { url = "https://files.pythonhosted.org/packages/6d/de/40a8f202b987d43afc4d54689600ff03ce65680ede2f31df348d7f368b8f/httptools-0.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:3e14f530fefa7499334a79b0cf7e7cd2992870eb893526fb097d51b4f2d0f321", size = 86694, upload-time = "2025-10-10T03:54:45.923Z" }, - { url = "https://files.pythonhosted.org/packages/09/8f/c77b1fcbfd262d422f12da02feb0d218fa228d52485b77b953832105bb90/httptools-0.7.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6babce6cfa2a99545c60bfef8bee0cc0545413cb0018f617c8059a30ad985de3", size = 202889, upload-time = "2025-10-10T03:54:47.089Z" }, - { url = "https://files.pythonhosted.org/packages/0a/1a/22887f53602feaa066354867bc49a68fc295c2293433177ee90870a7d517/httptools-0.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:601b7628de7504077dd3dcb3791c6b8694bbd967148a6d1f01806509254fb1ca", size = 108180, upload-time = "2025-10-10T03:54:48.052Z" }, - { url = "https://files.pythonhosted.org/packages/32/6a/6aaa91937f0010d288d3d124ca2946d48d60c3a5ee7ca62afe870e3ea011/httptools-0.7.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:04c6c0e6c5fb0739c5b8a9eb046d298650a0ff38cf42537fc372b28dc7e4472c", size = 478596, upload-time = "2025-10-10T03:54:48.919Z" }, - { url = "https://files.pythonhosted.org/packages/6d/70/023d7ce117993107be88d2cbca566a7c1323ccbaf0af7eabf2064fe356f6/httptools-0.7.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:69d4f9705c405ae3ee83d6a12283dc9feba8cc6aaec671b412917e644ab4fa66", size = 473268, upload-time = "2025-10-10T03:54:49.993Z" }, - { url = "https://files.pythonhosted.org/packages/32/4d/9dd616c38da088e3f436e9a616e1d0cc66544b8cdac405cc4e81c8679fc7/httptools-0.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:44c8f4347d4b31269c8a9205d8a5ee2df5322b09bbbd30f8f862185bb6b05346", size = 455517, upload-time = "2025-10-10T03:54:51.066Z" }, - { url = "https://files.pythonhosted.org/packages/1d/3a/a6c595c310b7df958e739aae88724e24f9246a514d909547778d776799be/httptools-0.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:465275d76db4d554918aba40bf1cbebe324670f3dfc979eaffaa5d108e2ed650", size = 458337, upload-time = "2025-10-10T03:54:52.196Z" }, - { url = "https://files.pythonhosted.org/packages/fd/82/88e8d6d2c51edc1cc391b6e044c6c435b6aebe97b1abc33db1b0b24cd582/httptools-0.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:322d00c2068d125bd570f7bf78b2d367dad02b919d8581d7476d8b75b294e3e6", size = 85743, upload-time = "2025-10-10T03:54:53.448Z" }, - { url = "https://files.pythonhosted.org/packages/34/50/9d095fcbb6de2d523e027a2f304d4551855c2f46e0b82befd718b8b20056/httptools-0.7.1-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:c08fe65728b8d70b6923ce31e3956f859d5e1e8548e6f22ec520a962c6757270", size = 203619, upload-time = "2025-10-10T03:54:54.321Z" }, - { url = "https://files.pythonhosted.org/packages/07/f0/89720dc5139ae54b03f861b5e2c55a37dba9a5da7d51e1e824a1f343627f/httptools-0.7.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7aea2e3c3953521c3c51106ee11487a910d45586e351202474d45472db7d72d3", size = 108714, upload-time = "2025-10-10T03:54:55.163Z" }, - { url = "https://files.pythonhosted.org/packages/b3/cb/eea88506f191fb552c11787c23f9a405f4c7b0c5799bf73f2249cd4f5228/httptools-0.7.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0e68b8582f4ea9166be62926077a3334064d422cf08ab87d8b74664f8e9058e1", size = 472909, upload-time = "2025-10-10T03:54:56.056Z" }, - { url = "https://files.pythonhosted.org/packages/e0/4a/a548bdfae6369c0d078bab5769f7b66f17f1bfaa6fa28f81d6be6959066b/httptools-0.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df091cf961a3be783d6aebae963cc9b71e00d57fa6f149025075217bc6a55a7b", size = 470831, upload-time = "2025-10-10T03:54:57.219Z" }, - { url = "https://files.pythonhosted.org/packages/4d/31/14df99e1c43bd132eec921c2e7e11cda7852f65619bc0fc5bdc2d0cb126c/httptools-0.7.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:f084813239e1eb403ddacd06a30de3d3e09a9b76e7894dcda2b22f8a726e9c60", size = 452631, upload-time = "2025-10-10T03:54:58.219Z" }, - { url = "https://files.pythonhosted.org/packages/22/d2/b7e131f7be8d854d48cb6d048113c30f9a46dca0c9a8b08fcb3fcd588cdc/httptools-0.7.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7347714368fb2b335e9063bc2b96f2f87a9ceffcd9758ac295f8bbcd3ffbc0ca", size = 452910, upload-time = "2025-10-10T03:54:59.366Z" }, - { url = "https://files.pythonhosted.org/packages/53/cf/878f3b91e4e6e011eff6d1fa9ca39f7eb17d19c9d7971b04873734112f30/httptools-0.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:cfabda2a5bb85aa2a904ce06d974a3f30fb36cc63d7feaddec05d2050acede96", size = 88205, upload-time = "2025-10-10T03:55:00.389Z" }, -] - [[package]] name = "httpx" version = "0.28.1" @@ -1736,50 +1530,23 @@ wheels = [ name = "nemo-switchyard" version = "0.2.0" source = { editable = "." } -dependencies = [ - { name = "anthropic" }, - { name = "httpx" }, - { name = "openai" }, - { name = "pydantic" }, -] [package.optional-dependencies] -all = [ - { name = "ddtrace" }, - { name = "fastapi" }, - { name = "prompt-toolkit" }, - { name = "sse-starlette" }, - { name = "uvicorn", extra = ["standard"] }, -] cli = [ { name = "prompt-toolkit" }, ] -server = [ - { name = "fastapi" }, - { name = "sse-starlette" }, - { name = "uvicorn", extra = ["standard"] }, -] -tracing = [ - { name = "ddtrace" }, -] [package.dev-dependencies] dev = [ { name = "harbor" }, - { name = "httpx" }, { name = "maturin" }, { name = "mypy" }, - { name = "nemo-switchyard", extra = ["server"] }, - { name = "prometheus-client" }, + { name = "nemo-switchyard", extra = ["cli"] }, { name = "pytest" }, { name = "pytest-asyncio" }, - { name = "pytest-cov" }, { name = "pytest-markdown-docs" }, - { name = "pytest-mock" }, { name = "pytest-timeout" }, - { name = "respx" }, { name = "ruff" }, - { name = "socksio" }, ] docs = [ { name = "mkdocs" }, @@ -1788,37 +1555,20 @@ docs = [ ] [package.metadata] -requires-dist = [ - { name = "anthropic", specifier = ">=0.99.0,<1.0" }, - { name = "ddtrace", marker = "extra == 'tracing'", specifier = ">=2.9,<4" }, - { name = "fastapi", marker = "extra == 'server'", specifier = ">=0.136.1,<1.0" }, - { name = "httpx", specifier = ">=0.28.1,<1.0" }, - { name = "nemo-switchyard", extras = ["server", "cli", "tracing"], marker = "extra == 'all'" }, - { name = "openai", specifier = ">=2.7,<3.0" }, - { name = "prompt-toolkit", marker = "extra == 'cli'", specifier = ">=3.0.52,<4.0" }, - { name = "pydantic", specifier = ">=2.13.3,<3.0" }, - { name = "sse-starlette", marker = "extra == 'server'", specifier = ">=3.4.1,<4.0" }, - { name = "uvicorn", extras = ["standard"], marker = "extra == 'server'", specifier = ">=0.46.0,<1.0" }, -] -provides-extras = ["server", "cli", "tracing", "all"] +requires-dist = [{ name = "prompt-toolkit", marker = "extra == 'cli'", specifier = ">=3.0.52,<4.0" }] +provides-extras = ["cli"] [package.metadata.requires-dev] dev = [ { name = "harbor", marker = "python_full_version >= '3.12'", git = "https://github.com/harbor-framework/harbor.git?rev=v0.6.4" }, - { name = "httpx", specifier = ">=0.28.1,<1.0" }, { name = "maturin", specifier = ">=1.9,<2.0" }, { name = "mypy", specifier = ">=1.20.2,<2.0" }, - { name = "nemo-switchyard", extras = ["server"] }, - { name = "prometheus-client", specifier = ">=0.21.0,<1.0" }, + { name = "nemo-switchyard", extras = ["cli"] }, { name = "pytest", specifier = ">=9.0.3,<10.0" }, { name = "pytest-asyncio", specifier = ">=1.3.0,<2.0" }, - { name = "pytest-cov", specifier = ">=7.1.0,<8.0" }, { name = "pytest-markdown-docs", specifier = ">=0.9.2" }, - { name = "pytest-mock", specifier = ">=3.15.1,<4.0" }, { name = "pytest-timeout", specifier = ">=2.4.0,<3.0" }, - { name = "respx", specifier = ">=0.23.1,<1.0" }, { name = "ruff", specifier = ">=0.15.12,<1.0" }, - { name = "socksio", specifier = ">=1.0.0,<2.0" }, ] docs = [ { name = "mkdocs", specifier = ">=1.6.0,<2.0" }, @@ -1906,18 +1656,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f2/40/f090499f10514515081d09cb9da09f25b821eb20497e9423afe4f07b4ecf/openai-2.34.0-py3-none-any.whl", hash = "sha256:c996a71b1a210f3569844572ad4c609307e978515fb76877cf449b72596e549e", size = 1316535, upload-time = "2026-05-04T17:34:06.773Z" }, ] -[[package]] -name = "opentelemetry-api" -version = "1.42.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b4/1c/125e1c936c0873796771b7f04f6c93b9f1bf5d424cea90fda94a99f61da8/opentelemetry_api-1.42.1.tar.gz", hash = "sha256:56c63bea9f77b62856be8c47600474acad853b2924b99b1687c4cb6297166716", size = 72296, upload-time = "2026-05-21T16:32:49.335Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/ca/9520cc1f3dfbbd03ac5903bbf55833e257bc64b1cf30fa8b0d6df374d821/opentelemetry_api-1.42.1-py3-none-any.whl", hash = "sha256:51a69edacadbc03a8950ace1c4c21099cacc538820ac2c9e36277e78cebba714", size = 61311, upload-time = "2026-05-21T16:32:28.822Z" }, -] - [[package]] name = "packaging" version = "26.2" @@ -2030,15 +1768,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/22/aa/ff2e09f99f95ea96fddeb373646bf907dd89a24fc00b5d38e5674ca7c9ca/postgrest-2.30.0-py3-none-any.whl", hash = "sha256:30631e7993da542419f4217cf3b60aa641084731ea15e66a18526a3a52e40a7d", size = 23108, upload-time = "2026-05-06T17:35:20.531Z" }, ] -[[package]] -name = "prometheus-client" -version = "0.25.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1b/fb/d9aa83ffe43ce1f19e557c0971d04b90561b0cfd50762aafb01968285553/prometheus_client-0.25.0.tar.gz", hash = "sha256:5e373b75c31afb3c86f1a52fa1ad470c9aace18082d39ec0d2f918d11cc9ba28", size = 86035, upload-time = "2026-04-09T19:53:42.359Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8d/9b/d4b1e644385499c8346fa9b622a3f030dce14cd6ef8a1871c221a17a67e7/prometheus_client-0.25.0-py3-none-any.whl", hash = "sha256:d5aec89e349a6ec230805d0df882f3807f74fd6c1a2fa86864e3c2279059fed1", size = 64154, upload-time = "2026-04-09T19:53:41.324Z" }, -] - [[package]] name = "prompt-toolkit" version = "3.0.52" @@ -2457,20 +2186,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl", hash = "sha256:611e26147c7f77640e6d0a92a38ed17c3e9848063698d5c93d5aa7aa11cebff5", size = 15075, upload-time = "2025-11-10T16:07:45.537Z" }, ] -[[package]] -name = "pytest-cov" -version = "7.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "coverage" }, - { name = "pluggy" }, - { name = "pytest" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, -] - [[package]] name = "pytest-markdown-docs" version = "0.9.2" @@ -2484,18 +2199,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0c/b9/c3df11997d29e69b3f8edae1e903bf44eaf4774ccf4c5b6ddcebde88931c/pytest_markdown_docs-0.9.2-py3-none-any.whl", hash = "sha256:9c05a5bee48214cb36583d4a5131d3a23b327a8d82c92ae860106053fd7d1f9e", size = 13536, upload-time = "2026-03-23T12:35:03.862Z" }, ] -[[package]] -name = "pytest-mock" -version = "3.15.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pytest" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/68/14/eb014d26be205d38ad5ad20d9a80f7d201472e08167f0bb4361e251084a9/pytest_mock-3.15.1.tar.gz", hash = "sha256:1849a238f6f396da19762269de72cb1814ab44416fa73a8686deac10b0d87a0f", size = 34036, upload-time = "2025-09-16T16:37:27.081Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/cc/06253936f4a7fa2e0f48dfe6d851d9c56df896a9ab09ac019d70b760619c/pytest_mock-3.15.1-py3-none-any.whl", hash = "sha256:0a25e2eb88fe5168d535041d09a4529a188176ae608a6d249ee65abc0949630d", size = 10095, upload-time = "2025-09-16T16:37:25.734Z" }, -] - [[package]] name = "pytest-timeout" version = "2.4.0" @@ -2743,18 +2446,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/8e/7540e8a2036f79a125c1d2ebadf69ed7901608859186c856fa0388ef4197/requests-2.33.1-py3-none-any.whl", hash = "sha256:4e6d1ef462f3626a1f0a0a9c42dd93c63bad33f9f1c1937509b8c5c8718ab56a", size = 64947, upload-time = "2026-03-30T16:09:13.83Z" }, ] -[[package]] -name = "respx" -version = "0.23.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "httpx" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/43/98/4e55c9c486404ec12373708d015ebce157966965a5ebe7f28ff2c784d41b/respx-0.23.1.tar.gz", hash = "sha256:242dcc6ce6b5b9bf621f5870c82a63997e8e82bc7c947f9ffe272b8f3dd5a780", size = 29243, upload-time = "2026-04-08T14:37:16.008Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/4a/221da6ca167db45693d8d26c7dc79ccfc978a440251bf6721c9aaf251ac0/respx-0.23.1-py2.py3-none-any.whl", hash = "sha256:b18004b029935384bccfa6d7d9d74b4ec9af73a081cc28600fffc0447f4b8c1a", size = 25557, upload-time = "2026-04-08T14:37:14.613Z" }, -] - [[package]] name = "rich" version = "14.3.4" @@ -2923,15 +2614,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, ] -[[package]] -name = "socksio" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f8/5c/48a7d9495be3d1c651198fd99dbb6ce190e2274d0f28b9051307bdec6b85/socksio-1.0.0.tar.gz", hash = "sha256:f88beb3da5b5c38b9890469de67d0cb0f9d494b78b106ca1845f96c10b91c4ac", size = 19055, upload-time = "2020-04-17T15:50:34.664Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/37/c3/6eeb6034408dac0fa653d126c9204ade96b819c936e136c5e8a6897eee9c/socksio-1.0.0-py3-none-any.whl", hash = "sha256:95dc1f15f9b34e8d7b16f06d74b8ccf48f609af32ab33c608d08761c5dcbb1f3", size = 12763, upload-time = "2020-04-17T15:50:31.878Z" }, -] - [[package]] name = "sse-starlette" version = "3.4.1" @@ -3211,49 +2893,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/31/a3/5b1562db76a5a488274b2332a97199b32d0442aca0ed193697fd47786316/uvicorn-0.46.0-py3-none-any.whl", hash = "sha256:bbebbcbed972d162afca128605223022bedd345b7bc7855ce66deb31487a9048", size = 70926, upload-time = "2026-04-23T07:15:58.355Z" }, ] -[package.optional-dependencies] -standard = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "httptools" }, - { name = "python-dotenv" }, - { name = "pyyaml" }, - { name = "uvloop", marker = "platform_python_implementation != 'PyPy' and sys_platform != 'cygwin' and sys_platform != 'win32'" }, - { name = "watchfiles" }, - { name = "websockets" }, -] - -[[package]] -name = "uvloop" -version = "0.22.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/06/f0/18d39dbd1971d6d62c4629cc7fa67f74821b0dc1f5a77af43719de7936a7/uvloop-0.22.1.tar.gz", hash = "sha256:6c84bae345b9147082b17371e3dd5d42775bddce91f885499017f4607fdaf39f", size = 2443250, upload-time = "2025-10-16T22:17:19.342Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/ff/7f72e8170be527b4977b033239a83a68d5c881cc4775fca255c677f7ac5d/uvloop-0.22.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:fe94b4564e865d968414598eea1a6de60adba0c040ba4ed05ac1300de402cd42", size = 1359936, upload-time = "2025-10-16T22:16:29.436Z" }, - { url = "https://files.pythonhosted.org/packages/c3/c6/e5d433f88fd54d81ef4be58b2b7b0cea13c442454a1db703a1eea0db1a59/uvloop-0.22.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:51eb9bd88391483410daad430813d982010f9c9c89512321f5b60e2cddbdddd6", size = 752769, upload-time = "2025-10-16T22:16:30.493Z" }, - { url = "https://files.pythonhosted.org/packages/24/68/a6ac446820273e71aa762fa21cdcc09861edd3536ff47c5cd3b7afb10eeb/uvloop-0.22.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:700e674a166ca5778255e0e1dc4e9d79ab2acc57b9171b79e65feba7184b3370", size = 4317413, upload-time = "2025-10-16T22:16:31.644Z" }, - { url = "https://files.pythonhosted.org/packages/5f/6f/e62b4dfc7ad6518e7eff2516f680d02a0f6eb62c0c212e152ca708a0085e/uvloop-0.22.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b5b1ac819a3f946d3b2ee07f09149578ae76066d70b44df3fa990add49a82e4", size = 4426307, upload-time = "2025-10-16T22:16:32.917Z" }, - { url = "https://files.pythonhosted.org/packages/90/60/97362554ac21e20e81bcef1150cb2a7e4ffdaf8ea1e5b2e8bf7a053caa18/uvloop-0.22.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e047cc068570bac9866237739607d1313b9253c3051ad84738cbb095be0537b2", size = 4131970, upload-time = "2025-10-16T22:16:34.015Z" }, - { url = "https://files.pythonhosted.org/packages/99/39/6b3f7d234ba3964c428a6e40006340f53ba37993f46ed6e111c6e9141d18/uvloop-0.22.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:512fec6815e2dd45161054592441ef76c830eddaad55c8aa30952e6fe1ed07c0", size = 4296343, upload-time = "2025-10-16T22:16:35.149Z" }, - { url = "https://files.pythonhosted.org/packages/89/8c/182a2a593195bfd39842ea68ebc084e20c850806117213f5a299dfc513d9/uvloop-0.22.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:561577354eb94200d75aca23fbde86ee11be36b00e52a4eaf8f50fb0c86b7705", size = 1358611, upload-time = "2025-10-16T22:16:36.833Z" }, - { url = "https://files.pythonhosted.org/packages/d2/14/e301ee96a6dc95224b6f1162cd3312f6d1217be3907b79173b06785f2fe7/uvloop-0.22.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1cdf5192ab3e674ca26da2eada35b288d2fa49fdd0f357a19f0e7c4e7d5077c8", size = 751811, upload-time = "2025-10-16T22:16:38.275Z" }, - { url = "https://files.pythonhosted.org/packages/b7/02/654426ce265ac19e2980bfd9ea6590ca96a56f10c76e63801a2df01c0486/uvloop-0.22.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e2ea3d6190a2968f4a14a23019d3b16870dd2190cd69c8180f7c632d21de68d", size = 4288562, upload-time = "2025-10-16T22:16:39.375Z" }, - { url = "https://files.pythonhosted.org/packages/15/c0/0be24758891ef825f2065cd5db8741aaddabe3e248ee6acc5e8a80f04005/uvloop-0.22.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0530a5fbad9c9e4ee3f2b33b148c6a64d47bbad8000ea63704fa8260f4cf728e", size = 4366890, upload-time = "2025-10-16T22:16:40.547Z" }, - { url = "https://files.pythonhosted.org/packages/d2/53/8369e5219a5855869bcee5f4d317f6da0e2c669aecf0ef7d371e3d084449/uvloop-0.22.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bc5ef13bbc10b5335792360623cc378d52d7e62c2de64660616478c32cd0598e", size = 4119472, upload-time = "2025-10-16T22:16:41.694Z" }, - { url = "https://files.pythonhosted.org/packages/f8/ba/d69adbe699b768f6b29a5eec7b47dd610bd17a69de51b251126a801369ea/uvloop-0.22.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1f38ec5e3f18c8a10ded09742f7fb8de0108796eb673f30ce7762ce1b8550cad", size = 4239051, upload-time = "2025-10-16T22:16:43.224Z" }, - { url = "https://files.pythonhosted.org/packages/90/cd/b62bdeaa429758aee8de8b00ac0dd26593a9de93d302bff3d21439e9791d/uvloop-0.22.1-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3879b88423ec7e97cd4eba2a443aa26ed4e59b45e6b76aabf13fe2f27023a142", size = 1362067, upload-time = "2025-10-16T22:16:44.503Z" }, - { url = "https://files.pythonhosted.org/packages/0d/f8/a132124dfda0777e489ca86732e85e69afcd1ff7686647000050ba670689/uvloop-0.22.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4baa86acedf1d62115c1dc6ad1e17134476688f08c6efd8a2ab076e815665c74", size = 752423, upload-time = "2025-10-16T22:16:45.968Z" }, - { url = "https://files.pythonhosted.org/packages/a3/94/94af78c156f88da4b3a733773ad5ba0b164393e357cc4bd0ab2e2677a7d6/uvloop-0.22.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:297c27d8003520596236bdb2335e6b3f649480bd09e00d1e3a99144b691d2a35", size = 4272437, upload-time = "2025-10-16T22:16:47.451Z" }, - { url = "https://files.pythonhosted.org/packages/b5/35/60249e9fd07b32c665192cec7af29e06c7cd96fa1d08b84f012a56a0b38e/uvloop-0.22.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c1955d5a1dd43198244d47664a5858082a3239766a839b2102a269aaff7a4e25", size = 4292101, upload-time = "2025-10-16T22:16:49.318Z" }, - { url = "https://files.pythonhosted.org/packages/02/62/67d382dfcb25d0a98ce73c11ed1a6fba5037a1a1d533dcbb7cab033a2636/uvloop-0.22.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b31dc2fccbd42adc73bc4e7cdbae4fc5086cf378979e53ca5d0301838c5682c6", size = 4114158, upload-time = "2025-10-16T22:16:50.517Z" }, - { url = "https://files.pythonhosted.org/packages/f0/7a/f1171b4a882a5d13c8b7576f348acfe6074d72eaf52cccef752f748d4a9f/uvloop-0.22.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:93f617675b2d03af4e72a5333ef89450dfaa5321303ede6e67ba9c9d26878079", size = 4177360, upload-time = "2025-10-16T22:16:52.646Z" }, - { url = "https://files.pythonhosted.org/packages/79/7b/b01414f31546caf0919da80ad57cbfe24c56b151d12af68cee1b04922ca8/uvloop-0.22.1-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:37554f70528f60cad66945b885eb01f1bb514f132d92b6eeed1c90fd54ed6289", size = 1454790, upload-time = "2025-10-16T22:16:54.355Z" }, - { url = "https://files.pythonhosted.org/packages/d4/31/0bb232318dd838cad3fa8fb0c68c8b40e1145b32025581975e18b11fab40/uvloop-0.22.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:b76324e2dc033a0b2f435f33eb88ff9913c156ef78e153fb210e03c13da746b3", size = 796783, upload-time = "2025-10-16T22:16:55.906Z" }, - { url = "https://files.pythonhosted.org/packages/42/38/c9b09f3271a7a723a5de69f8e237ab8e7803183131bc57c890db0b6bb872/uvloop-0.22.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:badb4d8e58ee08dad957002027830d5c3b06aea446a6a3744483c2b3b745345c", size = 4647548, upload-time = "2025-10-16T22:16:57.008Z" }, - { url = "https://files.pythonhosted.org/packages/c1/37/945b4ca0ac27e3dc4952642d4c900edd030b3da6c9634875af6e13ae80e5/uvloop-0.22.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b91328c72635f6f9e0282e4a57da7470c7350ab1c9f48546c0f2866205349d21", size = 4467065, upload-time = "2025-10-16T22:16:58.206Z" }, - { url = "https://files.pythonhosted.org/packages/97/cc/48d232f33d60e2e2e0b42f4e73455b146b76ebe216487e862700457fbf3c/uvloop-0.22.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:daf620c2995d193449393d6c62131b3fbd40a63bf7b307a1527856ace637fe88", size = 4328384, upload-time = "2025-10-16T22:16:59.36Z" }, - { url = "https://files.pythonhosted.org/packages/e4/16/c1fd27e9549f3c4baf1dc9c20c456cd2f822dbf8de9f463824b0c0357e06/uvloop-0.22.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6cde23eeda1a25c75b2e07d39970f3374105d5eafbaab2a4482be82f272d5a5e", size = 4296730, upload-time = "2025-10-16T22:17:00.744Z" }, -] - [[package]] name = "watchdog" version = "4.0.2" @@ -3278,76 +2917,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b0/0b/43b96a9ecdd65ff5545b1b13b687ca486da5c6249475b1a45f24d63a1858/watchdog-4.0.2-py3-none-win_ia64.whl", hash = "sha256:baececaa8edff42cd16558a639a9b0ddf425f93d892e8392a56bf904f5eff22c", size = 82933, upload-time = "2024-08-11T07:37:59.573Z" }, ] -[[package]] -name = "watchfiles" -version = "1.1.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c2/c9/8869df9b2a2d6c59d79220a4db37679e74f807c559ffe5265e08b227a210/watchfiles-1.1.1.tar.gz", hash = "sha256:a173cb5c16c4f40ab19cecf48a534c409f7ea983ab8fed0741304a1c0a31b3f2", size = 94440, upload-time = "2025-10-14T15:06:21.08Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/74/d5/f039e7e3c639d9b1d09b07ea412a6806d38123f0508e5f9b48a87b0a76cc/watchfiles-1.1.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:8c89f9f2f740a6b7dcc753140dd5e1ab9215966f7a3530d0c0705c83b401bd7d", size = 404745, upload-time = "2025-10-14T15:04:46.731Z" }, - { url = "https://files.pythonhosted.org/packages/a5/96/a881a13aa1349827490dab2d363c8039527060cfcc2c92cc6d13d1b1049e/watchfiles-1.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bd404be08018c37350f0d6e34676bd1e2889990117a2b90070b3007f172d0610", size = 391769, upload-time = "2025-10-14T15:04:48.003Z" }, - { url = "https://files.pythonhosted.org/packages/4b/5b/d3b460364aeb8da471c1989238ea0e56bec24b6042a68046adf3d9ddb01c/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8526e8f916bb5b9a0a777c8317c23ce65de259422bba5b31325a6fa6029d33af", size = 449374, upload-time = "2025-10-14T15:04:49.179Z" }, - { url = "https://files.pythonhosted.org/packages/b9/44/5769cb62d4ed055cb17417c0a109a92f007114a4e07f30812a73a4efdb11/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2edc3553362b1c38d9f06242416a5d8e9fe235c204a4072e988ce2e5bb1f69f6", size = 459485, upload-time = "2025-10-14T15:04:50.155Z" }, - { url = "https://files.pythonhosted.org/packages/19/0c/286b6301ded2eccd4ffd0041a1b726afda999926cf720aab63adb68a1e36/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:30f7da3fb3f2844259cba4720c3fc7138eb0f7b659c38f3bfa65084c7fc7abce", size = 488813, upload-time = "2025-10-14T15:04:51.059Z" }, - { url = "https://files.pythonhosted.org/packages/c7/2b/8530ed41112dd4a22f4dcfdb5ccf6a1baad1ff6eed8dc5a5f09e7e8c41c7/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8979280bdafff686ba5e4d8f97840f929a87ed9cdf133cbbd42f7766774d2aa", size = 594816, upload-time = "2025-10-14T15:04:52.031Z" }, - { url = "https://files.pythonhosted.org/packages/ce/d2/f5f9fb49489f184f18470d4f99f4e862a4b3e9ac2865688eb2099e3d837a/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dcc5c24523771db3a294c77d94771abcfcb82a0e0ee8efd910c37c59ec1b31bb", size = 475186, upload-time = "2025-10-14T15:04:53.064Z" }, - { url = "https://files.pythonhosted.org/packages/cf/68/5707da262a119fb06fbe214d82dd1fe4a6f4af32d2d14de368d0349eb52a/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1db5d7ae38ff20153d542460752ff397fcf5c96090c1230803713cf3147a6803", size = 456812, upload-time = "2025-10-14T15:04:55.174Z" }, - { url = "https://files.pythonhosted.org/packages/66/ab/3cbb8756323e8f9b6f9acb9ef4ec26d42b2109bce830cc1f3468df20511d/watchfiles-1.1.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:28475ddbde92df1874b6c5c8aaeb24ad5be47a11f87cde5a28ef3835932e3e94", size = 630196, upload-time = "2025-10-14T15:04:56.22Z" }, - { url = "https://files.pythonhosted.org/packages/78/46/7152ec29b8335f80167928944a94955015a345440f524d2dfe63fc2f437b/watchfiles-1.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:36193ed342f5b9842edd3532729a2ad55c4160ffcfa3700e0d54be496b70dd43", size = 622657, upload-time = "2025-10-14T15:04:57.521Z" }, - { url = "https://files.pythonhosted.org/packages/0a/bf/95895e78dd75efe9a7f31733607f384b42eb5feb54bd2eb6ed57cc2e94f4/watchfiles-1.1.1-cp312-cp312-win32.whl", hash = "sha256:859e43a1951717cc8de7f4c77674a6d389b106361585951d9e69572823f311d9", size = 272042, upload-time = "2025-10-14T15:04:59.046Z" }, - { url = "https://files.pythonhosted.org/packages/87/0a/90eb755f568de2688cb220171c4191df932232c20946966c27a59c400850/watchfiles-1.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:91d4c9a823a8c987cce8fa2690923b069966dabb196dd8d137ea2cede885fde9", size = 288410, upload-time = "2025-10-14T15:05:00.081Z" }, - { url = "https://files.pythonhosted.org/packages/36/76/f322701530586922fbd6723c4f91ace21364924822a8772c549483abed13/watchfiles-1.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:a625815d4a2bdca61953dbba5a39d60164451ef34c88d751f6c368c3ea73d404", size = 278209, upload-time = "2025-10-14T15:05:01.168Z" }, - { url = "https://files.pythonhosted.org/packages/bb/f4/f750b29225fe77139f7ae5de89d4949f5a99f934c65a1f1c0b248f26f747/watchfiles-1.1.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:130e4876309e8686a5e37dba7d5e9bc77e6ed908266996ca26572437a5271e18", size = 404321, upload-time = "2025-10-14T15:05:02.063Z" }, - { url = "https://files.pythonhosted.org/packages/2b/f9/f07a295cde762644aa4c4bb0f88921d2d141af45e735b965fb2e87858328/watchfiles-1.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5f3bde70f157f84ece3765b42b4a52c6ac1a50334903c6eaf765362f6ccca88a", size = 391783, upload-time = "2025-10-14T15:05:03.052Z" }, - { url = "https://files.pythonhosted.org/packages/bc/11/fc2502457e0bea39a5c958d86d2cb69e407a4d00b85735ca724bfa6e0d1a/watchfiles-1.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:14e0b1fe858430fc0251737ef3824c54027bedb8c37c38114488b8e131cf8219", size = 449279, upload-time = "2025-10-14T15:05:04.004Z" }, - { url = "https://files.pythonhosted.org/packages/e3/1f/d66bc15ea0b728df3ed96a539c777acfcad0eb78555ad9efcaa1274688f0/watchfiles-1.1.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f27db948078f3823a6bb3b465180db8ebecf26dd5dae6f6180bd87383b6b4428", size = 459405, upload-time = "2025-10-14T15:05:04.942Z" }, - { url = "https://files.pythonhosted.org/packages/be/90/9f4a65c0aec3ccf032703e6db02d89a157462fbb2cf20dd415128251cac0/watchfiles-1.1.1-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:059098c3a429f62fc98e8ec62b982230ef2c8df68c79e826e37b895bc359a9c0", size = 488976, upload-time = "2025-10-14T15:05:05.905Z" }, - { url = "https://files.pythonhosted.org/packages/37/57/ee347af605d867f712be7029bb94c8c071732a4b44792e3176fa3c612d39/watchfiles-1.1.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfb5862016acc9b869bb57284e6cb35fdf8e22fe59f7548858e2f971d045f150", size = 595506, upload-time = "2025-10-14T15:05:06.906Z" }, - { url = "https://files.pythonhosted.org/packages/a8/78/cc5ab0b86c122047f75e8fc471c67a04dee395daf847d3e59381996c8707/watchfiles-1.1.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:319b27255aacd9923b8a276bb14d21a5f7ff82564c744235fc5eae58d95422ae", size = 474936, upload-time = "2025-10-14T15:05:07.906Z" }, - { url = "https://files.pythonhosted.org/packages/62/da/def65b170a3815af7bd40a3e7010bf6ab53089ef1b75d05dd5385b87cf08/watchfiles-1.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c755367e51db90e75b19454b680903631d41f9e3607fbd941d296a020c2d752d", size = 456147, upload-time = "2025-10-14T15:05:09.138Z" }, - { url = "https://files.pythonhosted.org/packages/57/99/da6573ba71166e82d288d4df0839128004c67d2778d3b566c138695f5c0b/watchfiles-1.1.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:c22c776292a23bfc7237a98f791b9ad3144b02116ff10d820829ce62dff46d0b", size = 630007, upload-time = "2025-10-14T15:05:10.117Z" }, - { url = "https://files.pythonhosted.org/packages/a8/51/7439c4dd39511368849eb1e53279cd3454b4a4dbace80bab88feeb83c6b5/watchfiles-1.1.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:3a476189be23c3686bc2f4321dd501cb329c0a0469e77b7b534ee10129ae6374", size = 622280, upload-time = "2025-10-14T15:05:11.146Z" }, - { url = "https://files.pythonhosted.org/packages/95/9c/8ed97d4bba5db6fdcdb2b298d3898f2dd5c20f6b73aee04eabe56c59677e/watchfiles-1.1.1-cp313-cp313-win32.whl", hash = "sha256:bf0a91bfb5574a2f7fc223cf95eeea79abfefa404bf1ea5e339c0c1560ae99a0", size = 272056, upload-time = "2025-10-14T15:05:12.156Z" }, - { url = "https://files.pythonhosted.org/packages/1f/f3/c14e28429f744a260d8ceae18bf58c1d5fa56b50d006a7a9f80e1882cb0d/watchfiles-1.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:52e06553899e11e8074503c8e716d574adeeb7e68913115c4b3653c53f9bae42", size = 288162, upload-time = "2025-10-14T15:05:13.208Z" }, - { url = "https://files.pythonhosted.org/packages/dc/61/fe0e56c40d5cd29523e398d31153218718c5786b5e636d9ae8ae79453d27/watchfiles-1.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:ac3cc5759570cd02662b15fbcd9d917f7ecd47efe0d6b40474eafd246f91ea18", size = 277909, upload-time = "2025-10-14T15:05:14.49Z" }, - { url = "https://files.pythonhosted.org/packages/79/42/e0a7d749626f1e28c7108a99fb9bf524b501bbbeb9b261ceecde644d5a07/watchfiles-1.1.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:563b116874a9a7ce6f96f87cd0b94f7faf92d08d0021e837796f0a14318ef8da", size = 403389, upload-time = "2025-10-14T15:05:15.777Z" }, - { url = "https://files.pythonhosted.org/packages/15/49/08732f90ce0fbbc13913f9f215c689cfc9ced345fb1bcd8829a50007cc8d/watchfiles-1.1.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3ad9fe1dae4ab4212d8c91e80b832425e24f421703b5a42ef2e4a1e215aff051", size = 389964, upload-time = "2025-10-14T15:05:16.85Z" }, - { url = "https://files.pythonhosted.org/packages/27/0d/7c315d4bd5f2538910491a0393c56bf70d333d51bc5b34bee8e68e8cea19/watchfiles-1.1.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce70f96a46b894b36eba678f153f052967a0d06d5b5a19b336ab0dbbd029f73e", size = 448114, upload-time = "2025-10-14T15:05:17.876Z" }, - { url = "https://files.pythonhosted.org/packages/c3/24/9e096de47a4d11bc4df41e9d1e61776393eac4cb6eb11b3e23315b78b2cc/watchfiles-1.1.1-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cb467c999c2eff23a6417e58d75e5828716f42ed8289fe6b77a7e5a91036ca70", size = 460264, upload-time = "2025-10-14T15:05:18.962Z" }, - { url = "https://files.pythonhosted.org/packages/cc/0f/e8dea6375f1d3ba5fcb0b3583e2b493e77379834c74fd5a22d66d85d6540/watchfiles-1.1.1-cp313-cp313t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:836398932192dae4146c8f6f737d74baeac8b70ce14831a239bdb1ca882fc261", size = 487877, upload-time = "2025-10-14T15:05:20.094Z" }, - { url = "https://files.pythonhosted.org/packages/ac/5b/df24cfc6424a12deb41503b64d42fbea6b8cb357ec62ca84a5a3476f654a/watchfiles-1.1.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:743185e7372b7bc7c389e1badcc606931a827112fbbd37f14c537320fca08620", size = 595176, upload-time = "2025-10-14T15:05:21.134Z" }, - { url = "https://files.pythonhosted.org/packages/8f/b5/853b6757f7347de4e9b37e8cc3289283fb983cba1ab4d2d7144694871d9c/watchfiles-1.1.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:afaeff7696e0ad9f02cbb8f56365ff4686ab205fcf9c4c5b6fdfaaa16549dd04", size = 473577, upload-time = "2025-10-14T15:05:22.306Z" }, - { url = "https://files.pythonhosted.org/packages/e1/f7/0a4467be0a56e80447c8529c9fce5b38eab4f513cb3d9bf82e7392a5696b/watchfiles-1.1.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f7eb7da0eb23aa2ba036d4f616d46906013a68caf61b7fdbe42fc8b25132e77", size = 455425, upload-time = "2025-10-14T15:05:23.348Z" }, - { url = "https://files.pythonhosted.org/packages/8e/e0/82583485ea00137ddf69bc84a2db88bd92ab4a6e3c405e5fb878ead8d0e7/watchfiles-1.1.1-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:831a62658609f0e5c64178211c942ace999517f5770fe9436be4c2faeba0c0ef", size = 628826, upload-time = "2025-10-14T15:05:24.398Z" }, - { url = "https://files.pythonhosted.org/packages/28/9a/a785356fccf9fae84c0cc90570f11702ae9571036fb25932f1242c82191c/watchfiles-1.1.1-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:f9a2ae5c91cecc9edd47e041a930490c31c3afb1f5e6d71de3dc671bfaca02bf", size = 622208, upload-time = "2025-10-14T15:05:25.45Z" }, - { url = "https://files.pythonhosted.org/packages/c3/f4/0872229324ef69b2c3edec35e84bd57a1289e7d3fe74588048ed8947a323/watchfiles-1.1.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:d1715143123baeeaeadec0528bb7441103979a1d5f6fd0e1f915383fea7ea6d5", size = 404315, upload-time = "2025-10-14T15:05:26.501Z" }, - { url = "https://files.pythonhosted.org/packages/7b/22/16d5331eaed1cb107b873f6ae1b69e9ced582fcf0c59a50cd84f403b1c32/watchfiles-1.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:39574d6370c4579d7f5d0ad940ce5b20db0e4117444e39b6d8f99db5676c52fd", size = 390869, upload-time = "2025-10-14T15:05:27.649Z" }, - { url = "https://files.pythonhosted.org/packages/b2/7e/5643bfff5acb6539b18483128fdc0ef2cccc94a5b8fbda130c823e8ed636/watchfiles-1.1.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7365b92c2e69ee952902e8f70f3ba6360d0d596d9299d55d7d386df84b6941fb", size = 449919, upload-time = "2025-10-14T15:05:28.701Z" }, - { url = "https://files.pythonhosted.org/packages/51/2e/c410993ba5025a9f9357c376f48976ef0e1b1aefb73b97a5ae01a5972755/watchfiles-1.1.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bfff9740c69c0e4ed32416f013f3c45e2ae42ccedd1167ef2d805c000b6c71a5", size = 460845, upload-time = "2025-10-14T15:05:30.064Z" }, - { url = "https://files.pythonhosted.org/packages/8e/a4/2df3b404469122e8680f0fcd06079317e48db58a2da2950fb45020947734/watchfiles-1.1.1-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b27cf2eb1dda37b2089e3907d8ea92922b673c0c427886d4edc6b94d8dfe5db3", size = 489027, upload-time = "2025-10-14T15:05:31.064Z" }, - { url = "https://files.pythonhosted.org/packages/ea/84/4587ba5b1f267167ee715b7f66e6382cca6938e0a4b870adad93e44747e6/watchfiles-1.1.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:526e86aced14a65a5b0ec50827c745597c782ff46b571dbfe46192ab9e0b3c33", size = 595615, upload-time = "2025-10-14T15:05:32.074Z" }, - { url = "https://files.pythonhosted.org/packages/6a/0f/c6988c91d06e93cd0bb3d4a808bcf32375ca1904609835c3031799e3ecae/watchfiles-1.1.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:04e78dd0b6352db95507fd8cb46f39d185cf8c74e4cf1e4fbad1d3df96faf510", size = 474836, upload-time = "2025-10-14T15:05:33.209Z" }, - { url = "https://files.pythonhosted.org/packages/b4/36/ded8aebea91919485b7bbabbd14f5f359326cb5ec218cd67074d1e426d74/watchfiles-1.1.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c85794a4cfa094714fb9c08d4a218375b2b95b8ed1666e8677c349906246c05", size = 455099, upload-time = "2025-10-14T15:05:34.189Z" }, - { url = "https://files.pythonhosted.org/packages/98/e0/8c9bdba88af756a2fce230dd365fab2baf927ba42cd47521ee7498fd5211/watchfiles-1.1.1-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:74d5012b7630714b66be7b7b7a78855ef7ad58e8650c73afc4c076a1f480a8d6", size = 630626, upload-time = "2025-10-14T15:05:35.216Z" }, - { url = "https://files.pythonhosted.org/packages/2a/84/a95db05354bf2d19e438520d92a8ca475e578c647f78f53197f5a2f17aaf/watchfiles-1.1.1-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:8fbe85cb3201c7d380d3d0b90e63d520f15d6afe217165d7f98c9c649654db81", size = 622519, upload-time = "2025-10-14T15:05:36.259Z" }, - { url = "https://files.pythonhosted.org/packages/1d/ce/d8acdc8de545de995c339be67711e474c77d643555a9bb74a9334252bd55/watchfiles-1.1.1-cp314-cp314-win32.whl", hash = "sha256:3fa0b59c92278b5a7800d3ee7733da9d096d4aabcfabb9a928918bd276ef9b9b", size = 272078, upload-time = "2025-10-14T15:05:37.63Z" }, - { url = "https://files.pythonhosted.org/packages/c4/c9/a74487f72d0451524be827e8edec251da0cc1fcf111646a511ae752e1a3d/watchfiles-1.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:c2047d0b6cea13b3316bdbafbfa0c4228ae593d995030fda39089d36e64fc03a", size = 287664, upload-time = "2025-10-14T15:05:38.95Z" }, - { url = "https://files.pythonhosted.org/packages/df/b8/8ac000702cdd496cdce998c6f4ee0ca1f15977bba51bdf07d872ebdfc34c/watchfiles-1.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:842178b126593addc05acf6fce960d28bc5fae7afbaa2c6c1b3a7b9460e5be02", size = 277154, upload-time = "2025-10-14T15:05:39.954Z" }, - { url = "https://files.pythonhosted.org/packages/47/a8/e3af2184707c29f0f14b1963c0aace6529f9d1b8582d5b99f31bbf42f59e/watchfiles-1.1.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:88863fbbc1a7312972f1c511f202eb30866370ebb8493aef2812b9ff28156a21", size = 403820, upload-time = "2025-10-14T15:05:40.932Z" }, - { url = "https://files.pythonhosted.org/packages/c0/ec/e47e307c2f4bd75f9f9e8afbe3876679b18e1bcec449beca132a1c5ffb2d/watchfiles-1.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:55c7475190662e202c08c6c0f4d9e345a29367438cf8e8037f3155e10a88d5a5", size = 390510, upload-time = "2025-10-14T15:05:41.945Z" }, - { url = "https://files.pythonhosted.org/packages/d5/a0/ad235642118090f66e7b2f18fd5c42082418404a79205cdfca50b6309c13/watchfiles-1.1.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f53fa183d53a1d7a8852277c92b967ae99c2d4dcee2bfacff8868e6e30b15f7", size = 448408, upload-time = "2025-10-14T15:05:43.385Z" }, - { url = "https://files.pythonhosted.org/packages/df/85/97fa10fd5ff3332ae17e7e40e20784e419e28521549780869f1413742e9d/watchfiles-1.1.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6aae418a8b323732fa89721d86f39ec8f092fc2af67f4217a2b07fd3e93c6101", size = 458968, upload-time = "2025-10-14T15:05:44.404Z" }, - { url = "https://files.pythonhosted.org/packages/47/c2/9059c2e8966ea5ce678166617a7f75ecba6164375f3b288e50a40dc6d489/watchfiles-1.1.1-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f096076119da54a6080e8920cbdaac3dbee667eb91dcc5e5b78840b87415bd44", size = 488096, upload-time = "2025-10-14T15:05:45.398Z" }, - { url = "https://files.pythonhosted.org/packages/94/44/d90a9ec8ac309bc26db808a13e7bfc0e4e78b6fc051078a554e132e80160/watchfiles-1.1.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:00485f441d183717038ed2e887a7c868154f216877653121068107b227a2f64c", size = 596040, upload-time = "2025-10-14T15:05:46.502Z" }, - { url = "https://files.pythonhosted.org/packages/95/68/4e3479b20ca305cfc561db3ed207a8a1c745ee32bf24f2026a129d0ddb6e/watchfiles-1.1.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a55f3e9e493158d7bfdb60a1165035f1cf7d320914e7b7ea83fe22c6023b58fc", size = 473847, upload-time = "2025-10-14T15:05:47.484Z" }, - { url = "https://files.pythonhosted.org/packages/4f/55/2af26693fd15165c4ff7857e38330e1b61ab8c37d15dc79118cdba115b7a/watchfiles-1.1.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c91ed27800188c2ae96d16e3149f199d62f86c7af5f5f4d2c61a3ed8cd3666c", size = 455072, upload-time = "2025-10-14T15:05:48.928Z" }, - { url = "https://files.pythonhosted.org/packages/66/1d/d0d200b10c9311ec25d2273f8aad8c3ef7cc7ea11808022501811208a750/watchfiles-1.1.1-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:311ff15a0bae3714ffb603e6ba6dbfba4065ab60865d15a6ec544133bdb21099", size = 629104, upload-time = "2025-10-14T15:05:49.908Z" }, - { url = "https://files.pythonhosted.org/packages/e3/bd/fa9bb053192491b3867ba07d2343d9f2252e00811567d30ae8d0f78136fe/watchfiles-1.1.1-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:a916a2932da8f8ab582f242c065f5c81bed3462849ca79ee357dd9551b0e9b01", size = 622112, upload-time = "2025-10-14T15:05:50.941Z" }, -] - [[package]] name = "wcwidth" version = "0.7.0" @@ -3388,70 +2957,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/a8/5b41e0da817d64113292ab1f8247140aac61cbf6cfd085d6a0fa77f4984f/websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f", size = 169743, upload-time = "2025-03-05T20:03:39.41Z" }, ] -[[package]] -name = "wrapt" -version = "2.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2d/9f/06263fcd8ad6c405f05a3905fd7a84dd3176eb5ad46e44bccc0cd16348bb/wrapt-2.2.1.tar.gz", hash = "sha256:6744f504375775d7609c82c8d3d94af1c9a6f05586984536905908ba905277b9", size = 127620, upload-time = "2026-05-22T14:49:43.056Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/89/0c/bfae7b9401583b6d05938cd16dedc43857d96da2f8a3d50d78cc515bf6ff/wrapt-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3ffad790d9d11d8ecf9f17c4bb671a5b4089e4d8b575c46c5129597f41f836b0", size = 81021, upload-time = "2026-05-22T14:48:00.313Z" }, - { url = "https://files.pythonhosted.org/packages/26/58/80f6a6599f933f4caecc1cb3ee88a04faf81e8b9bddbd6109c688dd63e0f/wrapt-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:628f5220c7a904d5fc78f7075c8d7871433eb6d035c94728a22fdf85f193d2a8", size = 81692, upload-time = "2026-05-22T14:48:01.49Z" }, - { url = "https://files.pythonhosted.org/packages/17/93/fb357cc7847c58a8ae790be718903afa81a28d23e642c843dc4129e8a0b2/wrapt-2.2.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:61acce4257a9883669703c525447c5b4c392edf0f987ae77ec32668440158f0e", size = 169364, upload-time = "2026-05-22T14:48:02.791Z" }, - { url = "https://files.pythonhosted.org/packages/aa/0b/76b601ee309a8bd556af0eecb184394c20b3c49aa9c8e085aa1ffacc2568/wrapt-2.2.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:727ab4244622cd6ad2390f322642090c877d2e83a608d2653a7643ae5368d926", size = 171079, upload-time = "2026-05-22T14:48:04.22Z" }, - { url = "https://files.pythonhosted.org/packages/cd/87/ee3f32d5658e3e26d3e0e457922b47a36dd3bfbdfee7f97bb3e802344a66/wrapt-2.2.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:03df9ebed4c73ab93fa8c07e3d41d818dfca1852b15731a3de59457b27814624", size = 160205, upload-time = "2026-05-22T14:48:05.553Z" }, - { url = "https://files.pythonhosted.org/packages/b1/d0/ae2fd64277a67f5d7bffcf2d05eea1e476263fb2a072baf0b0129ab85984/wrapt-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0d9ff006f420b2ec8296aa56ade43ea7da3e997e85769f0aafc5e0661aacb710", size = 168922, upload-time = "2026-05-22T14:48:07.132Z" }, - { url = "https://files.pythonhosted.org/packages/b1/f3/2d541a060c5bbafb9400bca4917e4d78bfd1f239f404782c86831a8f6b29/wrapt-2.2.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:844c858fc3bb7eacc0ba8efa904935d16aac6a4470948ad1e7e55c9f5a2a665f", size = 158388, upload-time = "2026-05-22T14:48:08.629Z" }, - { url = "https://files.pythonhosted.org/packages/1d/68/8d92c8800c57e93cb116ae9e9d6cbafc34fade5ee9f9107b6f203fb4dc35/wrapt-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:87bacdaf225117a342a20d9c03438d701c02112f6e3f351ce9b7f32354f14797", size = 167682, upload-time = "2026-05-22T14:48:10.042Z" }, - { url = "https://files.pythonhosted.org/packages/30/72/83ea3790ea352439442349388e29ff07b76e0686265f9088bbb505d1608d/wrapt-2.2.1-cp312-cp312-win32.whl", hash = "sha256:2f8c90c8afde51969487be4e1343ae049b268854877d415c2510baf833775052", size = 77857, upload-time = "2026-05-22T14:48:11.782Z" }, - { url = "https://files.pythonhosted.org/packages/ef/cb/99450668dd3502d62a54a1c8aa56e44f34cb8c1261b381cfe2e7926c3b75/wrapt-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:6ce32763ac31ce94fe9aada947e479b1975012bff166da409b4b9e4e376cf7e5", size = 80825, upload-time = "2026-05-22T14:48:13.046Z" }, - { url = "https://files.pythonhosted.org/packages/e6/3a/87512881be64e743f9ee4c66f4cbe8e884974bef2a5989af71f999653ac7/wrapt-2.2.1-cp312-cp312-win_arm64.whl", hash = "sha256:8d1b4d0e0c2119587a31f5c029abd547e0c81d93b89d394566fe1588659eb579", size = 79087, upload-time = "2026-05-22T14:48:14.323Z" }, - { url = "https://files.pythonhosted.org/packages/88/d1/a1b08f8f4fac8cbb156fa51cf64ee2c7f7f74f9875ba3cf70b3c58368694/wrapt-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d2beb1c7cab10603aecdc42f8edd6ff013f9a32e4543474e38e6b77ce9975aeb", size = 80831, upload-time = "2026-05-22T14:48:15.598Z" }, - { url = "https://files.pythonhosted.org/packages/54/ce/57890814991446a845e09b3445ce8b694f27eb0577004f2c2a36a9772ed4/wrapt-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e0cb7e4dd71f4c32e5e84843cd3c4cd65dda034314004bbe1d7f99af2426ab80", size = 81375, upload-time = "2026-05-22T14:48:17.071Z" }, - { url = "https://files.pythonhosted.org/packages/38/65/08d7a6c76ac4493bdb668205ee9c1de1bd5daca61717c3e9aa49b4c01499/wrapt-2.2.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:95821352042722cd9f1108874579a47989d0a7e12a37d87d2fc4af20fd99ab8a", size = 167417, upload-time = "2026-05-22T14:48:18.303Z" }, - { url = "https://files.pythonhosted.org/packages/62/ce/f1ccbee7a1bfe5cdc6b3da6bab4b45713d628b9294da32a39f563d648140/wrapt-2.2.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:abd621552ede77c4c69be7fac44ba911225b0c812b6ba604e5964cf98085b474", size = 166948, upload-time = "2026-05-22T14:48:19.768Z" }, - { url = "https://files.pythonhosted.org/packages/86/2a/f85d48d1cd4869aee6704028d257d740a47c1c467b457ce396b4b5b55d07/wrapt-2.2.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e3677c7146ce694874941ba82b57092cc4875445aadf29d72807351023105143", size = 158148, upload-time = "2026-05-22T14:48:21.96Z" }, - { url = "https://files.pythonhosted.org/packages/fe/5c/93939ad11d4a12358ab1aab219a2ef5efa5612e0db6b9fc65af8af1a891b/wrapt-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9a5934eaea872e17936b5f45501eba5ab0bce9a74122e172b663d7c28c459c4a", size = 165905, upload-time = "2026-05-22T14:48:23.373Z" }, - { url = "https://files.pythonhosted.org/packages/e0/22/b8c2aa89862ff58605934d7abf4b70e6a5a1c33df96656f49035ccdf1c8a/wrapt-2.2.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:f5b9daf6b629fce418e0cc3dd0436eac045188fa35deadb7a7f3941d5b8203f9", size = 156712, upload-time = "2026-05-22T14:48:24.767Z" }, - { url = "https://files.pythonhosted.org/packages/5d/78/bf00a7b02239c12bb02ddcc3c0b971bfcc36e578c5a44f1ccfef5b458545/wrapt-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f53ac9f3ef573326d009ed809beff4efcac6451931c2b8132586da4b9e53ff31", size = 166560, upload-time = "2026-05-22T14:48:26.83Z" }, - { url = "https://files.pythonhosted.org/packages/fe/93/6390ca9c5b787683cef588d04f57c8d41b9a2323b5597a65f18638c90ef2/wrapt-2.2.1-cp313-cp313-win32.whl", hash = "sha256:1ffa9cfd4bdb581539951b14ae661ff20ed0c3599b3e911a131ee0ec5ac11337", size = 77817, upload-time = "2026-05-22T14:48:28.221Z" }, - { url = "https://files.pythonhosted.org/packages/97/73/ce10f0e71c0cfaa1a65faadb8efd4852028b3bb9ba28932b8889df769d38/wrapt-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:368eac1e20fd0bb03dd3cc42bf9887154c3861b60989389ccb5fac032617d215", size = 80736, upload-time = "2026-05-22T14:48:30.139Z" }, - { url = "https://files.pythonhosted.org/packages/c7/4c/89f4a6818fafbbd840330e4fa3873073e1bfc166133a64cac7f8fde7a5e3/wrapt-2.2.1-cp313-cp313-win_arm64.whl", hash = "sha256:c754dafdf5aaf0b401b644a90a30046929a0dd1a536e0ff0ec959a59155d9c7f", size = 79099, upload-time = "2026-05-22T14:48:31.405Z" }, - { url = "https://files.pythonhosted.org/packages/bf/f2/9a8741c46f8c208ac0a45b25ba170bcb4fb72a2781d5fb97dbd7b6be73cb/wrapt-2.2.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:ed928d0fda15fc0adc8d13305c8b3c0f2fba5b0669950c9e6d019d9162a3b3e8", size = 82802, upload-time = "2026-05-22T14:48:33.307Z" }, - { url = "https://files.pythonhosted.org/packages/9c/0d/e9c855716a3705eef1416456bdf062b60620726fdc59428ff670fc3c60dc/wrapt-2.2.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fafb4e739e43544d12cb4abd1605fd4683b6ca6a9ad682b7fd8f4d21973eafa8", size = 83329, upload-time = "2026-05-22T14:48:34.593Z" }, - { url = "https://files.pythonhosted.org/packages/3b/d6/a88f1c13112b7831adac75cea65d8310e0d696d570c8961844c90a57b865/wrapt-2.2.1-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:74d6a0c31472fe5d814917266b9f46495d7c61ed890af08b468acea92fb89a8d", size = 202937, upload-time = "2026-05-22T14:48:35.859Z" }, - { url = "https://files.pythonhosted.org/packages/42/65/e29d54aef06a4d898a5b8a25589a0b3769bde454f922fad8f6f89fbfb650/wrapt-2.2.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab5be648d5a0b86b7438864f8df3c705a65cef35a2fd3e5561e3e203167e0f27", size = 209997, upload-time = "2026-05-22T14:48:38.153Z" }, - { url = "https://files.pythonhosted.org/packages/2a/91/e4454263516cf0e12640912fbca9a83654e424f0a6ddb79f5cd7ce14bf33/wrapt-2.2.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9d8f204c8e3a8bf9ece17e0a83d137fd807440977f8a5e762d59306795011440", size = 194856, upload-time = "2026-05-22T14:48:39.69Z" }, - { url = "https://files.pythonhosted.org/packages/de/d0/fe0ee202286afdf4a7f77dd29f195703145764d572aec209c5086e57d924/wrapt-2.2.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d047f6498c973874ba08ac3f97c69a2c4b2211c8de6f4c205f75cb1c9522596e", size = 205654, upload-time = "2026-05-22T14:48:43.456Z" }, - { url = "https://files.pythonhosted.org/packages/23/b6/87d860dfc6460c246af70b1fd5c8b76df77571b42a493459423ded94fd7d/wrapt-2.2.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:7a4fdb9326aab4a5a477a1640e5ad786a8495901009d7e7b038371edd23a9d2b", size = 192206, upload-time = "2026-05-22T14:48:44.858Z" }, - { url = "https://files.pythonhosted.org/packages/df/46/3eea8cde077d985f239a38c0257087b8064fd9ee9b1a99e282d2c86da4ef/wrapt-2.2.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c8cc5094b08abeae52da9c73c8a32003623be691a5193df2f4e3eac3d557c394", size = 198428, upload-time = "2026-05-22T14:48:46.319Z" }, - { url = "https://files.pythonhosted.org/packages/18/dc/b927ee9c7fc67adc3a5658f246a0d275425eb840ba36e7b702e70f18bde8/wrapt-2.2.1-cp313-cp313t-win32.whl", hash = "sha256:9907a4402ab6db12b7077a0ea5d7a4d028ecb22c8eee2b53527080d347cd1562", size = 79448, upload-time = "2026-05-22T14:48:47.901Z" }, - { url = "https://files.pythonhosted.org/packages/ec/b3/fd30b473fe498c70e6b9a5f328b8d3fbaf1b8c3c481465f59724bba8eb70/wrapt-2.2.1-cp313-cp313t-win_amd64.whl", hash = "sha256:5590d63f5243251641cf543009b4c9314a79d0598fdb8a8e4cfc918494536c53", size = 83021, upload-time = "2026-05-22T14:48:49.201Z" }, - { url = "https://files.pythonhosted.org/packages/ee/f3/96c39153a8737a6e9aa85adef254ac4195bea3f2d24efc60472ccc3c9e2e/wrapt-2.2.1-cp313-cp313t-win_arm64.whl", hash = "sha256:c318a64b53d97b841d7b5e637517e50a27be64bc695128422953d4b21710954e", size = 80295, upload-time = "2026-05-22T14:48:50.479Z" }, - { url = "https://files.pythonhosted.org/packages/0a/a3/11d7f34ebbf3231bc907a3e6d5ee051b14d034c1bc7b65a97d5cc00516df/wrapt-2.2.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:6f56a647e4eaf5f0ca40330fb070f566bdf9f7b0db89a1af20d71c28dcd7a0ab", size = 80879, upload-time = "2026-05-22T14:48:51.802Z" }, - { url = "https://files.pythonhosted.org/packages/13/3c/b74cfd984cef560b900fb1a727af20352d89e1f06bf2e1114dd3f00f5f5a/wrapt-2.2.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:64b7deeda4b70408e382328d8bbe52a256fe9bc63ae3db86d804608367e5422c", size = 81462, upload-time = "2026-05-22T14:48:53.18Z" }, - { url = "https://files.pythonhosted.org/packages/15/a3/7c8f704b8dc07dfe0a5d01c2edbfd88317aa8e5e3fa7c743eb7a085ae767/wrapt-2.2.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b9cf53ba90717db2e292401de290776c498d4bbfb0d4a559ca2895db8b9dcb5c", size = 167251, upload-time = "2026-05-22T14:48:54.562Z" }, - { url = "https://files.pythonhosted.org/packages/80/85/a34d1888d97247da6c2ff6118c3a721c73ed8cc4dd198c00208bb73b6f80/wrapt-2.2.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cf3638274ab9d9b724c9baa0b4c04e132cd6faefb78b4dd3dd1a02a4bdaad41e", size = 166316, upload-time = "2026-05-22T14:48:56.065Z" }, - { url = "https://files.pythonhosted.org/packages/e9/d7/72ffaeb01eebc704afe3fb99e840480f4bda45f0fa66e3381b6a39251c8f/wrapt-2.2.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aed9658797d0b45d6c49adcfc6b41f66e6f2d0c6de3ec79e16cf4b1855df240f", size = 157952, upload-time = "2026-05-22T14:48:57.924Z" }, - { url = "https://files.pythonhosted.org/packages/24/5b/36f5d6b024e4edfdd90b140742d11ebcf7836daf5c9daf326c55c24db412/wrapt-2.2.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1d676ee388bc42a04d56dd7deb5605244dac2e35cc2fadbb43c9fa25bbd93508", size = 166130, upload-time = "2026-05-22T14:48:59.384Z" }, - { url = "https://files.pythonhosted.org/packages/81/06/9296d9e97bfdef5483dfcc859d57b095b257144b2bc5300ab521e06f4bc7/wrapt-2.2.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e395f7bc31851ef9b612050368cb446e9bc14cd7454b025018980349caf25ae5", size = 156604, upload-time = "2026-05-22T14:49:00.921Z" }, - { url = "https://files.pythonhosted.org/packages/53/37/16953929ed6776175720e58fc966e779926d8d71e2c7b2273230590ca71f/wrapt-2.2.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f1845c2a8cc1180ccccfa45785dd06f562730d19ef75be180334254012b6283", size = 166007, upload-time = "2026-05-22T14:49:02.332Z" }, - { url = "https://files.pythonhosted.org/packages/b9/73/20ee58c0612dae7c31131a7095345812ed2c7b389019e175f68cde34e5b4/wrapt-2.2.1-cp314-cp314-win32.whl", hash = "sha256:436addbc4bb4fc0a88c702577f51195d7d73683a7f3e0e5b253d8404d7847243", size = 78327, upload-time = "2026-05-22T14:49:03.722Z" }, - { url = "https://files.pythonhosted.org/packages/22/b3/ef7c3295d02e0448a71c639a36a057f46d524d057c9486291a7a3039e65c/wrapt-2.2.1-cp314-cp314-win_amd64.whl", hash = "sha256:50972a1d974ea07725a7f6b1cec5f8759008afd030a0024843ebe7d52de47f2b", size = 81144, upload-time = "2026-05-22T14:49:05.093Z" }, - { url = "https://files.pythonhosted.org/packages/ac/dc/7bdf336953f99f4ceb0a584bb8870e42c8f26f93ea10c87834dad62f1668/wrapt-2.2.1-cp314-cp314-win_arm64.whl", hash = "sha256:1c9934ea5d92957e3cd0adbc0845539dccfd62710ebe16195a8c66c53954db36", size = 79569, upload-time = "2026-05-22T14:49:06.413Z" }, - { url = "https://files.pythonhosted.org/packages/6a/6d/6dfae80150ff1919c356d1dd528f049bcdfaae29b4d284bc957e022caef4/wrapt-2.2.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:17de18fc12cea55b8a9587314cb830573e37fb33b247a7515696350863714188", size = 82892, upload-time = "2026-05-22T14:49:07.925Z" }, - { url = "https://files.pythonhosted.org/packages/82/7b/4e34766a7d7804ffce9e71befe47e9b3225dc350c49c94493c4ab39fd3a5/wrapt-2.2.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a9dec1aca52dddde7df94818310fa2fe79739c8f385b2014c4cb1035f5508199", size = 83333, upload-time = "2026-05-22T14:49:09.257Z" }, - { url = "https://files.pythonhosted.org/packages/9d/57/0b34db3e8de44ccfece62d7b337abd1631dd810f5adc5f3db571727836b5/wrapt-2.2.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:69f2e9244542cb34dd59c7f073445b9e54ad9f3fce8d93606c368a1b499fc413", size = 202899, upload-time = "2026-05-22T14:49:10.572Z" }, - { url = "https://files.pythonhosted.org/packages/e5/45/ac0c459f154b99d92789a6cba7ca727185b83513b986f8ec7fe2aacddcbf/wrapt-2.2.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2d83966dc7f4f45e8b97b5933685ac2e6e67fc0e19246ea314bceb9a8970c956", size = 209986, upload-time = "2026-05-22T14:49:12.229Z" }, - { url = "https://files.pythonhosted.org/packages/b7/e4/77e37ff33ad018fa81ade52c25fa327b80b56f81d734279a63614fcb4cbc/wrapt-2.2.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:78b0aa6bfb7be8deed0ab23e7aa028cc5210c29bc2d32a04d52b50e517a7307e", size = 194893, upload-time = "2026-05-22T14:49:14.139Z" }, - { url = "https://files.pythonhosted.org/packages/dd/9d/7ea651d1ab032fc5fa222fbec91d0f8a1397f6ae04ebb93fa7219aa921d7/wrapt-2.2.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:05d5cb74d1b232ec8cfa130a8f900708699ff2491d97b8f85a4cdc5996294b85", size = 205636, upload-time = "2026-05-22T14:49:15.714Z" }, - { url = "https://files.pythonhosted.org/packages/09/af/8e88031a701275b9085c54e64bc88c0b1cd55c77eadd400691c371cd76c4/wrapt-2.2.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f6518b94edb9150452e9aba08027d4cc293433753ec1fbefb4629a21cbc74181", size = 192267, upload-time = "2026-05-22T14:49:17.283Z" }, - { url = "https://files.pythonhosted.org/packages/bf/a8/e657ca876b06710194f243d81c4b0896ade646e244bdbec2d87c8c56a8bd/wrapt-2.2.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ed55af48b3eb28f43228ca2306788892bcb629eb2b5c4876e2a3659872c2f17a", size = 198378, upload-time = "2026-05-22T14:49:18.785Z" }, - { url = "https://files.pythonhosted.org/packages/c8/59/822efe4ea722a3961331bfa35b7d90937790d2c20f0616de1997ccc3aebd/wrapt-2.2.1-cp314-cp314t-win32.whl", hash = "sha256:2e08688ab16525897da6589d56d0aebaf417bbe91c2d8e3b96203b1efa596e85", size = 80226, upload-time = "2026-05-22T14:49:20.264Z" }, - { url = "https://files.pythonhosted.org/packages/ab/31/2a7dc5f6abb2fca0b6e1610e120419f603650aceb4f1d3ac4cae0354e162/wrapt-2.2.1-cp314-cp314t-win_amd64.whl", hash = "sha256:fd0135d34387f5fd087d9be368ea77ea89cf2451dc1cd1c622d35021bcb3ab50", size = 83835, upload-time = "2026-05-22T14:49:21.634Z" }, - { url = "https://files.pythonhosted.org/packages/9f/c0/782b86e28d1ceebeb74cccea12d2cd3d2ba0bd68e3dec20b1bc5873f6127/wrapt-2.2.1-cp314-cp314t-win_arm64.whl", hash = "sha256:f70db64e8266d7c45d3b735f2e08eeb434b5e03da9a479ae42b2e2e486a21a00", size = 80722, upload-time = "2026-05-22T14:49:23.59Z" }, - { url = "https://files.pythonhosted.org/packages/53/46/29ac9daf11a86c22a8c38cd9236c62928ccae83f7ceb06bd3b0467cf9d05/wrapt-2.2.1-py3-none-any.whl", hash = "sha256:3aafea2975caef8ca49400640dde02cc7426e798f24870ed01f490bc3cffd32f", size = 61000, upload-time = "2026-05-22T14:49:41.593Z" }, -] - [[package]] name = "xxhash" version = "3.7.0"