From 7856cb2ed6fe0eb597c3db81b7759d1d394b60d6 Mon Sep 17 00:00:00 2001 From: Matej Kosec Date: Mon, 6 Apr 2026 20:40:30 +0000 Subject: [PATCH] fix(vllm): preserve user-specified --runner, don't clobber with 'generate' default update_engine_config_with_dynamo placed "runner": "generate" in the `defaults` dict and then unconditionally applied every entry via setattr, silently overriding any value the user passed on the CLI (e.g. --runner embed required for embedding models). Move the runner default out of the blanket loop and guard it with an explicit None check: only set runner="generate" when the field is None, meaning the user did not specify --runner at all. Also adds TestRunnerDefaultNotOverridden unit tests that verify: - runner defaults to "generate" when unset - runner="embed" is preserved (embedding model use case) - runner="pooling" is preserved Fixes #7670 Signed-off-by: Matej Kosec Signed-off-by: Matej Kosec --- .github/scripts/parse_buildkit_output.py | 8 +- .github/workflows/detect_broken_links.py | 30 +- .../upload_complete_workflow_metrics.py | 18 +- .../scripts/analysis/create_report.py | 10 +- .../analysis/frontend_perf_analysis.py | 24 +- benchmarks/llm/plot_pareto.py | 4 +- .../multimodal/jsonl/generate_images.py | 6 +- benchmarks/nat_trace/convert.py | 8 +- .../prefix_data_generator/synthesizer.py | 12 +- .../tests/test_sampler.py | 6 +- .../tests/test_synthesizer.py | 24 +- .../dynamo/common/configuration/arg_group.py | 1 + .../multimodal_embedding_cache_manager.py | 6 +- .../common/multimodal/embedding_transfer.py | 14 +- .../dynamo/common/protocols/audio_protocol.py | 6 +- components/src/dynamo/common/storage.py | 1 + .../common/tests/configuration/test_utils.py | 1 + .../multimodal/test_embedding_transfer.py | 96 +++--- .../src/dynamo/common/tests/test_storage.py | 14 +- .../src/dynamo/common/utils/media_nixl.py | 6 +- .../src/dynamo/common/utils/nvtx_utils.py | 1 + .../src/dynamo/common/utils/otel_tracing.py | 1 - .../src/dynamo/common/utils/prometheus.py | 6 +- components/src/dynamo/frontend/main.py | 6 +- .../tests/test_sglang_processor_unit.py | 7 +- .../tests/test_vllm_processor_unit.py | 30 +- .../src/dynamo/frontend/vllm_processor.py | 6 +- .../tests/unit/test_scale_request_handler.py | 22 +- .../dynamo/planner/config/planner_config.py | 12 +- components/src/dynamo/planner/core/base.py | 6 +- .../planner/core/throughput/interpolation.py | 6 +- .../core/throughput/pre_swept_results.py | 14 +- .../dynamo/planner/monitoring/worker_info.py | 3 +- .../src/dynamo/planner/offline/dryrun.py | 6 +- .../tests/manual/scaling/scaling_e2e.py | 6 +- .../planner/tests/unit/load_generator.py | 9 +- .../tests/unit/test_load_predictors.py | 12 +- .../tests/unit/test_replica_calculation.py | 54 +-- .../integration/test_profile_sla_dgdr.py | 24 +- .../tests/unit/test_helpers_profile_sla.py | 30 +- .../unit/test_profile_sla_auto_backend.py | 30 +- .../utils/config_modifiers/protocol.py | 36 +- .../profiler/utils/config_modifiers/trtllm.py | 18 +- .../utils/replay_optimize/evaluate.py | 2 +- .../profiler/utils/replay_optimize/search.py | 2 +- .../profiler/utils/search_space_autogen.py | 6 +- components/src/dynamo/profiler/webui/utils.py | 18 +- components/src/dynamo/sglang/protocol.py | 6 +- .../multimodal/encode_worker_handler.py | 12 +- .../multimodal/worker_handler.py | 4 +- .../video_generation_handler.py | 6 +- components/src/dynamo/sglang/shutdown.py | 4 +- components/src/dynamo/sglang/snapshot.py | 1 - .../src/dynamo/trtllm/multimodal_processor.py | 9 +- components/src/dynamo/trtllm/publisher.py | 6 +- .../trtllm/request_handlers/handler_base.py | 12 +- .../video_diffusion/video_handler.py | 6 +- .../tests/test_trtllm_additional_metrics.py | 7 +- .../trtllm/tests/test_trtllm_autodeploy.py | 8 +- .../trtllm/tests/test_trtllm_handler_base.py | 6 +- .../dynamo/trtllm/tests/test_trtllm_unit.py | 14 +- .../tests/test_trtllm_video_diffusion.py | 55 +-- .../src/dynamo/trtllm/workers/llm_worker.py | 6 +- components/src/dynamo/vllm/args.py | 22 +- components/src/dynamo/vllm/backend_args.py | 5 +- components/src/dynamo/vllm/handlers.py | 8 +- .../src/dynamo/vllm/instrumented_scheduler.py | 2 +- .../encode_worker_handler.py | 33 +- .../vllm/multimodal_utils/chat_processor.py | 6 +- .../src/dynamo/vllm/multimodal_utils/model.py | 4 +- .../src/dynamo/vllm/omni/audio_handler.py | 3 +- components/src/dynamo/vllm/publisher.py | 6 +- .../tests/multimodal_utils/test_vllm_model.py | 1 - .../dynamo/vllm/tests/test_backend_args.py | 1 - .../vllm/tests/test_vllm_kv_events_api.py | 6 +- .../vllm/tests/test_vllm_renderer_api.py | 6 +- .../src/dynamo/vllm/tests/test_vllm_unit.py | 92 +++++ .../vllm/tests/test_vllm_worker_handler.py | 4 +- components/src/dynamo/vllm/worker_factory.py | 5 +- .../api/scripts/generate_pydantic_from_go.py | 2 +- deploy/operator/docs/fix-api-anchors.py | 1 + deploy/sanity_check.py | 2 +- deploy/utils/dynamo_deployment.py | 16 +- .../sglang/test_sglang_expert_info.py | 12 +- .../backends/sglang/test_sglang_profile.py | 4 +- examples/multimodal/components/worker.py | 6 +- examples/multimodal/utils/args.py | 6 +- examples/multimodal/utils/chat_processor.py | 14 +- fern/convert_callouts.py | 4 +- lib/bindings/kvbm/python/kvbm/_core.pyi | 25 +- .../connector/kvbm_connector_worker.py | 12 +- .../kvbm/vllm_integration/connector_leader.py | 12 +- lib/bindings/python/src/dynamo/_core.pyi | 207 +++++------ .../src/dynamo/nixl_connect/__init__.py | 12 +- .../tests/cancellation/test_cancellation.py | 18 +- .../python/tests/cancellation/test_example.py | 30 +- .../python/tests/test_example_hello_world.py | 2 +- lib/bindings/python/tests/test_kv_bindings.py | 60 ++-- .../integrations/sglang/memory_saver.py | 6 +- .../integrations/vllm/model_runner.py | 12 +- .../integrations/vllm/worker.py | 6 +- tests/basic/test_autodeploy_backend.py | 6 +- tests/basic/test_wheel_contents.py | 7 +- tests/dependencies/test_kvbm_imports.py | 6 +- tests/deploy/test_deploy.py | 42 +-- .../fault_tolerance/cancellation/test_vllm.py | 6 +- tests/fault_tolerance/deploy/parse_factory.py | 3 +- tests/fault_tolerance/etcd_ha/test_sglang.py | 36 +- tests/fault_tolerance/etcd_ha/test_trtllm.py | 36 +- tests/fault_tolerance/etcd_ha/test_vllm.py | 36 +- tests/fault_tolerance/etcd_ha/utils.py | 6 +- .../api_service/main.py | 12 +- .../helpers/cuda_fault_injection.py | 6 +- tests/fault_tolerance/migration/test_vllm.py | 6 +- tests/fault_tolerance/migration/utils.py | 12 +- tests/frontend/grpc/test_tensor_parameters.py | 6 +- tests/frontend/grpc/triton_echo_client.py | 30 +- .../frontend/test_completion_mocker_engine.py | 15 +- tests/frontend/test_prepost.py | 24 +- tests/frontend/test_prepost_mistral.py | 12 +- tests/frontend/test_prompt_embeds.py | 14 +- tests/frontend/test_vllm.py | 16 +- .../integration/test_external_weight_mgr.py | 18 +- .../integration/test_gms_shadow_failover.py | 17 +- tests/kvbm_integration/common.py | 60 ++-- .../kvbm_integration/test_chunked_prefill.py | 6 +- .../test_consolidator_router_e2e.py | 108 +++--- tests/kvbm_integration/test_cuda_graph.py | 12 +- .../kvbm_integration/test_determinism_agg.py | 6 +- tests/kvbm_integration/test_kvbm.py | 30 +- .../test_kvbm_vllm_integration.py | 2 +- tests/mm_router/test_mm_router_e2e.py | 12 +- tests/mm_router/test_vllm_mm_router_e2e.py | 12 +- tests/router/common.py | 320 ++++++++++-------- tests/router/helper.py | 30 +- tests/router/test_router_e2e_with_mockers.py | 9 +- tests/serve/common.py | 6 +- tests/serve/test_sglang.py | 6 +- tests/serve/test_trtllm.py | 12 +- tests/serve/test_vllm.py | 6 +- tests/serve/test_vllm_omni.py | 24 +- tests/serve/test_vllm_xpu.py | 6 +- tests/utils/client.py | 8 +- tests/utils/engine_process.py | 6 +- tests/utils/managed_deployment.py | 16 +- tests/utils/payloads.py | 258 +++++++------- tests/utils/profile_pytest.py | 3 +- tests/utils/pytest_parallel_gpu.py | 14 +- tests/utils/test_managed_process_teardown.py | 42 +-- 149 files changed, 1504 insertions(+), 1385 deletions(-) diff --git a/.github/scripts/parse_buildkit_output.py b/.github/scripts/parse_buildkit_output.py index 7560209185f9..717b091b5a7a 100755 --- a/.github/scripts/parse_buildkit_output.py +++ b/.github/scripts/parse_buildkit_output.py @@ -48,13 +48,13 @@ def parse_sccache_json_from_log( cleaned_json = "\n".join(cleaned_lines) if debug: - print(f"DEBUG: Parsing JSON block {i+1}", file=sys.stderr) + print(f"DEBUG: Parsing JSON block {i + 1}", file=sys.stderr) # Parse JSON section_data = json.loads(cleaned_json) # Extract key metrics into flat structure - section_name = section_data.get("section", f"section_{i+1}") + section_name = section_data.get("section", f"section_{i + 1}") timestamp = section_data.get("timestamp", "") native_stats = section_data.get("sccache_stats", {}) @@ -129,10 +129,10 @@ def parse_sccache_json_from_log( except json.JSONDecodeError as e: if debug: - print(f"DEBUG: JSON parse error in block {i+1}: {e}", file=sys.stderr) + print(f"DEBUG: JSON parse error in block {i + 1}: {e}", file=sys.stderr) except Exception as e: if debug: - print(f"DEBUG: Error parsing block {i+1}: {e}", file=sys.stderr) + print(f"DEBUG: Error parsing block {i + 1}: {e}", file=sys.stderr) return sccache_sections diff --git a/.github/workflows/detect_broken_links.py b/.github/workflows/detect_broken_links.py index 9126b4fe7908..a1a5048e5b6a 100755 --- a/.github/workflows/detect_broken_links.py +++ b/.github/workflows/detect_broken_links.py @@ -705,9 +705,9 @@ def detect_problematic_symlinks( # Check if symlink is broken (target doesn't exist) if not symlink_path.exists(): - symlink_info[ - "issue" - ] = f"Broken symlink: target '{target_path}' does not exist" + symlink_info["issue"] = ( + f"Broken symlink: target '{target_path}' does not exist" + ) problematic_symlinks["broken"].append(symlink_info) logger.warning(f"Broken symlink found: {symlink} -> {target_path}") continue @@ -723,9 +723,9 @@ def detect_problematic_symlinks( continue except (OSError, RuntimeError) as e: if "Too many levels of symbolic links" in str(e): - symlink_info[ - "issue" - ] = "Circular symlink: too many levels of symbolic links" + symlink_info["issue"] = ( + "Circular symlink: too many levels of symbolic links" + ) problematic_symlinks["circular"].append(symlink_info) logger.warning(f"Circular symlink found: {symlink}") continue @@ -735,9 +735,9 @@ def detect_problematic_symlinks( try: symlink_path.relative_to(git_root_path) except ValueError: - symlink_info[ - "issue" - ] = f"External symlink: points outside repository to '{symlink_path}'" + symlink_info["issue"] = ( + f"External symlink: points outside repository to '{symlink_path}'" + ) problematic_symlinks["external"].append(symlink_info) logger.warning( f"External symlink found: {symlink} -> {symlink_path}" @@ -746,17 +746,17 @@ def detect_problematic_symlinks( # Check for suspicious patterns (e.g., very long paths, unusual targets) if len(str(target_path)) > 200: - symlink_info[ - "issue" - ] = f"Suspicious symlink: unusually long target path ({len(str(target_path))} characters)" + symlink_info["issue"] = ( + f"Suspicious symlink: unusually long target path ({len(str(target_path))} characters)" + ) problematic_symlinks["suspicious"].append(symlink_info) logger.info(f"Suspicious symlink found: {symlink} (long path)") # Check if target is in a different directory tree (potential maintenance issue) if "../" in str(target_path) and str(target_path).count("../") > 3: - symlink_info[ - "issue" - ] = f"Suspicious symlink: target requires many directory traversals ('{target_path}')" + symlink_info["issue"] = ( + f"Suspicious symlink: target requires many directory traversals ('{target_path}')" + ) problematic_symlinks["suspicious"].append(symlink_info) logger.info(f"Suspicious symlink found: {symlink} (many traversals)") diff --git a/.github/workflows/upload_complete_workflow_metrics.py b/.github/workflows/upload_complete_workflow_metrics.py index c6cc4ef138d9..518e53d1b318 100644 --- a/.github/workflows/upload_complete_workflow_metrics.py +++ b/.github/workflows/upload_complete_workflow_metrics.py @@ -757,9 +757,9 @@ def _upload_container_metrics( # Identity & Context - container-specific fields only job_id = str(job_data["id"]) job_name = job_data["name"] - container_data[ - FIELD_ID - ] = f"github-container-{job_id}-{build_metrics.get('framework', 'unknown')}" + container_data[FIELD_ID] = ( + f"github-container-{job_id}-{build_metrics.get('framework', 'unknown')}" + ) container_data[FIELD_JOB_NAME] = str(job_name) container_data[FIELD_JOB_ID] = job_id @@ -937,9 +937,9 @@ def _upload_test_metrics(self, job_data: Dict[str, Any]) -> None: if test_classname else test_name ) - test_data[ - FIELD_ID - ] = f"github-test-{job_id}-{hash(test_full_name) & 0x7FFFFFFF}" # Use hash for unique ID + test_data[FIELD_ID] = ( + f"github-test-{job_id}-{hash(test_full_name) & 0x7FFFFFFF}" # Use hash for unique ID + ) test_data[FIELD_STEP_ID] = test_step_id test_data[FIELD_JOB_ID] = job_id @@ -991,9 +991,9 @@ def _upload_test_metrics(self, job_data: Dict[str, Any]) -> None: ) test_data[FIELD_TEST_STATUS] = test_status - test_data[ - FIELD_STATUS - ] = test_status # Also set general status field + test_data[FIELD_STATUS] = ( + test_status # Also set general status field + ) if error_msg: test_data[FIELD_ERROR_MESSAGE] = error_msg[ diff --git a/benchmarks/frontend/scripts/analysis/create_report.py b/benchmarks/frontend/scripts/analysis/create_report.py index 3508fde422c7..c271dc484d1e 100755 --- a/benchmarks/frontend/scripts/analysis/create_report.py +++ b/benchmarks/frontend/scripts/analysis/create_report.py @@ -200,13 +200,13 @@ def section_transport_breakdown(prom: Optional[PrometheusSnapshot]) -> Optional[ if prom.work_handler_network_transit: t = prom.work_handler_network_transit parts_rows.append( - f"| Part 1 - Network transit (T2-T1) | {t['p50']*1000:.2f} | {t['p95']*1000:.2f} | {t['p99']*1000:.2f} |" + f"| Part 1 - Network transit (T2-T1) | {t['p50'] * 1000:.2f} | {t['p95'] * 1000:.2f} | {t['p99'] * 1000:.2f} |" ) if prom.work_handler_time_to_first_response: t = prom.work_handler_time_to_first_response parts_rows.append( - f"| Part 2 - Processing (T3-T2) | {t['p50']*1000:.2f} | {t['p95']*1000:.2f} | {t['p99']*1000:.2f} |" + f"| Part 2 - Processing (T3-T2) | {t['p50'] * 1000:.2f} | {t['p95'] * 1000:.2f} | {t['p99'] * 1000:.2f} |" ) if parts_rows: @@ -581,7 +581,7 @@ def section_system_resources(obs_dir: Path) -> Optional[str]: if thread_data: values = [v for _, v in thread_data] rows.append( - f"| Threads | {min(values):.0f} | {max(values):.0f} | {sum(values)/len(values):.0f} | {len(values)} |" + f"| Threads | {min(values):.0f} | {max(values):.0f} | {sum(values) / len(values):.0f} | {len(values)} |" ) # FD count @@ -589,7 +589,7 @@ def section_system_resources(obs_dir: Path) -> Optional[str]: if fd_data: values = [v for _, v in fd_data] rows.append( - f"| FDs | {min(values):.0f} | {max(values):.0f} | {sum(values)/len(values):.0f} | {len(values)} |" + f"| FDs | {min(values):.0f} | {max(values):.0f} | {sum(values) / len(values):.0f} | {len(values)} |" ) if not rows: @@ -639,7 +639,7 @@ def section_key_findings( # Check transport breakdown if prom.request_plane_roundtrip_ttft_p50 > 0.1: findings.append( - f"High roundtrip TTFT p50: {prom.request_plane_roundtrip_ttft_p50*1000:.0f}ms (> 100ms)" + f"High roundtrip TTFT p50: {prom.request_plane_roundtrip_ttft_p50 * 1000:.0f}ms (> 100ms)" ) # Tokio health diff --git a/benchmarks/frontend/scripts/analysis/frontend_perf_analysis.py b/benchmarks/frontend/scripts/analysis/frontend_perf_analysis.py index bd281838f77c..243096ce90c7 100755 --- a/benchmarks/frontend/scripts/analysis/frontend_perf_analysis.py +++ b/benchmarks/frontend/scripts/analysis/frontend_perf_analysis.py @@ -200,9 +200,9 @@ def print_scalability_table(points: list[TestPoint]) -> None: concurrencies = sorted(set(p.concurrency for p in points)) for isl in isls: - print(f"\n{'='*80}") + print(f"\n{'=' * 80}") print(f"ISL = {isl}") - print(f"{'='*80}") + print(f"{'=' * 80}") print( f"{'Conc':>6} {'TTFT p50':>10} {'TTFT p95':>10} {'ITL p50':>10} " f"{'ITL p95':>10} {'Tput tok/s':>12} {'RPS':>8}" @@ -225,9 +225,9 @@ def print_scalability_table(points: list[TestPoint]) -> None: def print_stage_waterfall(points: list[TestPoint]) -> None: """Print stage breakdown at each load level.""" - print(f"\n{'='*80}") + print(f"\n{'=' * 80}") print("Pipeline Stage Breakdown (p50 seconds)") - print(f"{'='*80}") + print(f"{'=' * 80}") stages = ["preprocess", "route", "transport_roundtrip", "postprocess"] header = f"{'Key':>20}" @@ -248,9 +248,9 @@ def print_stage_waterfall(points: list[TestPoint]) -> None: def print_transport_breakdown(points: list[TestPoint]) -> None: """Print transport overhead: queue_seconds vs roundtrip_ttft_seconds.""" - print(f"\n{'='*80}") + print(f"\n{'=' * 80}") print("Transport Overhead (p50 seconds)") - print(f"{'='*80}") + print(f"{'=' * 80}") print(f"{'Key':>20} {'Queue (encode)':>16} {'RT TTFT (net)':>16} {'Inflight':>10}") print("-" * 70) @@ -266,9 +266,9 @@ def print_transport_breakdown(points: list[TestPoint]) -> None: def print_tokio_health(points: list[TestPoint]) -> None: """Print tokio health indicators.""" - print(f"\n{'='*80}") + print(f"\n{'=' * 80}") print("Tokio Health") - print(f"{'='*80}") + print(f"{'=' * 80}") print( f"{'Key':>20} {'Avg Poll ns':>12} {'Max Poll ns':>12} " f"{'Stalls':>8} {'Queue':>8} {'Yields':>8} {'Busy Avg':>10}" @@ -329,12 +329,12 @@ def compare_runs( print("ERROR: No matching test points between baseline and candidate.") return - print(f"\n{'='*100}") + print(f"\n{'=' * 100}") print( f"A/B Comparison: {baseline_dir.name} (baseline) vs {candidate_dir.name} (candidate)" ) print(f"Regression threshold: {threshold_pct}%") - print(f"{'='*100}") + print(f"{'=' * 100}") print( f"{'Key':>20} {'TTFT p50 B':>12} {'TTFT p50 C':>12} {'Delta%':>8} " f"{'ITL p50 B':>12} {'ITL p50 C':>12} {'Delta%':>8} {'Status':>12}" @@ -400,9 +400,9 @@ def print_heatmap(points: list[TestPoint]) -> None: point_map = {(p.concurrency, p.isl): p for p in points} - print(f"\n{'='*80}") + print(f"\n{'=' * 80}") print("TTFT p95 Heatmap (ms) — Concurrency x ISL") - print(f"{'='*80}") + print(f"{'=' * 80}") # Header header = f"{'Conc':>8}" diff --git a/benchmarks/llm/plot_pareto.py b/benchmarks/llm/plot_pareto.py index 5b82fa91a3df..0c71c80554a1 100755 --- a/benchmarks/llm/plot_pareto.py +++ b/benchmarks/llm/plot_pareto.py @@ -67,9 +67,7 @@ def parse_gpus(deployment_config_json_path): "prefill_tensor_parallelism" ) * deployment_config.get("prefill_data_parallelism") + deployment_config.get( "decode_tensor_parallelism" - ) * deployment_config.get( - "decode_data_parallelism" - ) + ) * deployment_config.get("decode_data_parallelism") def parse_kind_and_mode(deployment_config_json_path): diff --git a/benchmarks/multimodal/jsonl/generate_images.py b/benchmarks/multimodal/jsonl/generate_images.py index 7cbc0541e2f4..6fbd9d20ca5e 100644 --- a/benchmarks/multimodal/jsonl/generate_images.py +++ b/benchmarks/multimodal/jsonl/generate_images.py @@ -65,9 +65,9 @@ def sample_slots( """ pool_size = len(pool) total_slots = num_requests * images_per_request - assert ( - pool_size >= images_per_request - ), f"images-pool ({pool_size}) must be >= images-per-request ({images_per_request})" + assert pool_size >= images_per_request, ( + f"images-pool ({pool_size}) must be >= images-per-request ({images_per_request})" + ) assert total_slots >= pool_size, ( f"total slots ({num_requests}×{images_per_request}={total_slots}) < " f"images-pool ({pool_size}). Increase --num-requests or --images-per-request, " diff --git a/benchmarks/nat_trace/convert.py b/benchmarks/nat_trace/convert.py index 14726dc0b869..8b3207afe42b 100644 --- a/benchmarks/nat_trace/convert.py +++ b/benchmarks/nat_trace/convert.py @@ -343,7 +343,7 @@ def print_statistics(mooncake_data: list): turns_per_session = [len(turns) for turns in sessions.values()] print( - f"Turns per session: min={min(turns_per_session)}, max={max(turns_per_session)}, avg={sum(turns_per_session)/len(turns_per_session):.1f}" + f"Turns per session: min={min(turns_per_session)}, max={max(turns_per_session)}, avg={sum(turns_per_session) / len(turns_per_session):.1f}" ) print(f"Total LLM calls: {len(mooncake_data)}") @@ -355,19 +355,19 @@ def print_statistics(mooncake_data: list): print("\nInput Length (prompt_tokens):") print(f" Min: {min(input_lengths)}") print(f" Max: {max(input_lengths)}") - print(f" Avg: {sum(input_lengths)/len(input_lengths):.1f}") + print(f" Avg: {sum(input_lengths) / len(input_lengths):.1f}") print("\nOutput Length (completion_tokens):") print(f" Min: {min(output_lengths)}") print(f" Max: {max(output_lengths)}") - print(f" Avg: {sum(output_lengths)/len(output_lengths):.1f}") + print(f" Avg: {sum(output_lengths) / len(output_lengths):.1f}") # Hash statistics hash_lengths = [len(e["hash_ids"]) for e in mooncake_data] print("\nHash IDs per entry:") print(f" Min: {min(hash_lengths)}") print(f" Max: {max(hash_lengths)}") - print(f" Avg: {sum(hash_lengths)/len(hash_lengths):.1f}") + print(f" Avg: {sum(hash_lengths) / len(hash_lengths):.1f}") print("=" * 60) diff --git a/benchmarks/prefix_data_generator/synthesizer.py b/benchmarks/prefix_data_generator/synthesizer.py index 2dd57e51b023..949831161b73 100644 --- a/benchmarks/prefix_data_generator/synthesizer.py +++ b/benchmarks/prefix_data_generator/synthesizer.py @@ -87,12 +87,12 @@ def __init__( self.osl_multiplier = float(osl_multiplier) # assert correct arg bounds - assert ( - isinstance(self.num_copies, int) and self.num_copies >= 1 - ), "num_copies must be an integer greater than or equal to 1" - assert ( - isinstance(self.speedup_ratio, float) and self.speedup_ratio > 0 - ), "speedup_ratio must be a positive float" + assert isinstance(self.num_copies, int) and self.num_copies >= 1, ( + "num_copies must be an integer greater than or equal to 1" + ) + assert isinstance(self.speedup_ratio, float) and self.speedup_ratio > 0, ( + "speedup_ratio must be a positive float" + ) assert ( isinstance(self.prefix_len_multiplier, float) and self.prefix_len_multiplier > 0 diff --git a/benchmarks/prefix_data_generator/tests/test_sampler.py b/benchmarks/prefix_data_generator/tests/test_sampler.py index 5455742c20aa..b4b0ac6d4189 100644 --- a/benchmarks/prefix_data_generator/tests/test_sampler.py +++ b/benchmarks/prefix_data_generator/tests/test_sampler.py @@ -34,9 +34,9 @@ def test_empirical_sampler_distribution(): # Verify each number (1, 2, 3) appears between 300 and 400 times for value in [1, 2, 3]: - assert ( - 300 <= counts[value] <= 400 - ), f"Value {value} appeared {counts[value]} times, expected 300-400 times" + assert 300 <= counts[value] <= 400, ( + f"Value {value} appeared {counts[value]} times, expected 300-400 times" + ) # Verify no other values appear in the samples assert set(counts.keys()) == { diff --git a/benchmarks/prefix_data_generator/tests/test_synthesizer.py b/benchmarks/prefix_data_generator/tests/test_synthesizer.py index 6717ad9761d7..9363ef738dcc 100644 --- a/benchmarks/prefix_data_generator/tests/test_synthesizer.py +++ b/benchmarks/prefix_data_generator/tests/test_synthesizer.py @@ -47,27 +47,27 @@ def check_attributes( ): # Check children actual_children = list(graph.successors(node)) - assert sorted(actual_children) == sorted( - expected_children - ), f"Node {node} has children {actual_children}, expected {expected_children}" + assert sorted(actual_children) == sorted(expected_children), ( + f"Node {node} has children {actual_children}, expected {expected_children}" + ) # Check 'visited' attribute if expected if expected_visited is not None: - assert ( - graph.nodes[node].get("visited") == expected_visited - ), f"Node {node} has 'visited' value {graph.nodes[node].get('visited')}, expected {expected_visited}" + assert graph.nodes[node].get("visited") == expected_visited, ( + f"Node {node} has 'visited' value {graph.nodes[node].get('visited')}, expected {expected_visited}" + ) # Check 'length' attribute if expected if expected_length is not None: - assert ( - graph.nodes[node].get("length") == expected_length - ), f"Node {node} has 'length' value {graph.nodes[node].get('length')}, expected {expected_length}" + assert graph.nodes[node].get("length") == expected_length, ( + f"Node {node} has 'length' value {graph.nodes[node].get('length')}, expected {expected_length}" + ) # Check 'to_leaf' attribute if expected if expected_to_leaf is not None: - assert ( - graph.nodes[node].get("to_leaf") == expected_to_leaf - ), f"Node {node} has 'to_leaf' value {graph.nodes[node].get('to_leaf')}, expected {expected_to_leaf}" + assert graph.nodes[node].get("to_leaf") == expected_to_leaf, ( + f"Node {node} has 'to_leaf' value {graph.nodes[node].get('to_leaf')}, expected {expected_to_leaf}" + ) return True diff --git a/components/src/dynamo/common/configuration/arg_group.py b/components/src/dynamo/common/configuration/arg_group.py index 3e984500fb9d..560d14c53a3a 100644 --- a/components/src/dynamo/common/configuration/arg_group.py +++ b/components/src/dynamo/common/configuration/arg_group.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """Base ArgGroup interface.""" + import argparse from abc import ABC, abstractmethod diff --git a/components/src/dynamo/common/memory/multimodal_embedding_cache_manager.py b/components/src/dynamo/common/memory/multimodal_embedding_cache_manager.py index 47d8ce22d739..1acedb5ddddf 100644 --- a/components/src/dynamo/common/memory/multimodal_embedding_cache_manager.py +++ b/components/src/dynamo/common/memory/multimodal_embedding_cache_manager.py @@ -77,9 +77,9 @@ def _tensor_size(tensor: torch.Tensor) -> int: Raises: AssertionError: If tensor is not contiguous. """ - assert ( - tensor.is_contiguous() - ), "Tensor must be contiguous for accurate size calculation" + assert tensor.is_contiguous(), ( + "Tensor must be contiguous for accurate size calculation" + ) return tensor.element_size() * tensor.numel() def get(self, key: str) -> Optional[CachedEmbedding]: diff --git a/components/src/dynamo/common/multimodal/embedding_transfer.py b/components/src/dynamo/common/multimodal/embedding_transfer.py index 21fa7f9c0565..e8b685df818e 100644 --- a/components/src/dynamo/common/multimodal/embedding_transfer.py +++ b/components/src/dynamo/common/multimodal/embedding_transfer.py @@ -438,9 +438,9 @@ async def _state_update(self): # mark the transfer as completed to unblock the sender. self._complete_transfer(tensor_id) continue - self.remote_agents[ - remote_agent_id - ] = self.nixl_agent.add_remote_agent(remote_agent_metadata) + self.remote_agents[remote_agent_id] = ( + self.nixl_agent.add_remote_agent(remote_agent_metadata) + ) # initiate NIXL WRITE transfer source_tensor, source_desc, _ = self.transfer_tracker[tensor_id] @@ -660,10 +660,10 @@ async def receive_embeddings( raise ValueError( f"Missing agent metadata for new sender {nixl_request.sender_agent_id}" ) - self.remote_agents[ - nixl_request.sender_agent_id - ] = self.nixl_agent.add_remote_agent( - base64.b64decode(nixl_request.agent_metadata) + self.remote_agents[nixl_request.sender_agent_id] = ( + self.nixl_agent.add_remote_agent( + base64.b64decode(nixl_request.agent_metadata) + ) ) # Allocate tensor to be written into. diff --git a/components/src/dynamo/common/protocols/audio_protocol.py b/components/src/dynamo/common/protocols/audio_protocol.py index d12f62cf2608..5cd5029adb14 100644 --- a/components/src/dynamo/common/protocols/audio_protocol.py +++ b/components/src/dynamo/common/protocols/audio_protocol.py @@ -33,9 +33,9 @@ class NvCreateAudioSpeechRequest(BaseModel): voice: Optional[str] = None """Voice/speaker name (e.g., 'vivian', 'ryan', 'aiden').""" - response_format: Optional[ - Literal["wav", "pcm", "flac", "mp3", "aac", "opus"] - ] = "wav" + response_format: Optional[Literal["wav", "pcm", "flac", "mp3", "aac", "opus"]] = ( + "wav" + ) """Output format.""" speed: Optional[float] = Field(default=1.0, ge=0.25, le=4.0) diff --git a/components/src/dynamo/common/storage.py b/components/src/dynamo/common/storage.py index 2cec28570606..c71b521abf3e 100644 --- a/components/src/dynamo/common/storage.py +++ b/components/src/dynamo/common/storage.py @@ -32,6 +32,7 @@ """ + import asyncio from typing import Optional diff --git a/components/src/dynamo/common/tests/configuration/test_utils.py b/components/src/dynamo/common/tests/configuration/test_utils.py index 63bc10c81e49..f65ca1a280d8 100644 --- a/components/src/dynamo/common/tests/configuration/test_utils.py +++ b/components/src/dynamo/common/tests/configuration/test_utils.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """Tests for configuration utility functions.""" + import argparse import pytest diff --git a/components/src/dynamo/common/tests/multimodal/test_embedding_transfer.py b/components/src/dynamo/common/tests/multimodal/test_embedding_transfer.py index 6a65e014dd2f..de89c29e8f15 100644 --- a/components/src/dynamo/common/tests/multimodal/test_embedding_transfer.py +++ b/components/src/dynamo/common/tests/multimodal/test_embedding_transfer.py @@ -157,17 +157,17 @@ def test_simple(self): id, tensor = ring_buffer.get_buffer(byte_size) assert id is not None, f"Failed to get buffer for size {byte_size}" assert tensor is not None, f"Failed to get tensor for size {byte_size}" - assert ( - tensor.nbytes == byte_size - ), f"Expected buffer of size {byte_size}, got {tensor.nbytes}" + assert tensor.nbytes == byte_size, ( + f"Expected buffer of size {byte_size}, got {tensor.nbytes}" + ) ring_buffer.release_buffer(id) # Test allocation that exceeds buffer size id, tensor = ring_buffer.get_buffer(buffer_size + 1) assert id is None, "Expected None when requesting buffer larger than capacity" - assert ( - tensor is None - ), "Expected None when requesting buffer larger than capacity" + assert tensor is None, ( + "Expected None when requesting buffer larger than capacity" + ) def test_release(self): buffer_size = 128 @@ -181,9 +181,9 @@ def test_release(self): id, tensor = ring_buffer.get_buffer(byte_size) assert id is not None, f"Failed to get buffer for size {byte_size}" assert tensor is not None, f"Failed to get tensor for size {byte_size}" - assert ( - tensor.nbytes == byte_size - ), f"Expected buffer of size {byte_size}, got {tensor.nbytes}" + assert tensor.nbytes == byte_size, ( + f"Expected buffer of size {byte_size}, got {tensor.nbytes}" + ) allocated_ids.append(id) # Release buffers except the first one, ring buffer will not actually reuse the released space @@ -194,12 +194,12 @@ def test_release(self): ring_buffer.release_buffer(id) failed_id, failed_tensor = ring_buffer.get_buffer(64) - assert ( - failed_id is None - ), "Expected None when requesting buffer larger than remaining capacity" - assert ( - failed_tensor is None - ), "Expected None when requesting buffer larger than remaining capacity" + assert failed_id is None, ( + "Expected None when requesting buffer larger than remaining capacity" + ) + assert failed_tensor is None, ( + "Expected None when requesting buffer larger than remaining capacity" + ) # Release the first allocated buffer to make sure the ring buffer can reuse the released space. ring_buffer.release_buffer(allocated_ids[0]) @@ -228,18 +228,18 @@ def test_wrap_around(self): and allocated_id2 is not None and allocated_id3 is not None ), "Failed to allocate initial buffers" - assert ( - tensor1.nbytes == 32 and tensor2.nbytes == 32 and tensor3.nbytes == 32 - ), "Expected buffers of size 32" + assert tensor1.nbytes == 32 and tensor2.nbytes == 32 and tensor3.nbytes == 32, ( + "Expected buffers of size 32" + ) # Out of space failed_allocation_id, failed_allocation_tensor = ring_buffer.get_buffer(64) - assert ( - failed_allocation_id is None - ), "Expected None when requesting buffer larger than remaining capacity" - assert ( - failed_allocation_tensor is None - ), "Expected None when requesting buffer larger than remaining capacity" + assert failed_allocation_id is None, ( + "Expected None when requesting buffer larger than remaining capacity" + ) + assert failed_allocation_tensor is None, ( + "Expected None when requesting buffer larger than remaining capacity" + ) # Release the first buffer to create free space at the beginning, # but the 64 bytes allocation will fail as we don't allocate @@ -251,9 +251,9 @@ def test_wrap_around(self): # | 32 |-32-|-32-|-16-| 16 | # | | id2| id3| id4| | allocated_id4, tensor4 = ring_buffer.get_buffer(16) - assert ( - allocated_id4 is not None - ), "Failed to allocate buffer after releasing space" + assert allocated_id4 is not None, ( + "Failed to allocate buffer after releasing space" + ) assert tensor4.nbytes == 16, f"Expected buffer of size 16, got {tensor4.nbytes}" # Make room for large allocation @@ -262,18 +262,18 @@ def test_wrap_around(self): # | id5| id3| id4| | ring_buffer.release_buffer(allocated_id2) allocated_id5, tensor5 = ring_buffer.get_buffer(64) - assert ( - allocated_id5 is not None - ), "Failed to allocate buffer after releasing space" + assert allocated_id5 is not None, ( + "Failed to allocate buffer after releasing space" + ) assert tensor5.nbytes == 64, f"Expected buffer of size 64, got {tensor5.nbytes}" failed_allocation_id, failed_allocation_tensor = ring_buffer.get_buffer(8) - assert ( - failed_allocation_id is None - ), "Expected None when requesting buffer larger than remaining capacity" - assert ( - failed_allocation_tensor is None - ), "Expected None when requesting buffer larger than remaining capacity" + assert failed_allocation_id is None, ( + "Expected None when requesting buffer larger than remaining capacity" + ) + assert failed_allocation_tensor is None, ( + "Expected None when requesting buffer larger than remaining capacity" + ) # Release all and make sure we have full capacity again ring_buffer.release_buffer(allocated_id3) @@ -281,12 +281,12 @@ def test_wrap_around(self): ring_buffer.release_buffer(allocated_id5) print(ring_buffer) allocated_id6, tensor6 = ring_buffer.get_buffer(buffer_size) - assert ( - allocated_id6 is not None - ), "Failed to allocate buffer for full capacity after releasing all buffers" - assert ( - tensor6.nbytes == buffer_size - ), f"Expected buffer of size {buffer_size}, got {tensor6.nbytes}" + assert allocated_id6 is not None, ( + "Failed to allocate buffer for full capacity after releasing all buffers" + ) + assert tensor6.nbytes == buffer_size, ( + f"Expected buffer of size {buffer_size}, got {tensor6.nbytes}" + ) def test_looping(self): buffer_size = 64 * 3 @@ -306,12 +306,12 @@ def test_looping(self): while allocated_bytes < 64: new_byte_size = min(randint(8, 64), 64 - allocated_bytes) allocated_id, tensor = ring_buffer.get_buffer(new_byte_size) - assert ( - allocated_id is not None - ), "Failed to allocate buffer in looping test" - assert ( - tensor.nbytes == new_byte_size - ), f"Expected buffer of size {new_byte_size} in looping test" + assert allocated_id is not None, ( + "Failed to allocate buffer in looping test" + ) + assert tensor.nbytes == new_byte_size, ( + f"Expected buffer of size {new_byte_size} in looping test" + ) allocated_bytes += new_byte_size current_batch_ids.append(allocated_id) # Release previous batch diff --git a/components/src/dynamo/common/tests/test_storage.py b/components/src/dynamo/common/tests/test_storage.py index 3b05bb3f2739..8c4d641e33e7 100644 --- a/components/src/dynamo/common/tests/test_storage.py +++ b/components/src/dynamo/common/tests/test_storage.py @@ -46,9 +46,10 @@ def test_no_protocol_defaults_to_file(self, tmp_path): def test_s3_url_protocol(self): """Test s3:// URL extracts correct protocol and bucket path.""" - with patch("dynamo.common.storage.fsspec.filesystem") as mock_fsspec, patch( - "dynamo.common.storage.DirFileSystem" - ) as mock_dirfs: + with ( + patch("dynamo.common.storage.fsspec.filesystem") as mock_fsspec, + patch("dynamo.common.storage.DirFileSystem") as mock_dirfs, + ): mock_inner_fs = MagicMock(protocol="s3") mock_fsspec.return_value = mock_inner_fs get_fs("s3://my-bucket/prefix") @@ -59,9 +60,10 @@ def test_s3_url_protocol(self): def test_gs_url_protocol(self): """Test gs:// URL extracts correct protocol and path.""" - with patch("dynamo.common.storage.fsspec.filesystem") as mock_fsspec, patch( - "dynamo.common.storage.DirFileSystem" - ) as mock_dirfs: + with ( + patch("dynamo.common.storage.fsspec.filesystem") as mock_fsspec, + patch("dynamo.common.storage.DirFileSystem") as mock_dirfs, + ): mock_inner_fs = MagicMock(protocol="gs") mock_fsspec.return_value = mock_inner_fs get_fs("gs://my-gcs-bucket/data") diff --git a/components/src/dynamo/common/utils/media_nixl.py b/components/src/dynamo/common/utils/media_nixl.py index e8a2f02c7d29..af7e693624f8 100644 --- a/components/src/dynamo/common/utils/media_nixl.py +++ b/components/src/dynamo/common/utils/media_nixl.py @@ -20,8 +20,7 @@ async def read_decoded_media_via_nixl( connector: nixl_connect.Connector, decoded_meta: Dict[str, Any], return_metadata: Literal[False] = False, -) -> np.ndarray: - ... +) -> np.ndarray: ... @overload @@ -29,8 +28,7 @@ async def read_decoded_media_via_nixl( connector: nixl_connect.Connector, decoded_meta: Dict[str, Any], return_metadata: Literal[True], -) -> Tuple[np.ndarray, Dict[str, Any] | None]: - ... +) -> Tuple[np.ndarray, Dict[str, Any] | None]: ... async def read_decoded_media_via_nixl( diff --git a/components/src/dynamo/common/utils/nvtx_utils.py b/components/src/dynamo/common/utils/nvtx_utils.py index 37cc7c3b450c..f9853902dad4 100644 --- a/components/src/dynamo/common/utils/nvtx_utils.py +++ b/components/src/dynamo/common/utils/nvtx_utils.py @@ -31,6 +31,7 @@ async def my_async_gen(): start_range incur only a single dict lookup — no object allocation or domain cache lookups on the hot path. """ + import functools import inspect import os diff --git a/components/src/dynamo/common/utils/otel_tracing.py b/components/src/dynamo/common/utils/otel_tracing.py index d169d67f2db7..09c29482a735 100644 --- a/components/src/dynamo/common/utils/otel_tracing.py +++ b/components/src/dynamo/common/utils/otel_tracing.py @@ -5,7 +5,6 @@ OpenTelemetry tracing header utilities for Dynamo components. """ - from dynamo._core import Context diff --git a/components/src/dynamo/common/utils/prometheus.py b/components/src/dynamo/common/utils/prometheus.py index 80b851a612e5..66b0ab4cc48b 100644 --- a/components/src/dynamo/common/utils/prometheus.py +++ b/components/src/dynamo/common/utils/prometheus.py @@ -123,9 +123,9 @@ def register_engine_metrics_callback( # Add model labels if model_name is provided if model_name: auto_labels[labels.MODEL] = model_name # "model" (OpenAI standard) - auto_labels[ - labels.MODEL_NAME - ] = model_name # "model_name" (engine-native compatibility) + auto_labels[labels.MODEL_NAME] = ( + model_name # "model_name" (engine-native compatibility) + ) # Validate that user didn't provide conflicting auto-labels # Warn but don't error - custom labels have lower precedence than auto-labels diff --git a/components/src/dynamo/frontend/main.py b/components/src/dynamo/frontend/main.py index eb4c4be0a6c4..48cba093e527 100644 --- a/components/src/dynamo/frontend/main.py +++ b/components/src/dynamo/frontend/main.py @@ -289,9 +289,9 @@ def signal_handler(): os.environ.pop("DYN_ENABLE_STREAMING_REASONING_DISPATCH", None) if config.chat_processor == "vllm": - assert ( - vllm_flags is not None - ), "vllm_flags is required when chat processor is vllm" + assert vllm_flags is not None, ( + "vllm_flags is required when chat processor is vllm" + ) chat_engine_factory = setup_engine_factory( runtime, router_config, config, vllm_flags ).chat_engine_factory diff --git a/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py b/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py index c0e718cbe4c7..88aea3d9c7b4 100644 --- a/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py +++ b/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py @@ -9,7 +9,6 @@ Parallels test_vllm_unit.py for the vLLM backend. """ - import pytest from sglang.srt.utils.hf_transformers_utils import get_tokenizer @@ -587,9 +586,9 @@ def test_tool_choice_none_keeps_tools_when_flag_off(self, tokenizer): exclude_tools_when_tool_choice_none=False, ) # With flag off, both should have similar token counts (tools in template) - assert len(with_none.prompt_token_ids) == len( - with_auto.prompt_token_ids - ), "tool_choice=none with flag off should keep tools in template" + assert len(with_none.prompt_token_ids) == len(with_auto.prompt_token_ids), ( + "tool_choice=none with flag off should keep tools in template" + ) def test_init_worker_propagates_exclude_flag_true(self): """_init_worker sets the worker-global exclude_tools flag to True.""" diff --git a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py index 90b198ba5db4..22fc0659dfbe 100644 --- a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py +++ b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py @@ -54,9 +54,9 @@ def test_tool_choice_none_strips_tools_from_template(self, tokenizer): tool_parser_class=None, exclude_tools_when_tool_choice_none=True, ) - assert ( - chat_params.chat_template_kwargs["tools"] is None - ), "tool_choice=none with exclude flag should strip tools from template" + assert chat_params.chat_template_kwargs["tools"] is None, ( + "tool_choice=none with exclude flag should strip tools from template" + ) def test_tool_choice_none_keeps_tools_when_flag_off(self, tokenizer): """When exclude flag is off, tool_choice=none still includes tools in template kwargs.""" @@ -67,9 +67,9 @@ def test_tool_choice_none_keeps_tools_when_flag_off(self, tokenizer): exclude_tools_when_tool_choice_none=False, ) tools = chat_params.chat_template_kwargs["tools"] - assert ( - tools is not None and len(tools) == 1 - ), "tool_choice=none with flag off should keep tools in template" + assert tools is not None and len(tools) == 1, ( + "tool_choice=none with flag off should keep tools in template" + ) def test_tool_choice_auto_keeps_tools(self, tokenizer): """tool_choice=auto should always include tools regardless of flag.""" @@ -80,9 +80,9 @@ def test_tool_choice_auto_keeps_tools(self, tokenizer): exclude_tools_when_tool_choice_none=True, ) tools = chat_params.chat_template_kwargs["tools"] - assert ( - tools is not None and len(tools) == 1 - ), "tool_choice=auto should keep tools in template" + assert tools is not None and len(tools) == 1, ( + "tool_choice=auto should keep tools in template" + ) def test_tool_choice_required_keeps_tools(self, tokenizer): """tool_choice=required should always include tools regardless of flag.""" @@ -93,9 +93,9 @@ def test_tool_choice_required_keeps_tools(self, tokenizer): exclude_tools_when_tool_choice_none=True, ) tools = chat_params.chat_template_kwargs["tools"] - assert ( - tools is not None and len(tools) == 1 - ), "tool_choice=required should keep tools in template" + assert tools is not None and len(tools) == 1, ( + "tool_choice=required should keep tools in template" + ) def test_no_tools_in_request(self, tokenizer): """Request without tools should produce None tools in template kwargs.""" @@ -105,6 +105,6 @@ def test_no_tools_in_request(self, tokenizer): tool_parser_class=None, exclude_tools_when_tool_choice_none=True, ) - assert ( - chat_params.chat_template_kwargs["tools"] is None - ), "No tools in request should produce None tools in template" + assert chat_params.chat_template_kwargs["tools"] is None, ( + "No tools in request should produce None tools in template" + ) diff --git a/components/src/dynamo/frontend/vllm_processor.py b/components/src/dynamo/frontend/vllm_processor.py index dbd8c253b47b..f153d7471bd1 100644 --- a/components/src/dynamo/frontend/vllm_processor.py +++ b/components/src/dynamo/frontend/vllm_processor.py @@ -188,9 +188,9 @@ async def _generator_inner( if request_for_sampling.cache_salt is not None: prompt_inputs["cache_salt"] = request_for_sampling.cache_salt if request_for_sampling.mm_processor_kwargs is not None: - prompt_inputs[ - "mm_processor_kwargs" - ] = request_for_sampling.mm_processor_kwargs + prompt_inputs["mm_processor_kwargs"] = ( + request_for_sampling.mm_processor_kwargs + ) vllm_preproc: EngineCoreRequest = self.input_processor.process_inputs( request_id, diff --git a/components/src/dynamo/global_planner/tests/unit/test_scale_request_handler.py b/components/src/dynamo/global_planner/tests/unit/test_scale_request_handler.py index 04809c493a7b..3e703888c672 100644 --- a/components/src/dynamo/global_planner/tests/unit/test_scale_request_handler.py +++ b/components/src/dynamo/global_planner/tests/unit/test_scale_request_handler.py @@ -245,11 +245,12 @@ async def test_populate_connectors_explicit_mode(mock_runtime): max_total_gpus=-1, # Don't trigger discovery in __init__ ) - with patch( - "dynamo.global_planner.scale_handler.KubernetesAPI" - ) as mock_kube_cls, patch( - "dynamo.global_planner.scale_handler.KubernetesConnector" - ) as mock_connector_cls: + with ( + patch("dynamo.global_planner.scale_handler.KubernetesAPI") as mock_kube_cls, + patch( + "dynamo.global_planner.scale_handler.KubernetesConnector" + ) as mock_connector_cls, + ): mock_kube = MagicMock() mock_kube_cls.return_value = mock_kube mock_kube.list_graph_deployments.return_value = [ @@ -278,11 +279,12 @@ async def test_populate_connectors_implicit_mode(mock_runtime): max_total_gpus=-1, # Don't trigger discovery in __init__ ) - with patch( - "dynamo.global_planner.scale_handler.KubernetesAPI" - ) as mock_kube_cls, patch( - "dynamo.global_planner.scale_handler.KubernetesConnector" - ) as mock_connector_cls: + with ( + patch("dynamo.global_planner.scale_handler.KubernetesAPI") as mock_kube_cls, + patch( + "dynamo.global_planner.scale_handler.KubernetesConnector" + ) as mock_connector_cls, + ): mock_kube = MagicMock() mock_kube_cls.return_value = mock_kube mock_kube.list_graph_deployments.return_value = [ diff --git a/components/src/dynamo/planner/config/planner_config.py b/components/src/dynamo/planner/config/planner_config.py index d012fb6d5377..5528325a4c8f 100644 --- a/components/src/dynamo/planner/config/planner_config.py +++ b/components/src/dynamo/planner/config/planner_config.py @@ -46,9 +46,9 @@ class PlannerConfig(BaseModel): description='Controls pre-deployment sweeping mode for planner in-depth profiling. "none" means no pre-deployment sweep (only load-based scaling). "rapid" uses AI Configurator to simulate engine performance. "thorough" uses real GPUs to measure engine performance (takes several hours).', ) - environment: Literal[ - "kubernetes", "virtual", "global-planner" - ] = SLAPlannerDefaults.environment + environment: Literal["kubernetes", "virtual", "global-planner"] = ( + SLAPlannerDefaults.environment + ) namespace: str = Field( default_factory=lambda: os.environ.get("DYN_NAMESPACE", "dynamo") ) @@ -93,9 +93,9 @@ class PlannerConfig(BaseModel): metric_reporting_prometheus_port: int = Field( default_factory=lambda: int(os.environ.get("PLANNER_PROMETHEUS_PORT", 0)) ) - throughput_metrics_source: Literal[ - "frontend", "router" - ] = SLAPlannerDefaults.throughput_metrics_source + throughput_metrics_source: Literal["frontend", "router"] = ( + SLAPlannerDefaults.throughput_metrics_source + ) no_correction: bool = SLAPlannerDefaults.no_correction model_name: Optional[str] = None diff --git a/components/src/dynamo/planner/core/base.py b/components/src/dynamo/planner/core/base.py index e6b483922529..a15ad419c40e 100644 --- a/components/src/dynamo/planner/core/base.py +++ b/components/src/dynamo/planner/core/base.py @@ -475,9 +475,9 @@ async def observe_traffic_stats( ) # Prometheus returns seconds, convert to milliseconds - assert ( - self.model_name is not None - ), "model_name must be set before observing traffic stats" + assert self.model_name is not None, ( + "model_name must be set before observing traffic stats" + ) interval_str = f"{self.config.throughput_adjustment_interval}s" self.last_metrics.ttft = ( diff --git a/components/src/dynamo/planner/core/throughput/interpolation.py b/components/src/dynamo/planner/core/throughput/interpolation.py index 32f5e99070a2..c8de823f208b 100644 --- a/components/src/dynamo/planner/core/throughput/interpolation.py +++ b/components/src/dynamo/planner/core/throughput/interpolation.py @@ -62,7 +62,9 @@ def __init__( data = json.load(f) self.prefill_isl = np.array(data["prefill_isl"]) # type: ignore[index] self.prefill_ttft = np.array(data["prefill_ttft"]) # type: ignore[index] - self.prefill_thpt_per_gpu = np.array(data["prefill_thpt_per_gpu"]) # type: ignore[index] + self.prefill_thpt_per_gpu = np.array( + data["prefill_thpt_per_gpu"] + ) # type: ignore[index] except FileNotFoundError: raise FileNotFoundError( f"Prefill interpolation files not found: {prefill_npz_fn} and {json_fn}\n" @@ -299,7 +301,7 @@ def find_best_throughput_per_gpu( ) = decode_interpolator.find_best_throughput_per_gpu(args.itl, context_length) if est_itl <= args.itl: print( - f"\tEstimated ITL={est_itl:.2f}ms <= target ITL={args.itl:.2f}ms at {est_kv_usage*100:.2f}% active kv usage." + f"\tEstimated ITL={est_itl:.2f}ms <= target ITL={args.itl:.2f}ms at {est_kv_usage * 100:.2f}% active kv usage." ) print( f"\tEstimated throughput: {est_thpt_per_gpu:.2f} token/s/gpu. Request rate at {est_thpt_per_gpu / args.osl:.2f} requests/s will saturate one GPU." diff --git a/components/src/dynamo/planner/core/throughput/pre_swept_results.py b/components/src/dynamo/planner/core/throughput/pre_swept_results.py index 75ce7a340149..491643e3749d 100644 --- a/components/src/dynamo/planner/core/throughput/pre_swept_results.py +++ b/components/src/dynamo/planner/core/throughput/pre_swept_results.py @@ -168,9 +168,9 @@ def __init__(self, npz_file_path, configs: dict): for col in self.COLUMNS: # each row should include all the pre_swept_configs if col not in self.data.keys(): - assert ( - configs is not None and col in configs - ), f"Column {col} not found in pre_swept_configs: {configs}" + assert configs is not None and col in configs, ( + f"Column {col} not found in pre_swept_configs: {configs}" + ) self.data[col] = np.array([configs[col]]) updated = True if updated: @@ -203,9 +203,9 @@ def __init__(self, npz_file_path, configs: dict): updated = False for col in self.COLUMNS: if col not in self.data.keys(): - assert ( - configs is not None and col in configs - ), f"Column {col} not found in pre_swept_configs: {configs}" + assert configs is not None and col in configs, ( + f"Column {col} not found in pre_swept_configs: {configs}" + ) self.data[col] = np.array([configs[col]]) updated = True if updated: @@ -233,7 +233,7 @@ def merge_raw_data(raw_data_npz_path, configs, mode): raise ValueError(f"Invalid mode: {mode}") # merge the npz file - merged_npz_path = f'pre_swept_results/{configs["gpu_type"]}/{configs["framework"]}/{configs["model"]}/{mode}.npz' + merged_npz_path = f"pre_swept_results/{configs['gpu_type']}/{configs['framework']}/{configs['model']}/{mode}.npz" os.makedirs(os.path.dirname(merged_npz_path), exist_ok=True) if not os.path.exists(merged_npz_path): # copy the raw data npz file to the merged npz file and add one dimension. diff --git a/components/src/dynamo/planner/monitoring/worker_info.py b/components/src/dynamo/planner/monitoring/worker_info.py index c096914f1499..fd1177ec6c58 100644 --- a/components/src/dynamo/planner/monitoring/worker_info.py +++ b/components/src/dynamo/planner/monitoring/worker_info.py @@ -174,8 +174,7 @@ def resolve_worker_info( logger.info(f"Using model name from config: {model_name}") else: raise ValueError( - "Could not determine model name. " - "Please set model_name in the config." + "Could not determine model name. Please set model_name in the config." ) prefill_info.model_name = model_name diff --git a/components/src/dynamo/planner/offline/dryrun.py b/components/src/dynamo/planner/offline/dryrun.py index 81922759ac1c..f65aa4edd6c5 100644 --- a/components/src/dynamo/planner/offline/dryrun.py +++ b/components/src/dynamo/planner/offline/dryrun.py @@ -130,9 +130,9 @@ def compute_safe_d_thpt(num_d: int, isl: float, osl: float, itl: float): # predict_load() returns Optional[float] values; in dryrun mode with # pre-loaded data the predictors always return valid floats. - assert ( - _est_rr is not None and _est_isl is not None and _est_osl is not None - ), "predict_load() returned None in dryrun mode" + assert _est_rr is not None and _est_isl is not None and _est_osl is not None, ( + "predict_load() returned None in dryrun mode" + ) est_rr.append(_est_rr) est_isl.append(_est_isl) diff --git a/components/src/dynamo/planner/tests/manual/scaling/scaling_e2e.py b/components/src/dynamo/planner/tests/manual/scaling/scaling_e2e.py index a8d1aa9cc91d..6e689c32b776 100644 --- a/components/src/dynamo/planner/tests/manual/scaling/scaling_e2e.py +++ b/components/src/dynamo/planner/tests/manual/scaling/scaling_e2e.py @@ -286,9 +286,9 @@ def validate_test_results(self, results: Dict[str, Any]) -> Dict[str, Any]: validation["test_passed"] = True validation["summary"] = "PASS: Successfully scaled from 1P1D to 2P1D" else: - validation[ - "summary" - ] = "FAIL: Did not achieve expected 1P1D -> 2P1D scaling" + validation["summary"] = ( + "FAIL: Did not achieve expected 1P1D -> 2P1D scaling" + ) baseline = results.get("baseline_results", {}) trigger = results.get("trigger_results", {}) diff --git a/components/src/dynamo/planner/tests/unit/load_generator.py b/components/src/dynamo/planner/tests/unit/load_generator.py index cee2d2de0a4c..a8760ab95772 100644 --- a/components/src/dynamo/planner/tests/unit/load_generator.py +++ b/components/src/dynamo/planner/tests/unit/load_generator.py @@ -144,9 +144,10 @@ async def generate_load( stdout_path = os.path.join(artifact_dir, "aiperf.stdout.log") stderr_path = os.path.join(artifact_dir, "aiperf.stderr.log") try: - with open(stdout_path, "wb") as stdout_f, open( - stderr_path, "wb" - ) as stderr_f: + with ( + open(stdout_path, "wb") as stdout_f, + open(stderr_path, "wb") as stderr_f, + ): proc = await asyncio.create_subprocess_exec( *cmd, stdout=stdout_f, @@ -288,7 +289,7 @@ async def run_scaling_test(self, mode: str = "throughput") -> Dict[str, Any]: phase_results = {} for i, phase in enumerate(phases): - phase_name = f"phase{i+1}_{phase['name']}" + phase_name = f"phase{i + 1}_{phase['name']}" logger.info( f"Starting {phase_name}: {phase['rate']} req/s for {phase['duration']}s" ) diff --git a/components/src/dynamo/planner/tests/unit/test_load_predictors.py b/components/src/dynamo/planner/tests/unit/test_load_predictors.py index b4d682c67c5c..af2617b85c6f 100644 --- a/components/src/dynamo/planner/tests/unit/test_load_predictors.py +++ b/components/src/dynamo/planner/tests/unit/test_load_predictors.py @@ -161,9 +161,9 @@ def test_next_timestamp_uses_step_size(self): ) buggy_next_ts = predictor.start_date + timedelta(seconds=predictor.curr_step) - assert ( - expected_next_ts != buggy_next_ts - ), "Sanity check: the two timestamps must differ" + assert expected_next_ts != buggy_next_ts, ( + "Sanity check: the two timestamps must differ" + ) captured_future_df: list[pd.DataFrame] = [] @@ -401,9 +401,9 @@ def fake_predict(df): predictor.predict_next() actual_ts = captured[0]["ds"].iloc[0] - assert ( - actual_ts == expected_next_ts - ), f"step_size={step_size}: expected {expected_next_ts}, got {actual_ts}" + assert actual_ts == expected_next_ts, ( + f"step_size={step_size}: expected {expected_next_ts}, got {actual_ts}" + ) # --------------------------------------------------------------------------- diff --git a/components/src/dynamo/planner/tests/unit/test_replica_calculation.py b/components/src/dynamo/planner/tests/unit/test_replica_calculation.py index acae9043c507..a26cc337ca43 100644 --- a/components/src/dynamo/planner/tests/unit/test_replica_calculation.py +++ b/components/src/dynamo/planner/tests/unit/test_replica_calculation.py @@ -437,12 +437,12 @@ async def mock_get_workers_info(): ) print(f"Load {num_req} req/s: P={prefill_replicas}, D={decode_replicas}") - assert ( - prefill_replicas == expected_p - ), f"Prefill replicas mismatch: expected {expected_p}, got {prefill_replicas}" - assert ( - decode_replicas == expected_d - ), f"Decode replicas mismatch: expected {expected_d}, got {decode_replicas}" + assert prefill_replicas == expected_p, ( + f"Prefill replicas mismatch: expected {expected_p}, got {prefill_replicas}" + ) + assert decode_replicas == expected_d, ( + f"Decode replicas mismatch: expected {expected_d}, got {decode_replicas}" + ) @pytest.mark.nightly @pytest.mark.gpu_2 @@ -499,9 +499,9 @@ async def mock_get_workers_info(): f"GPU budget test: P={prefill_replicas}, D={decode_replicas}, Total GPUs={total_gpus}" ) - assert ( - total_gpus <= planner.config.max_gpu_budget - ), "Total GPU usage exceeds budget" + assert total_gpus <= planner.config.max_gpu_budget, ( + "Total GPU usage exceeds budget" + ) @pytest.mark.nightly @pytest.mark.gpu_2 @@ -550,12 +550,12 @@ async def mock_get_workers_info(): ) print(f"Min endpoint test: P={prefill_replicas}, D={decode_replicas}") - assert ( - prefill_replicas >= planner.config.min_endpoint - ), "Prefill replicas below minimum" - assert ( - decode_replicas >= planner.config.min_endpoint - ), "Decode replicas below minimum" + assert prefill_replicas >= planner.config.min_endpoint, ( + "Prefill replicas below minimum" + ) + assert decode_replicas >= planner.config.min_endpoint, ( + "Decode replicas below minimum" + ) @pytest.mark.nightly @pytest.mark.gpu_2 @@ -679,9 +679,9 @@ async def mock_get_workers_info(): ) # Should get a valid result (not crash) - assert ( - decode_replicas >= 1 - ), f"Should handle correction factor {correction_factor} gracefully" + assert decode_replicas >= 1, ( + f"Should handle correction factor {correction_factor} gracefully" + ) @pytest.mark.nightly @pytest.mark.gpu_2 @@ -811,15 +811,15 @@ async def mock_get_workers_info(): f"Complex GPU budget test: P={prefill_replicas}, D={decode_replicas}, Total GPUs={total_gpus}" ) - assert ( - total_gpus <= planner.config.max_gpu_budget - ), "Total GPU usage should not exceed budget" - assert ( - prefill_replicas >= planner.config.min_endpoint - ), "Should respect min_endpoint for prefill" - assert ( - decode_replicas >= planner.config.min_endpoint - ), "Should respect min_endpoint for decode" + assert total_gpus <= planner.config.max_gpu_budget, ( + "Total GPU usage should not exceed budget" + ) + assert prefill_replicas >= planner.config.min_endpoint, ( + "Should respect min_endpoint for prefill" + ) + assert decode_replicas >= planner.config.min_endpoint, ( + "Should respect min_endpoint for decode" + ) # No need for unittest.main() with pytest! diff --git a/components/src/dynamo/profiler/tests/integration/test_profile_sla_dgdr.py b/components/src/dynamo/profiler/tests/integration/test_profile_sla_dgdr.py index ad8d383a4488..c6d1d01779b7 100644 --- a/components/src/dynamo/profiler/tests/integration/test_profile_sla_dgdr.py +++ b/components/src/dynamo/profiler/tests/integration/test_profile_sla_dgdr.py @@ -108,9 +108,9 @@ def test_pvc_no_planner_with_load(self, tmp_path): assert config spec = config.get("spec", {}) pvcs = spec.get("pvcs", []) - assert any( - p.get("name") == "model-cache" for p in pvcs - ), "PVC should be mounted" + assert any(p.get("name") == "model-cache" for p in pvcs), ( + "PVC should be mounted" + ) @pytest.mark.pre_merge @pytest.mark.gpu_0 @@ -152,9 +152,9 @@ def test_planner_rapid_sweep(self, tmp_path): docs = list(yaml.safe_load_all(raw)) assert len(docs) >= 2, "Planner config should produce multi-doc YAML" dgd = docs[-1] - assert "Planner" in dgd.get("spec", {}).get( - "services", {} - ), "Planner service should be added" + assert "Planner" in dgd.get("spec", {}).get("services", {}), ( + "Planner service should be added" + ) class TestRapidUnsupported: @@ -506,9 +506,9 @@ def _assert_overrides_applied(final_config_path: Path, dgdr): if svc_name in dgd_services: dgd_svc = dgd_services[svc_name] if "sharedMemory" in svc_ovr: - assert ( - "sharedMemory" in dgd_svc - ), f"Override sharedMemory on {svc_name} should be applied" + assert "sharedMemory" in dgd_svc, ( + f"Override sharedMemory on {svc_name} should be applied" + ) mc = svc_ovr.get("extraPodSpec", {}).get("mainContainer", {}) if "args" in mc: dgd_args = ( @@ -517,9 +517,9 @@ def _assert_overrides_applied(final_config_path: Path, dgdr): .get("args", []) ) for arg in mc["args"]: - assert ( - arg in dgd_args - ), f"Override arg '{arg}' should be in {svc_name} args" + assert arg in dgd_args, ( + f"Override arg '{arg}' should be in {svc_name} args" + ) class TestThoroughMockedOverrides: diff --git a/components/src/dynamo/profiler/tests/unit/test_helpers_profile_sla.py b/components/src/dynamo/profiler/tests/unit/test_helpers_profile_sla.py index 9720f472840f..1dda1377619a 100644 --- a/components/src/dynamo/profiler/tests/unit/test_helpers_profile_sla.py +++ b/components/src/dynamo/profiler/tests/unit/test_helpers_profile_sla.py @@ -652,9 +652,9 @@ def test_mocker_disabled_no_planner_profile_data_in_workers(self, tmp_path): args = dgd["spec"]["services"][name]["extraPodSpec"]["mainContainer"][ "args" ] - assert ( - "--planner-profile-data" not in args - ), f"sglang worker '{name}' should not have --planner-profile-data" + assert "--planner-profile-data" not in args, ( + f"sglang worker '{name}' should not have --planner-profile-data" + ) @pytest.mark.pre_merge @pytest.mark.gpu_0 @@ -668,9 +668,9 @@ def test_mocker_enabled_injects_planner_profile_data(self, tmp_path): args = dgd["spec"]["services"][name]["extraPodSpec"]["mainContainer"][ "args" ] - assert ( - "--planner-profile-data" in args - ), f"mocker worker '{name}' should have --planner-profile-data" + assert "--planner-profile-data" in args, ( + f"mocker worker '{name}' should have --planner-profile-data" + ) # --------------------------------------------------------------------------- @@ -720,13 +720,13 @@ def test_naive_fallback_resolved_backend_auto(self): ) # The resolved backend must be a concrete name, not 'auto' - assert ( - "resolved_backend" in result - ), "result dict must contain 'resolved_backend' key" + assert "resolved_backend" in result, ( + "result dict must contain 'resolved_backend' key" + ) resolved = result["resolved_backend"] - assert ( - resolved != "auto" - ), f"resolved_backend must not be 'auto', got {resolved!r}" + assert resolved != "auto", ( + f"resolved_backend must not be 'auto', got {resolved!r}" + ) assert resolved in ( "vllm", "sglang", @@ -1020,6 +1020,6 @@ def test_run_profile_calls_interpolation_with_resolved_backend_for_disagg( if call_kwargs.args else call_kwargs.kwargs.get("backend") ) - assert ( - called_backend == "vllm" - ), f"run_interpolation must be called with resolved backend 'vllm', got {called_backend!r}" + assert called_backend == "vllm", ( + f"run_interpolation must be called with resolved backend 'vllm', got {called_backend!r}" + ) diff --git a/components/src/dynamo/profiler/tests/unit/test_profile_sla_auto_backend.py b/components/src/dynamo/profiler/tests/unit/test_profile_sla_auto_backend.py index c77857d81376..4925ce49104e 100644 --- a/components/src/dynamo/profiler/tests/unit/test_profile_sla_auto_backend.py +++ b/components/src/dynamo/profiler/tests/unit/test_profile_sla_auto_backend.py @@ -26,12 +26,12 @@ def test_autoscale_sim_resolves_auto_to_default() -> None: src = inspect.getsource(_run_autoscale_sim) # The function must guard against "auto" before TaskConfig is constructed. - assert ( - 'backend == "auto"' in src - ), "_run_autoscale_sim must resolve backend='auto' before constructing TaskConfig" - assert ( - "_DEFAULT_NAIVE_BACKEND" in src - ), "_run_autoscale_sim must fall back to _DEFAULT_NAIVE_BACKEND when backend='auto'" + assert 'backend == "auto"' in src, ( + "_run_autoscale_sim must resolve backend='auto' before constructing TaskConfig" + ) + assert "_DEFAULT_NAIVE_BACKEND" in src, ( + "_run_autoscale_sim must fall back to _DEFAULT_NAIVE_BACKEND when backend='auto'" + ) def test_autoscale_sim_returns_resolved_backend() -> None: @@ -43,9 +43,9 @@ def test_autoscale_sim_returns_resolved_backend() -> None: from dynamo.profiler.rapid import _run_autoscale_sim src = inspect.getsource(_run_autoscale_sim) - assert ( - '"resolved_backend"' in src - ), "_run_autoscale_sim must return 'resolved_backend' in its result dict" + assert '"resolved_backend"' in src, ( + "_run_autoscale_sim must return 'resolved_backend' in its result dict" + ) def test_naive_fallback_resolves_auto_to_default() -> None: @@ -60,9 +60,9 @@ def test_naive_fallback_resolves_auto_to_default() -> None: from dynamo.profiler.rapid import _run_naive_fallback src = inspect.getsource(_run_naive_fallback) - assert ( - 'backend == "auto"' in src - ), "_run_naive_fallback must resolve backend='auto' before calling AIC helpers" + assert 'backend == "auto"' in src, ( + "_run_naive_fallback must resolve backend='auto' before calling AIC helpers" + ) assert "_DEFAULT_NAIVE_BACKEND" in src @@ -78,9 +78,9 @@ def test_default_sim_returns_resolved_backend() -> None: from dynamo.profiler.rapid import _run_default_sim src = inspect.getsource(_run_default_sim) - assert ( - '"resolved_backend"' in src - ), "_run_default_sim must return 'resolved_backend' in its result dict" + assert '"resolved_backend"' in src, ( + "_run_default_sim must return 'resolved_backend' in its result dict" + ) def test_default_naive_backend_is_concrete() -> None: diff --git a/components/src/dynamo/profiler/utils/config_modifiers/protocol.py b/components/src/dynamo/profiler/utils/config_modifiers/protocol.py index e1ce6df9f73b..6b1244ecf4c8 100644 --- a/components/src/dynamo/profiler/utils/config_modifiers/protocol.py +++ b/components/src/dynamo/profiler/utils/config_modifiers/protocol.py @@ -43,8 +43,7 @@ def convert_config( config: dict, target: EngineType, is_moe_model: bool = False, - ) -> dict: - ... + ) -> dict: ... @classmethod def set_config_tp_size( @@ -52,8 +51,7 @@ def set_config_tp_size( config: dict, tp_size: int, component_type: SubComponentType = SubComponentType.DECODE, - ) -> dict: - ... + ) -> dict: ... @classmethod def set_config_tep_size( @@ -62,8 +60,7 @@ def set_config_tep_size( tep_size: int, num_gpus_per_node: int, component_type: SubComponentType = SubComponentType.DECODE, - ) -> dict: - ... + ) -> dict: ... @classmethod def set_config_dep_size( @@ -72,12 +69,10 @@ def set_config_dep_size( dep_size: int, num_gpus_per_node: int, component_type: SubComponentType = SubComponentType.DECODE, - ) -> dict: - ... + ) -> dict: ... @classmethod - def get_model_name(cls, config: dict) -> Tuple[str, str]: - ... + def get_model_name(cls, config: dict) -> Tuple[str, str]: ... @classmethod def set_prefill_config( @@ -86,32 +81,26 @@ def set_prefill_config( max_batch_size: int, max_num_tokens: int, component_type: SubComponentType = SubComponentType.DECODE, - ) -> dict: - ... + ) -> dict: ... @classmethod - def get_port(cls, config: dict) -> int: - ... + def get_port(cls, config: dict) -> int: ... @classmethod def get_kv_cache_size_from_dynamo_log( cls, dynamo_log_fn: str, attention_dp_size: int = 1 - ) -> int: - ... + ) -> int: ... @classmethod - def load_default_config(cls, mode: str = "disagg") -> dict: - ... + def load_default_config(cls, mode: str = "disagg") -> dict: ... @classmethod def update_model( cls, config: dict, model_name: str, model_path: str | None = None - ) -> dict: - ... + ) -> dict: ... @classmethod - def update_image(cls, config: dict, image: str) -> dict: - ... + def update_image(cls, config: dict, image: str) -> dict: ... @classmethod def update_model_from_pvc( @@ -121,8 +110,7 @@ def update_model_from_pvc( pvc_name: str, pvc_mount_path: str, pvc_path: str, - ) -> dict: - ... + ) -> dict: ... class BaseConfigModifier: diff --git a/components/src/dynamo/profiler/utils/config_modifiers/trtllm.py b/components/src/dynamo/profiler/utils/config_modifiers/trtllm.py index 21331f2c231b..b17d4a0904aa 100644 --- a/components/src/dynamo/profiler/utils/config_modifiers/trtllm.py +++ b/components/src/dynamo/profiler/utils/config_modifiers/trtllm.py @@ -129,12 +129,12 @@ def convert_config( ): override_dict["kv_cache_config"] = {} override_dict["kv_cache_config"]["enable_block_reuse"] = False - override_dict[ - "disable_overlap_scheduler" - ] = False # Enable overlap scheduler for agg - override_dict[ - "cache_transceiver_config" - ] = None # Remove cache transceiver for agg + override_dict["disable_overlap_scheduler"] = ( + False # Enable overlap scheduler for agg + ) + override_dict["cache_transceiver_config"] = ( + None # Remove cache transceiver for agg + ) override_str = json.dumps(override_dict) args = append_argument(args, ["--override-engine-args", override_str]) @@ -182,9 +182,9 @@ def convert_config( ): override_dict["kv_cache_config"] = {} override_dict["kv_cache_config"]["enable_block_reuse"] = True - override_dict[ - "cache_transceiver_config" - ] = None # Remove cache transceiver for agg + override_dict["cache_transceiver_config"] = ( + None # Remove cache transceiver for agg + ) override_str = json.dumps(override_dict) args = append_argument(args, ["--override-engine-args", override_str]) diff --git a/components/src/dynamo/profiler/utils/replay_optimize/evaluate.py b/components/src/dynamo/profiler/utils/replay_optimize/evaluate.py index 4c95fb8135d5..2a46d3747865 100644 --- a/components/src/dynamo/profiler/utils/replay_optimize/evaluate.py +++ b/components/src/dynamo/profiler/utils/replay_optimize/evaluate.py @@ -280,7 +280,7 @@ def _evaluate_state_from_json_payloads(payload: Mapping[str, Any]) -> dict[str, def _evaluate_agg_state_from_json_payloads( - payload: Mapping[str, Any] + payload: Mapping[str, Any], ) -> dict[str, Any]: return _evaluate_agg_state( state=payload["state"], diff --git a/components/src/dynamo/profiler/utils/replay_optimize/search.py b/components/src/dynamo/profiler/utils/replay_optimize/search.py index bbe0bba2b74f..93a776c74b21 100644 --- a/components/src/dynamo/profiler/utils/replay_optimize/search.py +++ b/components/src/dynamo/profiler/utils/replay_optimize/search.py @@ -282,7 +282,7 @@ def _record_to_state(record: Mapping[str, float | int]) -> DenseReplayState: def _record_to_agg_state( - record: Mapping[str, float | int | str] + record: Mapping[str, float | int | str], ) -> DenseAggReplayState: return DenseAggReplayState( tp=int(record["tp"]), diff --git a/components/src/dynamo/profiler/utils/search_space_autogen.py b/components/src/dynamo/profiler/utils/search_space_autogen.py index a5995f558929..65b4840cda64 100644 --- a/components/src/dynamo/profiler/utils/search_space_autogen.py +++ b/components/src/dynamo/profiler/utils/search_space_autogen.py @@ -127,9 +127,9 @@ def auto_generate_search_space(args: argparse.Namespace) -> None: logger.error(error_msg) raise RuntimeError(error_msg) - assert ( - model_info is not None - ), "model_info must be set when model is provided" + assert model_info is not None, ( + "model_info must be set when model is provided" + ) logger.info( f"Auto-generating search space: {args.num_gpus_per_node}x {args.gpu_model} GPUs with {args.gpu_vram_mib} MiB VRAM per GPU" diff --git a/components/src/dynamo/profiler/webui/utils.py b/components/src/dynamo/profiler/webui/utils.py index 18bda35ea437..c1cdb904557e 100644 --- a/components/src/dynamo/profiler/webui/utils.py +++ b/components/src/dynamo/profiler/webui/utils.py @@ -201,18 +201,18 @@ def generate_config_data( # Set SLA targets data[PlotType.PREFILL]["chart"]["target_line"]["value"] = args.ttft - data[PlotType.PREFILL]["chart"]["target_line"][ - "label" - ] = f"Target TTFT: {args.ttft} ms" + data[PlotType.PREFILL]["chart"]["target_line"]["label"] = ( + f"Target TTFT: {args.ttft} ms" + ) data[PlotType.DECODE]["chart"]["target_line"]["value"] = args.itl - data[PlotType.DECODE]["chart"]["target_line"][ - "label" - ] = f"Target ITL: {args.itl} ms" + data[PlotType.DECODE]["chart"]["target_line"]["label"] = ( + f"Target ITL: {args.itl} ms" + ) - data[PlotType.COST]["chart"][ - "title" - ] = f"GPU Hours Per 1000 i{args.isl}o{args.osl} requests" + data[PlotType.COST]["chart"]["title"] = ( + f"GPU Hours Per 1000 i{args.isl}o{args.osl} requests" + ) # Populate data sections populate_prefill_data(data, prefill_data, args) diff --git a/components/src/dynamo/sglang/protocol.py b/components/src/dynamo/sglang/protocol.py index 4d092c291378..aa73c2ef409e 100644 --- a/components/src/dynamo/sglang/protocol.py +++ b/components/src/dynamo/sglang/protocol.py @@ -54,9 +54,9 @@ class EmbeddingRequest(BaseModel): model: str input: EmbeddingInput user: Optional[str] = None - dimensions: Optional[ - int - ] = None # only supported in text-embedding-3 and later models from OpenAI + dimensions: Optional[int] = ( + None # only supported in text-embedding-3 and later models from OpenAI + ) class DisaggPreprocessedRequest(BaseModel): diff --git a/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py b/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py index d8218bf394c7..3f44a8fab875 100644 --- a/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py +++ b/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py @@ -102,18 +102,18 @@ def __init__( if image_token_str == "<|vision_start|><|image_pad|><|vision_end|>": # These are likely the individual special tokens for Qwen2.5-VL image_pad_id = self.tokenizer.convert_tokens_to_ids("<|image_pad|>") - assert isinstance( - image_pad_id, int - ), f"Expected int token id, got {type(image_pad_id)}" + assert isinstance(image_pad_id, int), ( + f"Expected int token id, got {type(image_pad_id)}" + ) # Use the image_pad token as the main image token self.image_token_id: int = image_pad_id else: # Fallback for other models token_id = self.tokenizer.convert_tokens_to_ids(image_token_str) - assert isinstance( - token_id, int - ), f"Expected int token id, got {type(token_id)}" + assert isinstance(token_id, int), ( + f"Expected int token id, got {type(token_id)}" + ) self.image_token_id = token_id self.min_workers = 1 diff --git a/components/src/dynamo/sglang/request_handlers/multimodal/worker_handler.py b/components/src/dynamo/sglang/request_handlers/multimodal/worker_handler.py index 828fdb1f32d9..536652397cda 100644 --- a/components/src/dynamo/sglang/request_handlers/multimodal/worker_handler.py +++ b/components/src/dynamo/sglang/request_handlers/multimodal/worker_handler.py @@ -87,7 +87,7 @@ async def process_embeddings( self, request: SglangMultimodalRequest ) -> tuple[torch.Tensor, int]: """Process one concatenated embedding tensor from serialized request.""" - logger.debug("Processing embeddings with shape: " f"{request.embeddings_shape}") + logger.debug(f"Processing embeddings with shape: {request.embeddings_shape}") multimodal_groups = request.multimodal_inputs if not multimodal_groups: @@ -459,7 +459,7 @@ async def _generate_aggregated( "Shape mismatch error - this likely indicates a tokenization/embedding alignment issue" ) logger.error(f"Request token IDs length: {len(input_ids)}") - logger.error("Embeddings shape: " f"{request.embeddings_shape}") + logger.error(f"Embeddings shape: {request.embeddings_shape}") logger.error(f"Token sequence preview: {input_ids[:20]}...") error_msg = ( f"Multimodal embedding alignment error: {str(e)}. " diff --git a/components/src/dynamo/sglang/request_handlers/video_generation/video_generation_handler.py b/components/src/dynamo/sglang/request_handlers/video_generation/video_generation_handler.py index 425d3a9d7a21..e34c92d508f0 100644 --- a/components/src/dynamo/sglang/request_handlers/video_generation/video_generation_handler.py +++ b/components/src/dynamo/sglang/request_handlers/video_generation/video_generation_handler.py @@ -117,9 +117,9 @@ async def generate( # Generate video context_id = context.id() assert context_id is not None - assert ( - nvext.num_inference_steps is not None - ), "Num inference steps is required" + assert nvext.num_inference_steps is not None, ( + "Num inference steps is required" + ) video_bytes = await self._generate_video( prompt=req.prompt, width=width, diff --git a/components/src/dynamo/sglang/shutdown.py b/components/src/dynamo/sglang/shutdown.py index cad58f276ef4..365352f74205 100644 --- a/components/src/dynamo/sglang/shutdown.py +++ b/components/src/dynamo/sglang/shutdown.py @@ -33,9 +33,7 @@ def install_graceful_shutdown( """ deferred_handlers: DefaultDict[ int, list[tuple[SignalCallback, tuple[Any, ...]]] - ] = defaultdict( - list - ) # type: ignore[assignment] + ] = defaultdict(list) # type: ignore[assignment] shutdown_started = False shutdown_signum: int | None = None diff --git a/components/src/dynamo/sglang/snapshot.py b/components/src/dynamo/sglang/snapshot.py index 79813ebc9b8d..f4cb003470ec 100644 --- a/components/src/dynamo/sglang/snapshot.py +++ b/components/src/dynamo/sglang/snapshot.py @@ -3,7 +3,6 @@ """Dynamo Snapshot integration for SGLang workers.""" - import logging import time diff --git a/components/src/dynamo/trtllm/multimodal_processor.py b/components/src/dynamo/trtllm/multimodal_processor.py index 8cd4a541efea..6d2f20cc6caa 100644 --- a/components/src/dynamo/trtllm/multimodal_processor.py +++ b/components/src/dynamo/trtllm/multimodal_processor.py @@ -43,8 +43,7 @@ def decode( token_ids: List[int], skip_special_tokens: bool = True, clean_up_tokenization_spaces: bool = True, - ) -> str: - ... + ) -> str: ... class MultimodalRequestProcessor: @@ -93,7 +92,7 @@ def load_tensor_from_path_or_url(self, path: str) -> torch.Tensor: data = response.read(self.max_file_size_bytes + 1) if len(data) > self.max_file_size_bytes: raise RuntimeError( - f"File size exceeds limit: {len(data) // (1024*1024)}MB > " + f"File size exceeds limit: {len(data) // (1024 * 1024)}MB > " f"{self.max_file_size_mb}MB " ) tensor_stream = BytesIO(data) @@ -135,8 +134,8 @@ def load_tensor_from_path_or_url(self, path: str) -> torch.Tensor: file_size = resolved_path.stat().st_size if file_size > self.max_file_size_bytes: raise RuntimeError( - f"File size ({file_size // (1024*1024)}MB) exceeds " - f"maximum allowed size ({self.max_file_size_bytes // (1024*1024)}MB)" + f"File size ({file_size // (1024 * 1024)}MB) exceeds " + f"maximum allowed size ({self.max_file_size_bytes // (1024 * 1024)}MB)" ) return torch.load(resolved_path, map_location="cpu", weights_only=True) except Exception as e: diff --git a/components/src/dynamo/trtllm/publisher.py b/components/src/dynamo/trtllm/publisher.py index 819e25ac1b87..ae15357ca10f 100644 --- a/components/src/dynamo/trtllm/publisher.py +++ b/components/src/dynamo/trtllm/publisher.py @@ -328,9 +328,9 @@ def __init__( # Needed by the events and metrics publishers self.metrics_publisher: Optional[WorkerMetricsPublisher] = None - self.kv_event_publishers: Optional[ - Dict[int, KvEventPublisher] - ] = None # One per attention_dp_rank + self.kv_event_publishers: Optional[Dict[int, KvEventPublisher]] = ( + None # One per attention_dp_rank + ) self.zmq_kv_event_publisher = None # ZMQ publisher for consolidator self.publish_kv_cache_events_thread: Optional[ManagedThread] = None self.publish_stats_thread: Optional[ManagedThread] = None diff --git a/components/src/dynamo/trtllm/request_handlers/handler_base.py b/components/src/dynamo/trtllm/request_handlers/handler_base.py index 500d9892c712..2614d6ad464c 100644 --- a/components/src/dynamo/trtllm/request_handlers/handler_base.py +++ b/components/src/dynamo/trtllm/request_handlers/handler_base.py @@ -69,13 +69,13 @@ class RequestHandlerConfig: publisher: Optional[Publisher] disaggregation_mode: DisaggregationMode encode_client: Optional[Client] = None - multimodal_processor: Optional[ - MultimodalRequestProcessor - ] = None # for multimodal support + multimodal_processor: Optional[MultimodalRequestProcessor] = ( + None # for multimodal support + ) connector: Optional[Connector] = None - runtime: Optional[ - DistributedRuntime - ] = None # DistributedRuntime reference for graceful shutdown + runtime: Optional[DistributedRuntime] = ( + None # DistributedRuntime reference for graceful shutdown + ) metrics_collector: Optional["MetricsCollector"] = None kv_block_size: int = 32 shutdown_event: Optional[asyncio.Event] = None diff --git a/components/src/dynamo/trtllm/request_handlers/video_diffusion/video_handler.py b/components/src/dynamo/trtllm/request_handlers/video_diffusion/video_handler.py index 8ec757e98fa7..b1ee6b580acd 100644 --- a/components/src/dynamo/trtllm/request_handlers/video_diffusion/video_handler.py +++ b/components/src/dynamo/trtllm/request_handlers/video_diffusion/video_handler.py @@ -242,9 +242,9 @@ async def generate( # MediaOutput.video is (B, T, H, W, C) uint8 since TRT-LLM rc9; # squeeze the batch dim to get (T, H, W, C) for MP4 encoding. video = output.video - assert ( - video.ndim == 5 and video.shape[0] == 1 - ), f"Expected video shape (1, T, H, W, C), got {video.shape}" + assert video.ndim == 5 and video.shape[0] == 1, ( + f"Expected video shape (1, T, H, W, C), got {video.shape}" + ) frames_np = video[0].cpu().numpy() logger.info( f"Request {request_id}: encoding video output " diff --git a/components/src/dynamo/trtllm/tests/test_trtllm_additional_metrics.py b/components/src/dynamo/trtllm/tests/test_trtllm_additional_metrics.py index 5b752e6316c5..d60f54a7aa45 100644 --- a/components/src/dynamo/trtllm/tests/test_trtllm_additional_metrics.py +++ b/components/src/dynamo/trtllm/tests/test_trtllm_additional_metrics.py @@ -30,9 +30,10 @@ def setUp(self): self.registry = CollectorRegistry() # Patch prometheus_client.Counter and Histogram to use our test registry - with patch("dynamo.trtllm.metrics.Counter") as MockCounter, patch( - "dynamo.trtllm.metrics.Histogram" - ) as MockHistogram: + with ( + patch("dynamo.trtllm.metrics.Counter") as MockCounter, + patch("dynamo.trtllm.metrics.Histogram") as MockHistogram, + ): from prometheus_client import Counter, Histogram def make_counter(name, documentation, labelnames=None, **_kw): diff --git a/components/src/dynamo/trtllm/tests/test_trtllm_autodeploy.py b/components/src/dynamo/trtllm/tests/test_trtllm_autodeploy.py index 69f953dca620..6bd32f134c74 100644 --- a/components/src/dynamo/trtllm/tests/test_trtllm_autodeploy.py +++ b/components/src/dynamo/trtllm/tests/test_trtllm_autodeploy.py @@ -80,9 +80,11 @@ async def test_unsupported_args_get_pruned_for_autodeploy( engine_args["backend"] = Backend.AUTODEPLOY # This allows us to catch cases where a field being pruned away is now supported by # AutoDeploy when bumping TRT-LLM. - with pytest.raises( - pydantic.ValidationError - ) if is_forbidden else contextlib.nullcontext(): + with ( + pytest.raises(pydantic.ValidationError) + if is_forbidden + else contextlib.nullcontext() + ): ADLlmArgs(model="foo", **engine_args) engine = TensorRTLLMEngine(engine_args=engine_args) diff --git a/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py b/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py index c4e8c21b94ab..fd42d2ea01d6 100644 --- a/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py +++ b/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py @@ -217,9 +217,9 @@ def test_guided_decoding_dict_is_converted(self): result = HandlerBase._override_sampling_params(sampling_params, request) - assert not isinstance( - result.guided_decoding, dict - ), "guided_decoding should be converted from dict to GuidedDecodingParams" + assert not isinstance(result.guided_decoding, dict), ( + "guided_decoding should be converted from dict to GuidedDecodingParams" + ) # Downstream code (TRT-LLM sampling_params.py) accesses these attributes: assert result.guided_decoding.json_object is False assert result.guided_decoding.json == self.GUIDED_DECODING_DICT["json"] diff --git a/components/src/dynamo/trtllm/tests/test_trtllm_unit.py b/components/src/dynamo/trtllm/tests/test_trtllm_unit.py index fb3ac7052f10..cea7c7fc9809 100644 --- a/components/src/dynamo/trtllm/tests/test_trtllm_unit.py +++ b/components/src/dynamo/trtllm/tests/test_trtllm_unit.py @@ -180,11 +180,15 @@ async def test_init_llm_worker_creates_multimodal_processor(): assert config.modality == Modality.MULTIMODAL # Mock everything init_llm_worker touches before MultimodalRequestProcessor. - with mock.patch("dynamo.trtllm.workers.llm_worker.tokenizer_factory"), mock.patch( - "dynamo.trtllm.workers.llm_worker.AutoConfig.from_pretrained", - ), mock.patch( - "dynamo.trtllm.workers.llm_worker.MultimodalRequestProcessor", - side_effect=MultimodalProcessorInstantiated, + with ( + mock.patch("dynamo.trtllm.workers.llm_worker.tokenizer_factory"), + mock.patch( + "dynamo.trtllm.workers.llm_worker.AutoConfig.from_pretrained", + ), + mock.patch( + "dynamo.trtllm.workers.llm_worker.MultimodalRequestProcessor", + side_effect=MultimodalProcessorInstantiated, + ), ): with pytest.raises(MultimodalProcessorInstantiated): await init_llm_worker( diff --git a/components/src/dynamo/trtllm/tests/test_trtllm_video_diffusion.py b/components/src/dynamo/trtllm/tests/test_trtllm_video_diffusion.py index dc6720281ad0..88ee4d146435 100644 --- a/components/src/dynamo/trtllm/tests/test_trtllm_video_diffusion.py +++ b/components/src/dynamo/trtllm/tests/test_trtllm_video_diffusion.py @@ -536,9 +536,9 @@ def __init__(self, **kwargs): num_inference_steps=1, ) - assert isinstance( - captured["prompt"], list - ), f"Expected list, got {type(captured['prompt'])}" + assert isinstance(captured["prompt"], list), ( + f"Expected list, got {type(captured['prompt'])}" + ) assert captured["prompt"] == ["a golden retriever"] @@ -695,12 +695,15 @@ async def run(): requests = [self._make_request() for _ in range(3)] - with patch( - "dynamo.trtllm.request_handlers.video_diffusion.video_handler.encode_to_mp4_bytes", - return_value=b"fake_mp4_bytes", - ), patch( - "dynamo.trtllm.request_handlers.video_diffusion.video_handler.upload_to_fs", - return_value="http://fake/video.mp4", + with ( + patch( + "dynamo.trtllm.request_handlers.video_diffusion.video_handler.encode_to_mp4_bytes", + return_value=b"fake_mp4_bytes", + ), + patch( + "dynamo.trtllm.request_handlers.video_diffusion.video_handler.upload_to_fs", + return_value="http://fake/video.mp4", + ), ): await asyncio.gather( *(self._drain_generator(handler, req) for req in requests) @@ -767,13 +770,16 @@ async def test_url_response_format(self): "response_format": "url", } - with patch( - "dynamo.trtllm.request_handlers.video_diffusion.video_handler.encode_to_mp4_bytes", - return_value=b"fake_mp4", - ), patch( - "dynamo.trtllm.request_handlers.video_diffusion.video_handler.upload_to_fs", - return_value="https://cdn.example.com/media/videos/test.mp4", - ) as mock_upload: + with ( + patch( + "dynamo.trtllm.request_handlers.video_diffusion.video_handler.encode_to_mp4_bytes", + return_value=b"fake_mp4", + ), + patch( + "dynamo.trtllm.request_handlers.video_diffusion.video_handler.upload_to_fs", + return_value="https://cdn.example.com/media/videos/test.mp4", + ) as mock_upload, + ): results = [] async for result in handler.generate(request, MagicMock()): results.append(result) @@ -831,13 +837,16 @@ async def test_default_response_format_is_url(self): # No response_format specified } - with patch( - "dynamo.trtllm.request_handlers.video_diffusion.video_handler.encode_to_mp4_bytes", - return_value=b"fake_mp4", - ), patch( - "dynamo.trtllm.request_handlers.video_diffusion.video_handler.upload_to_fs", - return_value="https://cdn.example.com/media/videos/test.mp4", - ) as mock_upload: + with ( + patch( + "dynamo.trtllm.request_handlers.video_diffusion.video_handler.encode_to_mp4_bytes", + return_value=b"fake_mp4", + ), + patch( + "dynamo.trtllm.request_handlers.video_diffusion.video_handler.upload_to_fs", + return_value="https://cdn.example.com/media/videos/test.mp4", + ) as mock_upload, + ): results = [] async for result in handler.generate(request, MagicMock()): results.append(result) diff --git a/components/src/dynamo/trtllm/workers/llm_worker.py b/components/src/dynamo/trtllm/workers/llm_worker.py index d6624746fc48..67a4c4c46fb7 100644 --- a/components/src/dynamo/trtllm/workers/llm_worker.py +++ b/components/src/dynamo/trtllm/workers/llm_worker.py @@ -223,9 +223,9 @@ async def init_llm_worker( arg_map["kv_cache_config"] = kv_config_dict elif isinstance(current_kv_config, dict): # Add event_buffer_max_size while preserving cache_transceiver_config and other YAML settings - current_kv_config[ - "event_buffer_max_size" - ] = DEFAULT_KV_EVENT_BUFFER_MAX_SIZE + current_kv_config["event_buffer_max_size"] = ( + DEFAULT_KV_EVENT_BUFFER_MAX_SIZE + ) # Only pytorch backend is supported for now to publish events and metrics. if "backend" not in arg_map: diff --git a/components/src/dynamo/vllm/args.py b/components/src/dynamo/vllm/args.py index 8e471b59c7ee..ae670817130a 100644 --- a/components/src/dynamo/vllm/args.py +++ b/components/src/dynamo/vllm/args.py @@ -234,9 +234,15 @@ def update_engine_config_with_dynamo( if _uses_nixl_connector(engine_config): ensure_side_channel_host() + # Set runner default only when the user did not explicitly pass --runner. + # vLLM 0.13+ renamed 'task' to 'runner'; None means the user did not + # specify a value, so we can safely apply the 'generate' default. + # Embedding models require --runner embed, which must not be overridden. + if hasattr(engine_config, "runner") and getattr(engine_config, "runner") is None: + engine_config.runner = "generate" + logger.debug(" engine_args.runner = generate (default)") + defaults = { - # vLLM 0.13+ renamed 'task' to 'runner' - "runner": "generate", # As of vLLM >=0.10.0 the engine unconditionally calls # `sampling_params.update_from_tokenizer(...)`, so we can no longer # skip tokenizer initialisation. Setting this to **False** avoids @@ -258,9 +264,9 @@ def update_engine_config_with_dynamo( if envs.is_set("DYN_FORWARDPASS_METRIC_PORT"): existing_cls = getattr(engine_config, "scheduler_cls", None) if existing_cls is None: - defaults[ - "scheduler_cls" - ] = "dynamo.vllm.instrumented_scheduler.InstrumentedScheduler" + defaults["scheduler_cls"] = ( + "dynamo.vllm.instrumented_scheduler.InstrumentedScheduler" + ) logger.info( "Forward pass metrics enabled: scheduler_cls set to InstrumentedScheduler " f"(port={envs.DYN_FORWARDPASS_METRIC_PORT})" @@ -281,9 +287,9 @@ def update_engine_config_with_dynamo( ) existing_cls = getattr(engine_config, "scheduler_cls", None) if existing_cls is None and not envs.is_set("DYN_FORWARDPASS_METRIC_PORT"): - defaults[ - "scheduler_cls" - ] = "dynamo.vllm.instrumented_scheduler.InstrumentedScheduler" + defaults["scheduler_cls"] = ( + "dynamo.vllm.instrumented_scheduler.InstrumentedScheduler" + ) logger.info("Benchmark mode: auto-enabling InstrumentedScheduler") elif existing_cls is not None and "InstrumentedScheduler" not in str( existing_cls diff --git a/components/src/dynamo/vllm/backend_args.py b/components/src/dynamo/vllm/backend_args.py index 28c9d0eb3034..fb31000e6e87 100644 --- a/components/src/dynamo/vllm/backend_args.py +++ b/components/src/dynamo/vllm/backend_args.py @@ -199,8 +199,7 @@ def add_arguments(self, parser) -> None: default=6, type=int, help=( - "Number of context length sample points for decode sweep " - "(default: 6)." + "Number of context length sample points for decode sweep (default: 6)." ), ) add_argument( @@ -210,7 +209,7 @@ def add_arguments(self, parser) -> None: default=6, type=int, help=( - "Number of batch size sample points per context length " "(default: 6)." + "Number of batch size sample points per context length (default: 6)." ), ) add_argument( diff --git a/components/src/dynamo/vllm/handlers.py b/components/src/dynamo/vllm/handlers.py index 6391cd478a96..0f168632d2f1 100644 --- a/components/src/dynamo/vllm/handlers.py +++ b/components/src/dynamo/vllm/handlers.py @@ -114,7 +114,7 @@ class LoRAInfo: def _compute_mm_uuids( - multi_modal_data: Dict[str, Any] | None + multi_modal_data: Dict[str, Any] | None, ) -> Dict[str, list[str]] | None: """ Compute multi_modal_uuids from multi_modal_data. @@ -1655,9 +1655,9 @@ async def _generate_token_mode(self, request, context, request_id): priority=priority, ): if prefill_result is not None and "completion_usage" in tok: - tok["completion_usage"][ - "prompt_tokens_details" - ] = prefill_prompt_tokens_details + tok["completion_usage"]["prompt_tokens_details"] = ( + prefill_prompt_tokens_details + ) yield tok except EngineDeadError as e: logger.error(f"vLLM EngineDeadError: {e}") diff --git a/components/src/dynamo/vllm/instrumented_scheduler.py b/components/src/dynamo/vllm/instrumented_scheduler.py index 07341ac2c54b..15fa5a69499d 100644 --- a/components/src/dynamo/vllm/instrumented_scheduler.py +++ b/components/src/dynamo/vllm/instrumented_scheduler.py @@ -636,7 +636,7 @@ def _bench_inject_fake_decode( ) if new_blocks is None: logger.warning( - "KV exhausted at ctx_len=%d after %d requests, " "truncating batch", + "KV exhausted at ctx_len=%d after %d requests, truncating batch", ctx_len, len(new_reqs_data), ) diff --git a/components/src/dynamo/vllm/multimodal_handlers/encode_worker_handler.py b/components/src/dynamo/vllm/multimodal_handlers/encode_worker_handler.py index f61493cddc65..a56959c3fec9 100644 --- a/components/src/dynamo/vllm/multimodal_handlers/encode_worker_handler.py +++ b/components/src/dynamo/vllm/multimodal_handlers/encode_worker_handler.py @@ -138,9 +138,9 @@ async def generate( logger.debug(f"Received encode request: {{ id: {request.request_id} }}.") request_id = request.request_id - assert ( - request.multimodal_inputs is not None - ), "multimodal_inputs must not be None for encode worker" + assert request.multimodal_inputs is not None, ( + "multimodal_inputs must not be None for encode worker" + ) # The following steps encode the requested image and provided useful embeddings. # 1. Open the image from the provided URL. @@ -184,10 +184,11 @@ async def generate( # keep track of key to avoid recompute of it need_encode_indexes.append((idx, embedding_key)) - with _nvtx.annotate( - "mm:enc:image_load", color="green" - ), time_and_log_code_section( - f"[ENCODE] request: {request_id} image loading" + with ( + _nvtx.annotate("mm:enc:image_load", color="green"), + time_and_log_code_section( + f"[ENCODE] request: {request_id} image loading" + ), ): # Load and generate image tensors image_tasks = [] @@ -221,19 +222,21 @@ async def generate( ) if loaded_images: - with _nvtx.annotate( - "mm:enc:image_preprocess", color="yellow" - ), time_and_log_code_section( - f"[ENCODE] request: {request_id} image processing" + with ( + _nvtx.annotate("mm:enc:image_preprocess", color="yellow"), + time_and_log_code_section( + f"[ENCODE] request: {request_id} image processing" + ), ): image_embeds = await asyncio.to_thread( self.image_processor, images=loaded_images, return_tensors="pt" ) - with _nvtx.annotate( - "mm:enc:vision_encode", color="red" - ), time_and_log_code_section( - f"[ENCODE] request: {request_id} encoding" + with ( + _nvtx.annotate("mm:enc:vision_encode", color="red"), + time_and_log_code_section( + f"[ENCODE] request: {request_id} encoding" + ), ): # Encode the image embeddings using model-specific encoder embeddings = await asyncio.to_thread( diff --git a/components/src/dynamo/vllm/multimodal_utils/chat_processor.py b/components/src/dynamo/vllm/multimodal_utils/chat_processor.py index 20a4157ba85d..7654fa50c990 100644 --- a/components/src/dynamo/vllm/multimodal_utils/chat_processor.py +++ b/components/src/dynamo/vllm/multimodal_utils/chat_processor.py @@ -278,9 +278,9 @@ async def stream_response( if content: # Extract only the new part from the full content new_content = content[num_output_text_so_far:] - full_response["choices"][0]["message"][ - "content" - ] += new_content + full_response["choices"][0]["message"]["content"] += ( + new_content + ) num_output_text_so_far = len(content) # Update finish reason if present diff --git a/components/src/dynamo/vllm/multimodal_utils/model.py b/components/src/dynamo/vllm/multimodal_utils/model.py index 186cc6e390ca..0f77fd323793 100644 --- a/components/src/dynamo/vllm/multimodal_utils/model.py +++ b/components/src/dynamo/vllm/multimodal_utils/model.py @@ -180,9 +180,7 @@ def load_vision_model(model_id: str, enforce_eager: bool = False) -> torch.nn.Mo mm_encoder_only=True, enable_prefix_caching=False, ) - return ( - vllm_model.llm_engine.engine_core.engine_core.model_executor.driver_worker.worker.model_runner.model.visual - ) + return vllm_model.llm_engine.engine_core.engine_core.model_executor.driver_worker.worker.model_runner.model.visual return AutoModel.from_pretrained( model_id, device_map="auto", torch_dtype=torch.float16, trust_remote_code=True ) diff --git a/components/src/dynamo/vllm/omni/audio_handler.py b/components/src/dynamo/vllm/omni/audio_handler.py index eccef6d51bb7..150818a2ef10 100644 --- a/components/src/dynamo/vllm/omni/audio_handler.py +++ b/components/src/dynamo/vllm/omni/audio_handler.py @@ -307,8 +307,7 @@ def _validate_tts_request(self, req: NvCreateAudioSpeechRequest) -> None: ) if req.max_new_tokens > self.config.tts_max_new_tokens_max: raise ValueError( - f"max_new_tokens cannot exceed " - f"{self.config.tts_max_new_tokens_max}" + f"max_new_tokens cannot exceed {self.config.tts_max_new_tokens_max}" ) async def _resolve_ref_audio(self, ref_audio_str: str) -> tuple: diff --git a/components/src/dynamo/vllm/publisher.py b/components/src/dynamo/vllm/publisher.py index fb4c0af08188..38accf5b8456 100644 --- a/components/src/dynamo/vllm/publisher.py +++ b/components/src/dynamo/vllm/publisher.py @@ -96,9 +96,9 @@ def __init__( def create_stat_logger(self, dp_rank: int) -> StatLoggerBase: # component_gauges must be set by setup_vllm_engine() before vLLM # calls create_stat_logger() during engine initialization. - assert ( - self.component_gauges is not None - ), "component_gauges must be set before creating stat loggers" + assert self.component_gauges is not None, ( + "component_gauges must be set before creating stat loggers" + ) logger = DynamoStatLoggerPublisher( endpoint=self.endpoint, dp_rank=dp_rank, diff --git a/components/src/dynamo/vllm/tests/multimodal_utils/test_vllm_model.py b/components/src/dynamo/vllm/tests/multimodal_utils/test_vllm_model.py index e45a7c6f3fc6..1380b60b8768 100644 --- a/components/src/dynamo/vllm/tests/multimodal_utils/test_vllm_model.py +++ b/components/src/dynamo/vllm/tests/multimodal_utils/test_vllm_model.py @@ -3,7 +3,6 @@ """Unit tests for DynamoMultimodalEmbeddingCacheConnector.""" - import pytest import torch diff --git a/components/src/dynamo/vllm/tests/test_backend_args.py b/components/src/dynamo/vllm/tests/test_backend_args.py index 41c54e4d3122..120705dc2f46 100644 --- a/components/src/dynamo/vllm/tests/test_backend_args.py +++ b/components/src/dynamo/vllm/tests/test_backend_args.py @@ -8,7 +8,6 @@ need to add more tests to cover different code paths of DynamoVllmConfig. """ - import pytest from dynamo.vllm.backend_args import DisaggregationMode, DynamoVllmConfig diff --git a/components/src/dynamo/vllm/tests/test_vllm_kv_events_api.py b/components/src/dynamo/vllm/tests/test_vllm_kv_events_api.py index d434a7915a84..45b99da5e5df 100755 --- a/components/src/dynamo/vllm/tests/test_vllm_kv_events_api.py +++ b/components/src/dynamo/vllm/tests/test_vllm_kv_events_api.py @@ -174,9 +174,9 @@ def test_block_stored_serialization_format(self): # Should be an array with tag as first element assert isinstance(decoded, list), f"Expected list, got {type(decoded)}" - assert ( - decoded[0] == "BlockStored" - ), f"Expected tag 'BlockStored', got {decoded[0]}" + assert decoded[0] == "BlockStored", ( + f"Expected tag 'BlockStored', got {decoded[0]}" + ) # Verify field count (tag + 8 fields = 9 elements) assert len(decoded) == 9, ( diff --git a/components/src/dynamo/vllm/tests/test_vllm_renderer_api.py b/components/src/dynamo/vllm/tests/test_vllm_renderer_api.py index 95984cda6eee..16309c383a3b 100755 --- a/components/src/dynamo/vllm/tests/test_vllm_renderer_api.py +++ b/components/src/dynamo/vllm/tests/test_vllm_renderer_api.py @@ -283,9 +283,9 @@ def test_input_processor_attributes(self): "update EngineFactory in " "components/src/dynamo/frontend/vllm_processor.py" ) - assert callable( - getattr(InputProcessor, "get_tokenizer") - ), "InputProcessor.get_tokenizer is not callable" + assert callable(getattr(InputProcessor, "get_tokenizer")), ( + "InputProcessor.get_tokenizer is not callable" + ) get_tok_sig = inspect.signature(InputProcessor.get_tokenizer) assert list(get_tok_sig.parameters) == ["self"], ( "InputProcessor.get_tokenizer signature changed; " diff --git a/components/src/dynamo/vllm/tests/test_vllm_unit.py b/components/src/dynamo/vllm/tests/test_vllm_unit.py index c5401a2247aa..575489a4dd21 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_unit.py +++ b/components/src/dynamo/vllm/tests/test_vllm_unit.py @@ -22,6 +22,7 @@ ensure_side_channel_host, get_host_ip, parse_args, + update_engine_config_with_dynamo, ) from dynamo.vllm.constants import DisaggregationMode from dynamo.vllm.tests.conftest import make_cli_args_fixture @@ -721,3 +722,94 @@ def test_decode_grid_skips_large_ctx(self): total_kv = 100 * 16 for ctx_len, bs in points: assert ctx_len <= total_kv + + +# --------------------------------------------------------------------------- +# update_engine_config_with_dynamo: runner default override tests (issue #7670) +# --------------------------------------------------------------------------- + + +def _make_dynamo_config_stub(): + """Minimal dynamo config stub for update_engine_config_with_dynamo tests.""" + from dynamo.vllm.constants import DisaggregationMode + + stub = SimpleNamespace( + disaggregation_mode=DisaggregationMode.AGGREGATED, + multimodal_worker=False, + multimodal_decode_worker=False, + benchmark_mode=None, + use_kv_events=False, + connector=[], + ) + return stub + + +def _make_engine_config_stub(runner=None): + """Minimal engine config stub with the fields touched by update_engine_config_with_dynamo.""" + stub = SimpleNamespace( + runner=runner, + enable_prefix_caching=None, + skip_tokenizer_init=None, + enable_log_requests=None, + disable_log_stats=None, + kv_events_config=None, + kv_transfer_config=None, + block_size=None, + scheduler_cls=None, + load_format=None, + ) + return stub + + +class TestRunnerDefaultNotOverridden: + """Tests that update_engine_config_with_dynamo does not clobber user-specified --runner.""" + + def test_runner_defaults_to_generate_when_not_specified(self): + """When runner is None (not user-specified), it should be set to 'generate'.""" + dynamo_cfg = _make_dynamo_config_stub() + engine_cfg = _make_engine_config_stub(runner=None) + + with ( + patch("dynamo.vllm.args.create_kv_events_config", return_value=None), + patch("dynamo.vllm.args._uses_nixl_connector", return_value=False), + patch("dynamo.vllm.args.envs") as mock_envs, + ): + mock_envs.is_set.return_value = False + update_engine_config_with_dynamo(dynamo_cfg, engine_cfg) + + assert engine_cfg.runner == "generate" + + def test_runner_embed_is_preserved(self): + """When runner='embed' (user-specified), it must NOT be overwritten with 'generate'.""" + dynamo_cfg = _make_dynamo_config_stub() + engine_cfg = _make_engine_config_stub(runner="embed") + + with ( + patch("dynamo.vllm.args.create_kv_events_config", return_value=None), + patch("dynamo.vllm.args._uses_nixl_connector", return_value=False), + patch("dynamo.vllm.args.envs") as mock_envs, + ): + mock_envs.is_set.return_value = False + update_engine_config_with_dynamo(dynamo_cfg, engine_cfg) + + assert engine_cfg.runner == "embed", ( + f"Expected runner='embed' to be preserved, but got runner='{engine_cfg.runner}'. " + "update_engine_config_with_dynamo is unconditionally overriding --runner." + ) + + def test_runner_pooling_is_preserved(self): + """When runner='pooling' (user-specified), it must NOT be overwritten.""" + dynamo_cfg = _make_dynamo_config_stub() + engine_cfg = _make_engine_config_stub(runner="pooling") + + with ( + patch("dynamo.vllm.args.create_kv_events_config", return_value=None), + patch("dynamo.vllm.args._uses_nixl_connector", return_value=False), + patch("dynamo.vllm.args.envs") as mock_envs, + ): + mock_envs.is_set.return_value = False + update_engine_config_with_dynamo(dynamo_cfg, engine_cfg) + + assert engine_cfg.runner == "pooling", ( + f"Expected runner='pooling' to be preserved, but got runner='{engine_cfg.runner}'." + ) diff --git a/components/src/dynamo/vllm/tests/test_vllm_worker_handler.py b/components/src/dynamo/vllm/tests/test_vllm_worker_handler.py index 55e141856be9..3b1eea1d109b 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_worker_handler.py +++ b/components/src/dynamo/vllm/tests/test_vllm_worker_handler.py @@ -58,9 +58,7 @@ def _make_config( config.multimodal_embedding_cache_capacity_gb = ( multimodal_embedding_cache_capacity_gb ) - config.engine_args.create_model_config.return_value.get_diff_sampling_param.return_value = ( - {} - ) + config.engine_args.create_model_config.return_value.get_diff_sampling_param.return_value = {} return config diff --git a/components/src/dynamo/vllm/worker_factory.py b/components/src/dynamo/vllm/worker_factory.py index 6619a550a913..74a484f6d155 100644 --- a/components/src/dynamo/vllm/worker_factory.py +++ b/components/src/dynamo/vllm/worker_factory.py @@ -67,7 +67,7 @@ async def _wait_and_load_benchmark(bench_cfg: dict, vllm_config: VllmConfig) -> while not p.exists(): if _time.monotonic() > deadline: raise TimeoutError( - f"Benchmark did not complete within {timeout}s. " f"Missing: {p}" + f"Benchmark did not complete within {timeout}s. Missing: {p}" ) await asyncio.sleep(0.1) @@ -167,7 +167,8 @@ async def _create_multimodal_encode_worker( shutdown_endpoints[:] = [generate_endpoint] handler = EncodeWorkerHandler( - config.engine_args, config.embedding_transfer_mode # type: ignore[arg-type] + config.engine_args, + config.embedding_transfer_mode, # type: ignore[arg-type] ) await handler.async_init(runtime) logger.info("Starting to serve the encode worker endpoint...") diff --git a/deploy/operator/api/scripts/generate_pydantic_from_go.py b/deploy/operator/api/scripts/generate_pydantic_from_go.py index 5981a0d8efa1..4d8f6508671e 100755 --- a/deploy/operator/api/scripts/generate_pydantic_from_go.py +++ b/deploy/operator/api/scripts/generate_pydantic_from_go.py @@ -587,7 +587,7 @@ def generate_pydantic(self) -> str: comment_escaped = go_field.comment.replace('"', '\\"') field_args.append(f'description="{comment_escaped}"') - field_def += f' = Field({", ".join(field_args)})' + field_def += f" = Field({', '.join(field_args)})" lines.append(field_def) diff --git a/deploy/operator/docs/fix-api-anchors.py b/deploy/operator/docs/fix-api-anchors.py index 76e888b93f9a..d625aba93b07 100644 --- a/deploy/operator/docs/fix-api-anchors.py +++ b/deploy/operator/docs/fix-api-anchors.py @@ -21,6 +21,7 @@ v1alpha1 section. This script prepends "v1beta1 " to the affected headings in the v1beta1 section and updates all intra-section links to match the new anchors. """ + import re import sys diff --git a/deploy/sanity_check.py b/deploy/sanity_check.py index c2cd18cbea69..5d1d6ea02dd3 100755 --- a/deploy/sanity_check.py +++ b/deploy/sanity_check.py @@ -1847,7 +1847,7 @@ def _add_model_details(self, models: List[tuple]): for i, model_info in enumerate(models): model_name, download_date, size_str = model_info model_node = NodeInfo( - label=f"Model {i+1}", + label=f"Model {i + 1}", desc=f"{model_name}, downloaded={download_date}, size={size_str}", status=NodeStatus.INFO, ) diff --git a/deploy/utils/dynamo_deployment.py b/deploy/utils/dynamo_deployment.py index 09586cfec8f2..c69bfc368210 100644 --- a/deploy/utils/dynamo_deployment.py +++ b/deploy/utils/dynamo_deployment.py @@ -46,7 +46,7 @@ def find_available_port(start_port: int = 8000) -> int: except OSError: continue raise RuntimeError( - f"No available ports found in range {start_port}-{start_port+99}" + f"No available ports found in range {start_port}-{start_port + 99}" ) @@ -120,9 +120,9 @@ def __init__( self.model_name = model_name self.service_name = service_name or f"{self.deployment_name}-frontend" self.components: List[str] = [] # Will store component names from CR - self.deployment_spec: Optional[ - Dict[str, Any] - ] = None # Will store the full deployment spec + self.deployment_spec: Optional[Dict[str, Any]] = ( + None # Will store the full deployment spec + ) self.base_log_dir = Path(base_log_dir) if base_log_dir else Path("logs") self.frontend_port = frontend_port self.port_forward_process: Optional[subprocess.Popen[bytes]] = None @@ -236,9 +236,9 @@ async def create_deployment(self, deployment: Union[dict, str]): self.deployment_spec = deployment # Ensure deployment_spec is properly loaded - assert ( - self.deployment_spec is not None - ), "Failed to load deployment specification" + assert self.deployment_spec is not None, ( + "Failed to load deployment specification" + ) # Extract component names (original case for label queries, lowercase for directories) self._original_components = list( @@ -356,7 +356,7 @@ async def wait_for_deployment_ready( # Show first 2 components, abbreviate if more components_str = ", ".join(not_ready_components[:2]) if len(not_ready_components) > 2: - components_str += f" +{len(not_ready_components)-2} more" + components_str += f" +{len(not_ready_components) - 2} more" status_str = f"Waiting for: {components_str}" else: status_str = f"State: {current_state}" diff --git a/examples/backends/sglang/test_sglang_expert_info.py b/examples/backends/sglang/test_sglang_expert_info.py index 2877860fcd0a..30a3ae9dc6d7 100644 --- a/examples/backends/sglang/test_sglang_expert_info.py +++ b/examples/backends/sglang/test_sglang_expert_info.py @@ -113,9 +113,9 @@ def start_sglang_backend(): def validate_routed_experts(routed_experts): """Check that routed_experts is a base64-encoded string of int32 expert IDs.""" - assert isinstance( - routed_experts, str - ), f"Expected base64 string, got {type(routed_experts)}" + assert isinstance(routed_experts, str), ( + f"Expected base64 string, got {type(routed_experts)}" + ) decoded = np.frombuffer( pybase64.b64decode(routed_experts.encode("utf-8")), dtype=np.int32 ) @@ -145,9 +145,9 @@ def test_completions_non_streaming(): assert len(data["choices"]) > 0 nvext = data.get("nvext", {}) - assert ( - "routed_experts" in nvext - ), f"Expected routed_experts in nvext, got keys: {list(nvext.keys())}" + assert "routed_experts" in nvext, ( + f"Expected routed_experts in nvext, got keys: {list(nvext.keys())}" + ) validate_routed_experts(nvext["routed_experts"]) print(f" routed_experts shape: {len(nvext['routed_experts'])} layers") print(" PASSED") diff --git a/examples/backends/sglang/test_sglang_profile.py b/examples/backends/sglang/test_sglang_profile.py index 75ce367ee9b0..b5168b35109f 100644 --- a/examples/backends/sglang/test_sglang_profile.py +++ b/examples/backends/sglang/test_sglang_profile.py @@ -184,12 +184,12 @@ def test_profiling_endpoints(): inference_url, json={ "model": MODEL, - "prompt": f"Hello, this is test request {i+1}. ", + "prompt": f"Hello, this is test request {i + 1}. ", "max_tokens": 10, "temperature": 0.8, }, ) - print(f" Request {i+1}: {response.status_code}") + print(f" Request {i + 1}: {response.status_code}") if response.status_code != 200: print(f" Response: {response.text[:200]}") time.sleep(0.5) diff --git a/examples/multimodal/components/worker.py b/examples/multimodal/components/worker.py index 1837f6d43615..30b3253729fc 100644 --- a/examples/multimodal/components/worker.py +++ b/examples/multimodal/components/worker.py @@ -334,9 +334,9 @@ async def generate(self, request: vLLMMultimodalRequest): # Update the prompt token id in the decode request to the one # in response, which has image templated filled in. So that # the decode worker will fetch correct amount of KV blocks. - decode_request.engine_prompt[ - "prompt_token_ids" - ] = prefill_response.prompt_token_ids + decode_request.engine_prompt["prompt_token_ids"] = ( + prefill_response.prompt_token_ids + ) logger.debug( f"Prefill response kv_transfer_params: {prefill_response.kv_transfer_params}" ) diff --git a/examples/multimodal/utils/args.py b/examples/multimodal/utils/args.py index 5e6a5ad96f8f..6c3bd2eadd34 100644 --- a/examples/multimodal/utils/args.py +++ b/examples/multimodal/utils/args.py @@ -80,9 +80,9 @@ def base_parse_args( config = Config() config.model = args.model if args.served_model_name: - assert ( - len(args.served_model_name) <= 1 - ), "We do not support multiple model names." + assert len(args.served_model_name) <= 1, ( + "We do not support multiple model names." + ) config.served_model_name = args.served_model_name[0] else: # This becomes an `Option` on the Rust side diff --git a/examples/multimodal/utils/chat_processor.py b/examples/multimodal/utils/chat_processor.py index 84c644a17f67..250ac6433162 100644 --- a/examples/multimodal/utils/chat_processor.py +++ b/examples/multimodal/utils/chat_processor.py @@ -190,7 +190,9 @@ async def stream_response( if request.stream: # Handle streaming response num_output_text_so_far = 0 - async for raw_response in self.openai_serving.chat_completion_stream_generator( + async for ( + raw_response + ) in self.openai_serving.chat_completion_stream_generator( request, result_generator, request_id, @@ -223,7 +225,9 @@ async def stream_response( # Collect all chunks into a single response full_response = None num_output_text_so_far = 0 - async for raw_response in self.openai_serving.chat_completion_stream_generator( + async for ( + raw_response + ) in self.openai_serving.chat_completion_stream_generator( request, result_generator, request_id, @@ -258,9 +262,9 @@ async def stream_response( if content: # Extract only the new part from the full content new_content = content[num_output_text_so_far:] - full_response["choices"][0]["message"][ - "content" - ] += new_content + full_response["choices"][0]["message"]["content"] += ( + new_content + ) num_output_text_so_far = len(content) # Update finish reason if present diff --git a/fern/convert_callouts.py b/fern/convert_callouts.py index 13fb5b613f6f..0194200a38b3 100755 --- a/fern/convert_callouts.py +++ b/fern/convert_callouts.py @@ -368,9 +368,9 @@ def test(name: str, input_text: str, expected: str): "\nLine one.\nLine two.\n\n\nAfter.\n", ) - print(f"\n{'='*50}") + print(f"\n{'=' * 50}") print(f"Results: {passed} passed, {failed} failed") - print(f"{'='*50}") + print(f"{'=' * 50}") return failed == 0 diff --git a/lib/bindings/kvbm/python/kvbm/_core.pyi b/lib/bindings/kvbm/python/kvbm/_core.pyi index d9e0e803e553..d4464deb17c1 100644 --- a/lib/bindings/kvbm/python/kvbm/_core.pyi +++ b/lib/bindings/kvbm/python/kvbm/_core.pyi @@ -10,7 +10,13 @@ class Layer: ... - def __dlpack__(self, stream: Optional[Any] = None, max_version: Optional[Any] = None, dl_device: Optional[Any] = None, copy: Optional[bool] = None) -> Any: + def __dlpack__( + self, + stream: Optional[Any] = None, + max_version: Optional[Any] = None, + dl_device: Optional[Any] = None, + copy: Optional[bool] = None, + ) -> Any: """ Get a dlpack capsule of the layer """ @@ -41,7 +47,7 @@ class Block: """ ... - def __iter__(self) -> 'Block': + def __iter__(self) -> "Block": """ Get an iterator over the layers """ @@ -59,7 +65,13 @@ class Block: """ ... - def __dlpack__(self, stream: Optional[Any] = None, max_version: Optional[Any] = None, dl_device: Optional[Any] = None, copy: Optional[bool] = None) -> Any: + def __dlpack__( + self, + stream: Optional[Any] = None, + max_version: Optional[Any] = None, + dl_device: Optional[Any] = None, + copy: Optional[bool] = None, + ) -> Any: """ Get a dlpack capsule of the block Exception raised if the block is not contiguous @@ -91,7 +103,7 @@ class BlockList: """ ... - def __iter__(self) -> 'BlockList': + def __iter__(self) -> "BlockList": """ Get an iterator over the blocks """ @@ -123,7 +135,7 @@ class BlockManager: dtype: Optional[str] = None, host_num_blocks: Optional[int] = None, device_num_blocks: Optional[int] = None, - device_id: int = 0 + device_id: int = 0, ) -> None: """ Create a `BlockManager` object @@ -218,5 +230,4 @@ class KvbmRequest: A request for KV cache """ - def __init__(self, request_id: int, tokens: List[int], block_size: int) -> None: - ... + def __init__(self, request_id: int, tokens: List[int], block_size: int) -> None: ... diff --git a/lib/bindings/kvbm/python/kvbm/trtllm_integration/connector/kvbm_connector_worker.py b/lib/bindings/kvbm/python/kvbm/trtllm_integration/connector/kvbm_connector_worker.py index 55ca5f84bd51..9bcb4467f88b 100644 --- a/lib/bindings/kvbm/python/kvbm/trtllm_integration/connector/kvbm_connector_worker.py +++ b/lib/bindings/kvbm/python/kvbm/trtllm_integration/connector/kvbm_connector_worker.py @@ -17,12 +17,12 @@ class DynamoKVBMConnectorWorker(KvCacheConnectorWorker): def _callable_object(self) -> callable: - assert ( - self._connector is not None - ), "Expected cache connector worker to have non-None _connector obj" - assert ( - self.event is not None - ), "Expected cache connector worker to have non-None event obj" + assert self._connector is not None, ( + "Expected cache connector worker to have non-None _connector obj" + ) + assert self.event is not None, ( + "Expected cache connector worker to have non-None event obj" + ) def callback(): self.event.record() diff --git a/lib/bindings/kvbm/python/kvbm/vllm_integration/connector_leader.py b/lib/bindings/kvbm/python/kvbm/vllm_integration/connector_leader.py index e381ba725557..b762b552d474 100644 --- a/lib/bindings/kvbm/python/kvbm/vllm_integration/connector_leader.py +++ b/lib/bindings/kvbm/python/kvbm/vllm_integration/connector_leader.py @@ -172,12 +172,12 @@ def build_connector_meta(self, scheduler_output: SchedulerOutput) -> bytes: # If A == B, and A == C, then B == C cached_reqs = scheduler_output.scheduled_cached_reqs - assert len(cached_reqs.req_ids) == len( - cached_reqs.new_block_ids - ), "Number of cached req_ids doesn't match the number of cached new_block_ids" - assert len(cached_reqs.req_ids) == len( - cached_reqs.num_computed_tokens - ), "Number of cached req_ids doesn't match the number of cached num_computed_tokens" + assert len(cached_reqs.req_ids) == len(cached_reqs.new_block_ids), ( + "Number of cached req_ids doesn't match the number of cached new_block_ids" + ) + assert len(cached_reqs.req_ids) == len(cached_reqs.num_computed_tokens), ( + "Number of cached req_ids doesn't match the number of cached num_computed_tokens" + ) # In https://github.com/vllm-project/vllm/pull/26388/changes#diff-9eeca590fd99f15621897e559dba39b3ec4e7c2c65ec3c3229711689e008b5f4L732-L736, # new_token_ids was changed to return an empty list unless pipeline diff --git a/lib/bindings/python/src/dynamo/_core.pyi b/lib/bindings/python/src/dynamo/_core.pyi index be31090e3bba..c8d87bb30f87 100644 --- a/lib/bindings/python/src/dynamo/_core.pyi +++ b/lib/bindings/python/src/dynamo/_core.pyi @@ -125,7 +125,6 @@ class DistributedRuntime: """ ... - class Endpoint: """ An Endpoint is a single API endpoint @@ -133,7 +132,13 @@ class Endpoint: ... - async def serve_endpoint(self, handler: RequestHandler, graceful_shutdown: bool = True, metrics_labels: Optional[List[Tuple[str, str]]] = None, health_check_payload: Optional[Dict[str, Any]] = None) -> None: + async def serve_endpoint( + self, + handler: RequestHandler, + graceful_shutdown: bool = True, + metrics_labels: Optional[List[Tuple[str, str]]] = None, + health_check_payload: Optional[Dict[str, Any]] = None, + ) -> None: """ Serve an endpoint discoverable by all connected clients at `{{ namespace }}/components/{{ component_name }}/endpoints/{{ endpoint_name }}` @@ -217,51 +222,50 @@ class Client: ... async def random( - self, - request: JsonLike, - annotated: bool | None = True, - context: Context | None = None, - ) -> AsyncIterator[JsonLike]: + self, + request: JsonLike, + annotated: bool | None = True, + context: Context | None = None, + ) -> AsyncIterator[JsonLike]: """ Pick a random instance of the endpoint and issue the request """ ... async def round_robin( - self, - request: JsonLike, - annotated: bool | None = True, - context: Context | None = None, - ) -> AsyncIterator[JsonLike]: + self, + request: JsonLike, + annotated: bool | None = True, + context: Context | None = None, + ) -> AsyncIterator[JsonLike]: """ Pick the next instance of the endpoint in a round-robin fashion """ ... async def direct( - self, - request: JsonLike, - instance_id: int, - annotated: bool | None = True, - context: Context | None = None, - ) -> AsyncIterator[JsonLike]: + self, + request: JsonLike, + instance_id: int, + annotated: bool | None = True, + context: Context | None = None, + ) -> AsyncIterator[JsonLike]: """ Pick a specific instance of the endpoint """ ... async def generate( - self, - request: JsonLike, - annotated: bool | None = True, - context: Context | None = None, - ) -> AsyncIterator[JsonLike]: + self, + request: JsonLike, + annotated: bool | None = True, + context: Context | None = None, + ) -> AsyncIterator[JsonLike]: """ Generate a response from the endpoint """ ... - class ModelCardInstanceId: """ Unique identifier for a worker instance: namespace, component, endpoint and instance_id. @@ -273,7 +277,6 @@ class ModelCardInstanceId: """ ... - def compute_block_hash_for_seq( tokens: List[int], kv_block_size: int, @@ -490,7 +493,6 @@ class ModelRuntimeConfig: bootstrap_port: int | None def __init__(self) -> None: ... - def set_engine_specific(self, key: str, value: Any) -> None: """Set an engine-specific runtime configuration value""" ... @@ -500,10 +502,10 @@ class ModelRuntimeConfig: ... def set_disaggregated_endpoint( - self, - bootstrap_host: str | None = None, - bootstrap_port: int | None = None, - ) -> None: + self, + bootstrap_host: str | None = None, + bootstrap_port: int | None = None, + ) -> None: """Set the disaggregated endpoint for the model""" ... @@ -640,7 +642,10 @@ class KvIndexer: ... def find_matches_for_request( - self, token_ids: List[int], lora_name: Optional[str] = None, is_eagle: Optional[bool] = None + self, + token_ids: List[int], + lora_name: Optional[str] = None, + is_eagle: Optional[bool] = None, ) -> OverlapScores: """ Return the overlapping scores of workers for the given token ids. @@ -688,7 +693,10 @@ class ApproxKvIndexer: ... def find_matches_for_request( - self, token_ids: List[int], lora_name: Optional[str] = None, is_eagle: Optional[bool] = None + self, + token_ids: List[int], + lora_name: Optional[str] = None, + is_eagle: Optional[bool] = None, ) -> OverlapScores: """ Return the overlapping scores of workers for the given token ids. @@ -727,7 +735,6 @@ class ApproxKvIndexer: """ ... - class KvEventPublisher: """ A KV event publisher will publish KV events corresponding to the component. @@ -809,7 +816,6 @@ class KvEventPublisher: """ ... - class FpmEventRelay: """ Relay that bridges ForwardPassMetrics from a local raw ZMQ PUB socket @@ -836,7 +842,6 @@ class FpmEventRelay: """Shut down the relay task.""" ... - class FpmEventSubscriber: """ Subscriber for ForwardPassMetrics from the Dynamo event plane. @@ -912,7 +917,6 @@ class FpmEventSubscriber: """Shut down the subscriber (all background tasks).""" ... - class HttpService: """ A HTTP service for dynamo applications. @@ -952,8 +956,6 @@ class PythonAsyncEngine: """Wrap a Python generator and event loop for use with Dynamo services.""" ... - - class HttpAsyncEngine: """ An async engine for a distributed Dynamo http service. This is an extension of the @@ -1099,13 +1101,14 @@ class KserveGrpcService: class ModelInput: """What type of request this model needs: Text, Tokens or Tensor""" + Text: ModelInput Tokens: ModelInput Tensor: ModelInput - class ModelType: """What type of request this model needs: Chat, Completions, Embedding, Tensor, Images, Videos or Prefill""" + Chat: ModelType Completions: ModelType Embedding: ModelType @@ -1115,15 +1118,14 @@ class ModelType: Audios: ModelType Videos: ModelType - def __or__(self, other: ModelType) -> ModelType: - ... - + def __or__(self, other: ModelType) -> ModelType: ... def supports_chat(self) -> bool: """Return True if this model type supports chat.""" ... class RouterMode: """Router mode for load balancing requests across workers""" + RoundRobin: "RouterMode" Random: "RouterMode" PowerOfTwoChoices: "RouterMode" @@ -1134,6 +1136,7 @@ class RouterMode: class RouterConfig: """How to route the request""" + router_mode: RouterMode kv_router_config: KvRouterConfig @@ -1218,19 +1221,13 @@ class KvRouterConfig: ... @staticmethod - def from_json(config_json: str) -> "KvRouterConfig": - ... - + def from_json(config_json: str) -> "KvRouterConfig": ... def dump_json(self) -> str: ... - def copy(self) -> "KvRouterConfig": ... - @property def overlap_score_weight(self) -> float: ... - @overlap_score_weight.setter def overlap_score_weight(self, value: float) -> None: ... - def with_overrides( self, overlap_score_weight: Optional[float] = None, @@ -1242,8 +1239,7 @@ class ReasoningConfig: start_thinking_token_id: int, end_thinking_token_id: int, thinking_ratio: float, - ) -> None: - ... + ) -> None: ... class SglangArgs: def __init__( @@ -1254,8 +1250,7 @@ class SglangArgs: chunked_prefill_size: Optional[int] = None, clip_max_new_tokens: Optional[int] = None, schedule_conservativeness: Optional[float] = None, - ) -> None: - ... + ) -> None: ... class MockEngineArgs: def __init__( @@ -1288,87 +1283,57 @@ class MockEngineArgs: preemption_mode: str = "lifo", router_queue_policy: Optional[str] = None, sglang: Optional[SglangArgs] = None, - ) -> None: - ... - + ) -> None: ... @staticmethod - def from_json(config_json: str) -> "MockEngineArgs": - ... - + def from_json(config_json: str) -> "MockEngineArgs": ... def copy(self) -> "MockEngineArgs": ... - def dump_json(self) -> str: ... - @property def block_size(self) -> int: ... - @property def num_gpu_blocks(self) -> int: ... - @num_gpu_blocks.setter def num_gpu_blocks(self, value: int) -> None: ... - @property def max_num_seqs(self) -> Optional[int]: ... - @property def max_num_batched_tokens(self) -> Optional[int]: ... - @property def enable_prefix_caching(self) -> bool: ... - @enable_prefix_caching.setter def enable_prefix_caching(self, value: bool) -> None: ... - @property def enable_local_indexer(self) -> bool: ... - @property def dp_size(self) -> int: ... - @property def bootstrap_port(self) -> Optional[int]: ... - @property def aic_backend(self) -> Optional[str]: ... - @aic_backend.setter def aic_backend(self, value: Optional[str]) -> None: ... - @property def aic_system(self) -> Optional[str]: ... - @aic_system.setter def aic_system(self, value: Optional[str]) -> None: ... - @property def aic_backend_version(self) -> Optional[str]: ... - @aic_backend_version.setter def aic_backend_version(self, value: Optional[str]) -> None: ... - @property def aic_tp_size(self) -> Optional[int]: ... - @aic_tp_size.setter def aic_tp_size(self, value: Optional[int]) -> None: ... - @property def aic_model_path(self) -> Optional[str]: ... - @aic_model_path.setter def aic_model_path(self, value: Optional[str]) -> None: ... - @property def worker_type(self) -> str: ... - @worker_type.setter def worker_type(self, value: str) -> None: ... - def is_prefill(self) -> bool: ... - def is_decode(self) -> bool: ... - def with_overrides( self, bootstrap_port: Optional[int] = None, @@ -1439,18 +1404,15 @@ class LoRADownloader: def get_cache_path(self, cache_key: str) -> str: ... def is_cached(self, cache_key: str) -> bool: ... def validate_cached(self, cache_key: str) -> bool: ... - @staticmethod def uri_to_cache_key(uri: str) -> str: ... - class MediaDecoder: """Media decoder for image and video preprocessing.""" def __init__(self) -> None: ... def enable_image(self, decoder_options: Dict[str, Any]) -> None: ... - class MediaFetcher: """Media fetcher for loading remote image/video URLs.""" @@ -1476,13 +1438,18 @@ unregister_llm = unregister_model class EngineConfig: """Holds internal configuration for a Dynamo engine.""" + ... -async def make_engine(distributed_runtime: DistributedRuntime, args: EntrypointArgs) -> EngineConfig: +async def make_engine( + distributed_runtime: DistributedRuntime, args: EntrypointArgs +) -> EngineConfig: """Make an engine matching the args""" ... -async def run_input(runtime: DistributedRuntime, input: str, engine_config: EngineConfig) -> None: +async def run_input( + runtime: DistributedRuntime, input: str, engine_config: EngineConfig +) -> None: """Start an engine, connect it to an input, and run until stopped.""" ... @@ -1534,7 +1501,13 @@ class Layer: ... - def __dlpack__(self, stream: Optional[Any] = None, max_version: Optional[Any] = None, dl_device: Optional[Any] = None, copy: Optional[bool] = None) -> Any: + def __dlpack__( + self, + stream: Optional[Any] = None, + max_version: Optional[Any] = None, + dl_device: Optional[Any] = None, + copy: Optional[bool] = None, + ) -> Any: """ Get a dlpack capsule of the layer """ @@ -1565,7 +1538,7 @@ class Block: """ ... - def __iter__(self) -> 'Block': + def __iter__(self) -> "Block": """ Get an iterator over the layers """ @@ -1583,7 +1556,13 @@ class Block: """ ... - def __dlpack__(self, stream: Optional[Any] = None, max_version: Optional[Any] = None, dl_device: Optional[Any] = None, copy: Optional[bool] = None) -> Any: + def __dlpack__( + self, + stream: Optional[Any] = None, + max_version: Optional[Any] = None, + dl_device: Optional[Any] = None, + copy: Optional[bool] = None, + ) -> Any: """ Get a dlpack capsule of the block Exception raised if the block is not contiguous @@ -1615,7 +1594,7 @@ class BlockList: """ ... - def __iter__(self) -> 'BlockList': + def __iter__(self) -> "BlockList": """ Get an iterator over the blocks """ @@ -1647,7 +1626,7 @@ class BlockManager: dtype: Optional[str] = None, host_num_blocks: Optional[int] = None, device_num_blocks: Optional[int] = None, - device_id: int = 0 + device_id: int = 0, ) -> None: """ Create a `BlockManager` object @@ -1742,8 +1721,7 @@ class KvbmRequest: A request for KV cache """ - def __init__(self, request_id: int, tokens: List[int], block_size: int) -> None: - ... + def __init__(self, request_id: int, tokens: List[int], block_size: int) -> None: ... class KvRouter: """ @@ -1940,6 +1918,7 @@ class KvRouter: class EngineType: """Engine type for Dynamo workers""" + Echo: "EngineType" Dynamic: "EngineType" Mocker: "EngineType" @@ -2009,6 +1988,7 @@ class PlannerDecision: -1 in any of those fields mean not set, usually because planner hasn't decided anything yet. Call VirtualConnectorClient.complete(event) when action is completed. """ + num_prefill_workers: int num_decode_workers: int ... @@ -2016,9 +1996,14 @@ class PlannerDecision: class VirtualConnectorCoordinator: """Internal planner virtual connector component""" - def __init__(self, runtime: DistributedRuntime, dynamo_namespace: str, check_interval_secs: int, max_wait_time_secs: int, max_retries: int) -> None: - ... - + def __init__( + self, + runtime: DistributedRuntime, + dynamo_namespace: str, + check_interval_secs: int, + max_wait_time_secs: int, + max_retries: int, + ) -> None: ... async def async_init(self) -> None: """Call this before using the object""" ... @@ -2027,29 +2012,21 @@ class VirtualConnectorCoordinator: """Get the current values. Most for test / debug.""" ... - async def update_scaling_decision(self, num_prefill: Optional[int] = None, num_decode: Optional[int] = None) -> None: - ... - - async def wait_for_scaling_completion(self) -> None: - ... + async def update_scaling_decision( + self, num_prefill: Optional[int] = None, num_decode: Optional[int] = None + ) -> None: ... + async def wait_for_scaling_completion(self) -> None: ... class VirtualConnectorClient: """How a client discovers planner requests and marks them complete""" - def __init__(self, runtime: DistributedRuntime, dynamo_namespace: str) -> None: - ... - - async def get(self) -> PlannerDecision: - ... - - async def complete(self, decision: PlannerDecision) -> None: - ... - + def __init__(self, runtime: DistributedRuntime, dynamo_namespace: str) -> None: ... + async def get(self) -> PlannerDecision: ... + async def complete(self, decision: PlannerDecision) -> None: ... async def wait(self) -> None: """Blocks until there is a new decision to fetch using 'get'""" ... - # ============================================================================= # Dynamo Exception Types # diff --git a/lib/bindings/python/src/dynamo/nixl_connect/__init__.py b/lib/bindings/python/src/dynamo/nixl_connect/__init__.py index 06c8c1b62341..a47f8cf80cde 100644 --- a/lib/bindings/python/src/dynamo/nixl_connect/__init__.py +++ b/lib/bindings/python/src/dynamo/nixl_connect/__init__.py @@ -485,7 +485,11 @@ def status(self) -> OperationStatus: """ # Early return if the operation is already complete, errored, or cancelled. match self._status: - case OperationStatus.COMPLETE | OperationStatus.ERRORED | OperationStatus.CANCELLED: + case ( + OperationStatus.COMPLETE + | OperationStatus.ERRORED + | OperationStatus.CANCELLED + ): return self._status if self._xfer_hndl is None: @@ -1462,7 +1466,11 @@ def status(self) -> OperationStatus: """ # Early return if the operation is already complete, errored, or cancelled. match self._status: - case OperationStatus.COMPLETE | OperationStatus.ERRORED | OperationStatus.CANCELLED: + case ( + OperationStatus.COMPLETE + | OperationStatus.ERRORED + | OperationStatus.CANCELLED + ): return self._status old_status = self._status diff --git a/lib/bindings/python/tests/cancellation/test_cancellation.py b/lib/bindings/python/tests/cancellation/test_cancellation.py index 8f91142531b8..3a8bccbb1e21 100644 --- a/lib/bindings/python/tests/cancellation/test_cancellation.py +++ b/lib/bindings/python/tests/cancellation/test_cancellation.py @@ -30,9 +30,9 @@ async def generate(self, request, context): self.context_is_killed = False method_name = request - assert hasattr( - self, method_name - ), f"Method '{method_name}' not found on {self.__class__.__name__}" + assert hasattr(self, method_name), ( + f"Method '{method_name}' not found on {self.__class__.__name__}" + ) method = getattr(self, method_name) async for response in method(request, context): yield response @@ -65,9 +65,9 @@ async def _generate_until_context_cancelled(self, request, context): print(f"Sending iteration {i}") yield i - assert ( - False - ), "Test failed: generate_until_cancelled did not raise CancelledError" + assert False, ( + "Test failed: generate_until_cancelled did not raise CancelledError" + ) async def _generate_until_asyncio_cancelled(self, request, context): """ @@ -86,9 +86,9 @@ async def _generate_until_asyncio_cancelled(self, request, context): self.context_is_killed = context.is_killed() raise - assert ( - False - ), "Test failed: generate_until_cancelled did not raise CancelledError" + assert False, ( + "Test failed: generate_until_cancelled did not raise CancelledError" + ) async def _generate_and_cancel_context(self, request, context): """ diff --git a/lib/bindings/python/tests/cancellation/test_example.py b/lib/bindings/python/tests/cancellation/test_example.py index a3192f1fbc3f..0b068dd343e0 100644 --- a/lib/bindings/python/tests/cancellation/test_example.py +++ b/lib/bindings/python/tests/cancellation/test_example.py @@ -116,12 +116,12 @@ async def test_direct_connection_cancellation( server_output = stop_process("server_process", server_process) # Assert expected messages - assert ( - "Client: Cancelling after 3 responses..." in client_output - ), f"Client output: {client_output}" - assert ( - "Server: Cancelled at iteration" in server_output - ), f"Server output: {server_output}" + assert "Client: Cancelling after 3 responses..." in client_output, ( + f"Client output: {client_output}" + ) + assert "Server: Cancelled at iteration" in server_output, ( + f"Server output: {server_output}" + ) @pytest.mark.asyncio @@ -141,12 +141,12 @@ async def test_middle_server_cancellation( middle_output = stop_process("middle_server_process", middle_server_process) # Assert expected messages - assert ( - "Client: Cancelling after 3 responses..." in client_output - ), f"Client output: {client_output}" - assert ( - "Middle server: Forwarding response 2" in middle_output - ), f"Middle server output: {middle_output}" - assert ( - "Server: Cancelled at iteration" in server_output - ), f"Server output: {server_output}" + assert "Client: Cancelling after 3 responses..." in client_output, ( + f"Client output: {client_output}" + ) + assert "Middle server: Forwarding response 2" in middle_output, ( + f"Middle server output: {middle_output}" + ) + assert "Server: Cancelled at iteration" in server_output, ( + f"Server output: {server_output}" + ) diff --git a/lib/bindings/python/tests/test_example_hello_world.py b/lib/bindings/python/tests/test_example_hello_world.py index 84a2dd24cdb7..1415de0b700a 100644 --- a/lib/bindings/python/tests/test_example_hello_world.py +++ b/lib/bindings/python/tests/test_example_hello_world.py @@ -85,5 +85,5 @@ async def test_hello_world(example_dir, server_process): expected_lines = ["Hello world!", "Hello sun!", "Hello moon!", "Hello star!"] for expected_line in expected_lines: assert expected_line in lines, ( - f"Expected line '{expected_line}' not found in output.\n" f"Lines: {lines}" + f"Expected line '{expected_line}' not found in output.\nLines: {lines}" ) diff --git a/lib/bindings/python/tests/test_kv_bindings.py b/lib/bindings/python/tests/test_kv_bindings.py index d1df56a104a8..11ea33850c6e 100644 --- a/lib/bindings/python/tests/test_kv_bindings.py +++ b/lib/bindings/python/tests/test_kv_bindings.py @@ -68,16 +68,16 @@ def test_radix_tree_binding(): # Verify the results # Note: scores is now Dict[(worker_id, dp_rank), score] assert overlap_scores.scores is not None - assert ( - len(overlap_scores.scores) == 1 - ), f"Expected 1 worker in scores, got {len(overlap_scores.scores)}" + assert len(overlap_scores.scores) == 1, ( + f"Expected 1 worker in scores, got {len(overlap_scores.scores)}" + ) worker_key = (worker_id, 0) # (worker_id, dp_rank) - assert ( - worker_key in overlap_scores.scores - ), f"Worker {worker_key} not found in scores" - assert ( - overlap_scores.scores[worker_key] == 1 - ), f"Expected score 1 for worker {worker_key}, got {overlap_scores.scores[worker_key]}" + assert worker_key in overlap_scores.scores, ( + f"Worker {worker_key} not found in scores" + ) + assert overlap_scores.scores[worker_key] == 1, ( + f"Expected score 1 for worker {worker_key}, got {overlap_scores.scores[worker_key]}" + ) blocks = radix_tree.dump_tree_as_events() assert len(blocks) == 1, f"Expected 1 block event, got {len(blocks)}" @@ -86,9 +86,9 @@ def test_radix_tree_binding(): # cleanup radix_tree.remove_worker(worker_id) blocks_empty = radix_tree.dump_tree_as_events() - assert ( - len(blocks_empty) == 0 - ), f"Expected 0 block events after removal, got {len(blocks_empty)}" + assert len(blocks_empty) == 0, ( + f"Expected 0 block events after removal, got {len(blocks_empty)}" + ) print( f"✓ RadixTree test passed: worker {worker_key} has score {overlap_scores.scores[worker_key]}" @@ -148,9 +148,9 @@ def worker(worker_id, prepopulate_worker_ids: bool = False): if prepopulate_worker_ids: for i in range(num_threads): worker(i, prepopulate_worker_ids=True) - assert ( - exception_counter == 0 - ), f"Warmup: expected 0 exceptions, got {exception_counter}" + assert exception_counter == 0, ( + f"Warmup: expected 0 exceptions, got {exception_counter}" + ) for i in range(num_threads): if is_threaded: @@ -165,32 +165,32 @@ def worker(worker_id, prepopulate_worker_ids: bool = False): t.join(timeout) assert not t.is_alive(), "Thread timed out" assert exception_counter == 0, f"Expected 0 exceptions, got {exception_counter}" - assert ( - done_counter == num_threads - ), f"Expected {num_threads} done, got {done_counter}" + assert done_counter == num_threads, ( + f"Expected {num_threads} done, got {done_counter}" + ) for i in range(num_threads): overlap_scores = radix_tree.find_matches([i]) assert overlap_scores.scores is not None worker_key = (i, 0) - assert ( - worker_key in overlap_scores.scores - ), f"Worker {worker_key} not found in scores" - assert ( - overlap_scores.scores[worker_key] == 1 - ), f"Expected score 1 for worker {worker_key}, got {overlap_scores.scores[worker_key]}" + assert worker_key in overlap_scores.scores, ( + f"Worker {worker_key} not found in scores" + ) + assert overlap_scores.scores[worker_key] == 1, ( + f"Expected score 1 for worker {worker_key}, got {overlap_scores.scores[worker_key]}" + ) # get all blocks blocks = radix_tree.dump_tree_as_events() expected_blocks = num_threads + (prepopulate_worker_ids * num_threads) - assert ( - len(blocks) == expected_blocks - ), f"Expected {expected_blocks} block events, got {len(blocks)}" + assert len(blocks) == expected_blocks, ( + f"Expected {expected_blocks} block events, got {len(blocks)}" + ) # remove single worker radix_tree.remove_worker(0) expected_blocks_after_removal = expected_blocks - ( 2 if prepopulate_worker_ids else 1 ) blocks_after_removal = radix_tree.dump_tree_as_events() - assert ( - len(blocks_after_removal) == expected_blocks_after_removal - ), f"Expected {expected_blocks_after_removal} block events after removal, got {len(blocks_after_removal)}" + assert len(blocks_after_removal) == expected_blocks_after_removal, ( + f"Expected {expected_blocks_after_removal} block events after removal, got {len(blocks_after_removal)}" + ) diff --git a/lib/gpu_memory_service/integrations/sglang/memory_saver.py b/lib/gpu_memory_service/integrations/sglang/memory_saver.py index d27b34d9b622..3bde5525032d 100644 --- a/lib/gpu_memory_service/integrations/sglang/memory_saver.py +++ b/lib/gpu_memory_service/integrations/sglang/memory_saver.py @@ -70,7 +70,11 @@ def __init__( def _init_allocators( self, - ) -> tuple[Optional["GMSClientMemoryManager"], "GMSClientMemoryManager", str,]: + ) -> tuple[ + Optional["GMSClientMemoryManager"], + "GMSClientMemoryManager", + str, + ]: """Create allocator with mode from config (default: RW_OR_RO).""" mode = self._requested_mode or RequestedLockType.RW_OR_RO weights_allocator = get_or_create_gms_client_memory_manager( diff --git a/lib/gpu_memory_service/integrations/vllm/model_runner.py b/lib/gpu_memory_service/integrations/vllm/model_runner.py index 3e1255f58669..574481cca5c4 100644 --- a/lib/gpu_memory_service/integrations/vllm/model_runner.py +++ b/lib/gpu_memory_service/integrations/vllm/model_runner.py @@ -92,12 +92,12 @@ def allocate_kv_cache_on_wake(self) -> dict: dying engine's abort() releases the lock and frees memory before we can connect. """ - assert hasattr( - self, "_shadow_kv_cache_config" - ), "_shadow_kv_cache_config not set — was enter_shadow_init() called?" - assert hasattr( - self, "_shadow_kernel_block_sizes" - ), "_shadow_kernel_block_sizes not set — was enter_shadow_init() called?" + assert hasattr(self, "_shadow_kv_cache_config"), ( + "_shadow_kv_cache_config not set — was enter_shadow_init() called?" + ) + assert hasattr(self, "_shadow_kernel_block_sizes"), ( + "_shadow_kernel_block_sizes not set — was enter_shadow_init() called?" + ) config = self._shadow_kv_cache_config diff --git a/lib/gpu_memory_service/integrations/vllm/worker.py b/lib/gpu_memory_service/integrations/vllm/worker.py index 404457b4e5fd..7c79709a13b3 100644 --- a/lib/gpu_memory_service/integrations/vllm/worker.py +++ b/lib/gpu_memory_service/integrations/vllm/worker.py @@ -313,9 +313,9 @@ def wake_up(self, tags: Optional[List[str]] = None) -> None: else: # Normal case: KV cache was allocated via GMS, reconnect + reallocate + remap kv_cache_manager = get_gms_client_memory_manager("kv_cache") - assert ( - kv_cache_manager is not None - ), "GMS KV cache client is not initialized" + assert kv_cache_manager is not None, ( + "GMS KV cache client is not initialized" + ) assert kv_cache_manager.is_unmapped, "GMS KV cache is not unmapped" kv_cache_manager.connect(RequestedLockType.RW) kv_cache_manager.reallocate_all_handles(tag="kv_cache") diff --git a/tests/basic/test_autodeploy_backend.py b/tests/basic/test_autodeploy_backend.py index 471dd65f510e..6412ce4242b2 100644 --- a/tests/basic/test_autodeploy_backend.py +++ b/tests/basic/test_autodeploy_backend.py @@ -188,7 +188,7 @@ def test_smoke(request, runtime_services): response = send_completion_request( prompt=PROMPT, max_tokens=100, timeout=20 ) - assert ( - response.ok - ), f"Expected successful status, got {response.status_code}" + assert response.ok, ( + f"Expected successful status, got {response.status_code}" + ) logger.info(f"Completion request succeeded: {response.status_code}") diff --git a/tests/basic/test_wheel_contents.py b/tests/basic/test_wheel_contents.py index 62d15a80180a..6019334c558f 100644 --- a/tests/basic/test_wheel_contents.py +++ b/tests/basic/test_wheel_contents.py @@ -31,8 +31,7 @@ def test_no_bundled_shared_libraries(): str(f) for f in installed_files if ".libs/" in str(f) and ".so" in str(f) ] - assert ( - not bundled_libs - ), "Unexpected shared libraries bundled in ai-dynamo-runtime:\n" + "\n".join( - f" {lib}" for lib in bundled_libs + assert not bundled_libs, ( + "Unexpected shared libraries bundled in ai-dynamo-runtime:\n" + + "\n".join(f" {lib}" for lib in bundled_libs) ) diff --git a/tests/dependencies/test_kvbm_imports.py b/tests/dependencies/test_kvbm_imports.py index 68715270cd67..b36349724e4e 100644 --- a/tests/dependencies/test_kvbm_imports.py +++ b/tests/dependencies/test_kvbm_imports.py @@ -34,9 +34,9 @@ def _check_kvbm_wheel_exists(): f"stdout: {result.stdout}\n" f"stderr: {result.stderr}" ) - assert ( - "kvbm" in result.stdout - ), f"Expected kvbm wheel in output, got: {result.stdout}" + assert "kvbm" in result.stdout, ( + f"Expected kvbm wheel in output, got: {result.stdout}" + ) def _check_kvbm_imports(): diff --git a/tests/deploy/test_deploy.py b/tests/deploy/test_deploy.py index bab709da3844..65382e6eb2a2 100644 --- a/tests/deploy/test_deploy.py +++ b/tests/deploy/test_deploy.py @@ -86,9 +86,9 @@ def validate_chat_response( assert "message" in choice, f"Choice missing 'message' field: {choice}" message = choice["message"] - assert ( - message.get("role") == "assistant" - ), f"Expected role 'assistant', got '{message.get('role')}'" + assert message.get("role") == "assistant", ( + f"Expected role 'assistant', got '{message.get('role')}'" + ) assert "content" in message, f"Message missing 'content' field: {message}" content = message["content"] @@ -98,9 +98,9 @@ def validate_chat_response( ) assert "model" in data, f"Response missing 'model' field: {data}" - assert ( - data["model"] == expected_model - ), f"Expected model '{expected_model}', got '{data['model']}'" + assert data["model"] == expected_model, ( + f"Expected model '{expected_model}', got '{data['model']}'" + ) logger.info( f"Response validation passed: model={data['model']}, " @@ -169,9 +169,9 @@ async def test_deployment( frontend_pods = deployment.get_pods([deployment.frontend_service_name]) frontend_pod_list = frontend_pods.get(deployment.frontend_service_name, []) - assert ( - len(frontend_pod_list) > 0 - ), f"No frontend pods found for deployment {deployment_spec.name}" + assert len(frontend_pod_list) > 0, ( + f"No frontend pods found for deployment {deployment_spec.name}" + ) frontend_pod = frontend_pod_list[0] logger.info(f"Found frontend pod: {frontend_pod.name}") @@ -179,9 +179,9 @@ async def test_deployment( # Setup port forwarding port = deployment_spec.port port_forward = deployment.port_forward(frontend_pod, port) - assert ( - port_forward is not None - ), f"Failed to establish port forward to {frontend_pod.name}:{port}" + assert port_forward is not None, ( + f"Failed to establish port forward to {frontend_pod.name}:{port}" + ) base_url = f"http://localhost:{port_forward.local_port}" logger.info(f"Port forwarding established: {base_url}") @@ -196,9 +196,9 @@ async def test_deployment( max_attempts=30, ) - assert ( - model_ready - ), f"Model '{model}' did not become available within the timeout period" + assert model_ready, ( + f"Model '{model}' did not become available within the timeout period" + ) # Send test request url = f"{base_url}{endpoint}" @@ -258,9 +258,9 @@ async def test_gaie_deployment( httproute_path = os.path.join(gaie_dir, "http-route.yaml") assert os.path.exists(disagg_path), f"disagg.yaml not found: {disagg_path}" - assert os.path.exists( - httproute_path - ), f"http-route.yaml not found: {httproute_path}" + assert os.path.exists(httproute_path), ( + f"http-route.yaml not found: {httproute_path}" + ) deployment_spec = DeploymentSpec(disagg_path) deployment_spec.namespace = namespace @@ -364,9 +364,9 @@ async def test_gaie_deployment( gateway_svcs = list( kr8s.get("services", "inference-gateway", namespace=namespace) ) - assert ( - len(gateway_svcs) > 0 - ), f"inference-gateway service not found in namespace {namespace}" + assert len(gateway_svcs) > 0, ( + f"inference-gateway service not found in namespace {namespace}" + ) gateway_pf = gateway_svcs[0].portforward(remote_port=80, local_port=0) gateway_pf.start() time.sleep(2) diff --git a/tests/fault_tolerance/cancellation/test_vllm.py b/tests/fault_tolerance/cancellation/test_vllm.py index ab3608022be6..2806fb0f035c 100644 --- a/tests/fault_tolerance/cancellation/test_vllm.py +++ b/tests/fault_tolerance/cancellation/test_vllm.py @@ -132,9 +132,9 @@ def __init__( ), ] ) - env[ - "VLLM_NIXL_SIDE_CHANNEL_PORT" - ] = "5601" # TODO: use dynamic port allocation + env["VLLM_NIXL_SIDE_CHANNEL_PORT"] = ( + "5601" # TODO: use dynamic port allocation + ) # Set log directory based on worker type if is_prefill is True: diff --git a/tests/fault_tolerance/deploy/parse_factory.py b/tests/fault_tolerance/deploy/parse_factory.py index 809fb9376626..a799b9c2df57 100644 --- a/tests/fault_tolerance/deploy/parse_factory.py +++ b/tests/fault_tolerance/deploy/parse_factory.py @@ -92,8 +92,7 @@ def detect_result_type(log_dir: str) -> Optional[str]: else: # No clear indicators logging.warning( - f"Unable to detect result type in {log_dir}. " - f"No client result files found." + f"Unable to detect result type in {log_dir}. No client result files found." ) return None diff --git a/tests/fault_tolerance/etcd_ha/test_sglang.py b/tests/fault_tolerance/etcd_ha/test_sglang.py index 5542065cd2a9..048e5cafa63b 100644 --- a/tests/fault_tolerance/etcd_ha/test_sglang.py +++ b/tests/fault_tolerance/etcd_ha/test_sglang.py @@ -200,9 +200,9 @@ def test_etcd_ha_failover_sglang_aggregated(request, predownload_models): # Step 5: Send initial inference request to verify system is working logger.info("Sending initial inference request") result = send_inference_request("What is 2+2? The answer is") - assert ( - "4" in result.lower() or "four" in result.lower() - ), f"Expected '4' or 'four' in response, got: '{result}'" + assert "4" in result.lower() or "four" in result.lower(), ( + f"Expected '4' or 'four' in response, got: '{result}'" + ) # Step 6: Cycle through each replica to terminate/verify/restart for i in range(num_replicas): @@ -217,9 +217,9 @@ def test_etcd_ha_failover_sglang_aggregated(request, predownload_models): result = send_inference_request( "The capital of France is", max_tokens=20 ) - assert ( - "paris" in result.lower() - ), f"Iteration {i}: Expected 'Paris' in response, got: '{result}'" + assert "paris" in result.lower(), ( + f"Iteration {i}: Expected 'Paris' in response, got: '{result}'" + ) # Restart the terminated replica logger.info(f"Iteration {i}: Restarting replica etcd-{i}") @@ -281,9 +281,9 @@ def test_etcd_ha_failover_sglang_disaggregated( # Step 6: Send initial inference request to verify system is working logger.info("Sending initial inference request") result = send_inference_request("What is 2+2? The answer is") - assert ( - "4" in result.lower() or "four" in result.lower() - ), f"Expected '4' or 'four' in response, got: '{result}'" + assert "4" in result.lower() or "four" in result.lower(), ( + f"Expected '4' or 'four' in response, got: '{result}'" + ) # Step 7: Cycle through each replica to terminate/verify/restart for i in range(num_replicas): @@ -298,9 +298,9 @@ def test_etcd_ha_failover_sglang_disaggregated( result = send_inference_request( "The capital of France is", max_tokens=20 ) - assert ( - "paris" in result.lower() - ), f"Iteration {i}: Expected 'Paris' in response, got: '{result}'" + assert "paris" in result.lower(), ( + f"Iteration {i}: Expected 'Paris' in response, got: '{result}'" + ) # Restart the terminated replica logger.info(f"Iteration {i}: Restarting replica etcd-{i}") @@ -348,9 +348,9 @@ def test_etcd_non_ha_shutdown_sglang_aggregated(request, predownload_models): # Step 5: Send inference request to verify system is working logger.info("Sending inference request") result = send_inference_request("What is 2+2? The answer is") - assert ( - "4" in result.lower() or "four" in result.lower() - ), f"Expected '4' or 'four' in response, got: '{result}'" + assert "4" in result.lower() or "four" in result.lower(), ( + f"Expected '4' or 'four' in response, got: '{result}'" + ) logger.info("System is working correctly with single ETCD node") @@ -417,9 +417,9 @@ def test_etcd_non_ha_shutdown_sglang_disaggregated( # Step 6: Send inference request to verify system is working logger.info("Sending inference request") result = send_inference_request("What is 2+2? The answer is") - assert ( - "4" in result.lower() or "four" in result.lower() - ), f"Expected '4' or 'four' in response, got: '{result}'" + assert "4" in result.lower() or "four" in result.lower(), ( + f"Expected '4' or 'four' in response, got: '{result}'" + ) logger.info( "System is working correctly with single ETCD node in disaggregated mode" diff --git a/tests/fault_tolerance/etcd_ha/test_trtllm.py b/tests/fault_tolerance/etcd_ha/test_trtllm.py index bed1cb1828b6..f2e42f9a7d43 100644 --- a/tests/fault_tolerance/etcd_ha/test_trtllm.py +++ b/tests/fault_tolerance/etcd_ha/test_trtllm.py @@ -177,9 +177,9 @@ def test_etcd_ha_failover_trtllm_aggregated(request, predownload_models): # Step 5: Send initial inference request to verify system is working logger.info("Sending initial inference request") result = send_inference_request("What is 2+2? The answer is") - assert ( - "4" in result.lower() or "four" in result.lower() - ), f"Expected '4' or 'four' in response, got: '{result}'" + assert "4" in result.lower() or "four" in result.lower(), ( + f"Expected '4' or 'four' in response, got: '{result}'" + ) # Step 6: Cycle through each replica to terminate/verify/restart for i in range(num_replicas): @@ -194,9 +194,9 @@ def test_etcd_ha_failover_trtllm_aggregated(request, predownload_models): result = send_inference_request( "The capital of France is", max_tokens=20 ) - assert ( - "paris" in result.lower() - ), f"Iteration {i}: Expected 'Paris' in response, got: '{result}'" + assert "paris" in result.lower(), ( + f"Iteration {i}: Expected 'Paris' in response, got: '{result}'" + ) # Restart the terminated replica logger.info(f"Iteration {i}: Restarting replica etcd-{i}") @@ -257,9 +257,9 @@ def test_etcd_ha_failover_trtllm_disaggregated( # Step 6: Send initial inference request to verify system is working logger.info("Sending initial inference request") result = send_inference_request("What is 2+2? The answer is") - assert ( - "4" in result.lower() or "four" in result.lower() - ), f"Expected '4' or 'four' in response, got: '{result}'" + assert "4" in result.lower() or "four" in result.lower(), ( + f"Expected '4' or 'four' in response, got: '{result}'" + ) # Step 7: Cycle through each replica to terminate/verify/restart for i in range(num_replicas): @@ -274,9 +274,9 @@ def test_etcd_ha_failover_trtllm_disaggregated( result = send_inference_request( "The capital of France is", max_tokens=20 ) - assert ( - "paris" in result.lower() - ), f"Iteration {i}: Expected 'Paris' in response, got: '{result}'" + assert "paris" in result.lower(), ( + f"Iteration {i}: Expected 'Paris' in response, got: '{result}'" + ) # Restart the terminated replica logger.info(f"Iteration {i}: Restarting replica etcd-{i}") @@ -327,9 +327,9 @@ def test_etcd_non_ha_shutdown_trtllm_aggregated(request, predownload_models): # Step 5: Send inference request to verify system is working logger.info("Sending inference request") result = send_inference_request("What is 2+2? The answer is") - assert ( - "4" in result.lower() or "four" in result.lower() - ), f"Expected '4' or 'four' in response, got: '{result}'" + assert "4" in result.lower() or "four" in result.lower(), ( + f"Expected '4' or 'four' in response, got: '{result}'" + ) logger.info("System is working correctly with single ETCD node") @@ -395,9 +395,9 @@ def test_etcd_non_ha_shutdown_trtllm_disaggregated( # Step 6: Send inference request to verify system is working logger.info("Sending inference request") result = send_inference_request("What is 2+2? The answer is") - assert ( - "4" in result.lower() or "four" in result.lower() - ), f"Expected '4' or 'four' in response, got: '{result}'" + assert "4" in result.lower() or "four" in result.lower(), ( + f"Expected '4' or 'four' in response, got: '{result}'" + ) logger.info( "System is working correctly with single ETCD node in disaggregated mode" diff --git a/tests/fault_tolerance/etcd_ha/test_vllm.py b/tests/fault_tolerance/etcd_ha/test_vllm.py index 0df8c865d365..c56649c129a7 100644 --- a/tests/fault_tolerance/etcd_ha/test_vllm.py +++ b/tests/fault_tolerance/etcd_ha/test_vllm.py @@ -199,9 +199,9 @@ def test_etcd_ha_failover_vllm_aggregated(request, predownload_models): # Step 5: Send initial inference request to verify system is working logger.info("Sending initial inference request") result = send_inference_request("What is 2+2? The answer is") - assert ( - "4" in result.lower() or "four" in result.lower() - ), f"Expected '4' or 'four' in response, got: '{result}'" + assert "4" in result.lower() or "four" in result.lower(), ( + f"Expected '4' or 'four' in response, got: '{result}'" + ) # Step 6: Cycle through each replica to terminate/verify/restart for i in range(num_replicas): @@ -216,9 +216,9 @@ def test_etcd_ha_failover_vllm_aggregated(request, predownload_models): result = send_inference_request( "The capital of France is", max_tokens=20 ) - assert ( - "paris" in result.lower() - ), f"Iteration {i}: Expected 'Paris' in response, got: '{result}'" + assert "paris" in result.lower(), ( + f"Iteration {i}: Expected 'Paris' in response, got: '{result}'" + ) # Restart the terminated replica logger.info(f"Iteration {i}: Restarting replica etcd-{i}") @@ -280,9 +280,9 @@ def test_etcd_ha_failover_vllm_disaggregated( # Step 6: Send initial inference request to verify system is working logger.info("Sending initial inference request") result = send_inference_request("What is 2+2? The answer is") - assert ( - "4" in result.lower() or "four" in result.lower() - ), f"Expected '4' or 'four' in response, got: '{result}'" + assert "4" in result.lower() or "four" in result.lower(), ( + f"Expected '4' or 'four' in response, got: '{result}'" + ) # Step 7: Cycle through each replica to terminate/verify/restart for i in range(num_replicas): @@ -297,9 +297,9 @@ def test_etcd_ha_failover_vllm_disaggregated( result = send_inference_request( "The capital of France is", max_tokens=20 ) - assert ( - "paris" in result.lower() - ), f"Iteration {i}: Expected 'Paris' in response, got: '{result}'" + assert "paris" in result.lower(), ( + f"Iteration {i}: Expected 'Paris' in response, got: '{result}'" + ) # Restart the terminated replica logger.info(f"Iteration {i}: Restarting replica etcd-{i}") @@ -345,9 +345,9 @@ def test_etcd_non_ha_shutdown_vllm_aggregated(request, predownload_models): # Step 5: Send inference request to verify system is working logger.info("Sending inference request") result = send_inference_request("What is 2+2? The answer is") - assert ( - "4" in result.lower() or "four" in result.lower() - ), f"Expected '4' or 'four' in response, got: '{result}'" + assert "4" in result.lower() or "four" in result.lower(), ( + f"Expected '4' or 'four' in response, got: '{result}'" + ) logger.info("System is working correctly with single ETCD node") @@ -410,9 +410,9 @@ def test_etcd_non_ha_shutdown_vllm_disaggregated( # Step 6: Send inference request to verify system is working logger.info("Sending inference request") result = send_inference_request("What is 2+2? The answer is") - assert ( - "4" in result.lower() or "four" in result.lower() - ), f"Expected '4' or 'four' in response, got: '{result}'" + assert "4" in result.lower() or "four" in result.lower(), ( + f"Expected '4' or 'four' in response, got: '{result}'" + ) logger.info( "System is working correctly with single ETCD node in disaggregated mode" diff --git a/tests/fault_tolerance/etcd_ha/utils.py b/tests/fault_tolerance/etcd_ha/utils.py index b52a5d0e9d05..78a0c2d9dba7 100644 --- a/tests/fault_tolerance/etcd_ha/utils.py +++ b/tests/fault_tolerance/etcd_ha/utils.py @@ -250,9 +250,9 @@ def _replace_member(self, idx: int): # Set ETCDCTL_ENDPOINTS for etcdctl commands etcdctl_env = os.environ.copy() - etcdctl_env[ - "ETCDCTL_ENDPOINTS" - ] = f"http://127.0.0.1:{healthy_replica.client_port}" + etcdctl_env["ETCDCTL_ENDPOINTS"] = ( + f"http://127.0.0.1:{healthy_replica.client_port}" + ) etcdctl_env["ETCDCTL_API"] = "3" # First, get member list to find the old member's ID diff --git a/tests/fault_tolerance/hardware/fault_injection_service/api_service/main.py b/tests/fault_tolerance/hardware/fault_injection_service/api_service/main.py index 3ca648fec97d..c4dccd422e31 100644 --- a/tests/fault_tolerance/hardware/fault_injection_service/api_service/main.py +++ b/tests/fault_tolerance/hardware/fault_injection_service/api_service/main.py @@ -140,13 +140,13 @@ class MetricsResponse(BaseModel): timestamp: str namespace: str - gpu_metrics: Optional[ - dict[str, Any] - ] = None # GPU utilization, memory, temperature, power + gpu_metrics: Optional[dict[str, Any]] = ( + None # GPU utilization, memory, temperature, power + ) network_metrics: Optional[dict[str, Any]] = None # Latency, packet loss, throughput - inference_metrics: Optional[ - dict[str, Any] - ] = None # Inference latency, throughput, accuracy + inference_metrics: Optional[dict[str, Any]] = ( + None # Inference latency, throughput, accuracy + ) node_health: Optional[dict[str, Any]] = None # Node status, resource availability diff --git a/tests/fault_tolerance/hardware/fault_injection_service/helpers/cuda_fault_injection.py b/tests/fault_tolerance/hardware/fault_injection_service/helpers/cuda_fault_injection.py index a43b6610cddc..8ded56b84224 100644 --- a/tests/fault_tolerance/hardware/fault_injection_service/helpers/cuda_fault_injection.py +++ b/tests/fault_tolerance/hardware/fault_injection_service/helpers/cuda_fault_injection.py @@ -322,17 +322,17 @@ def cleanup_cuda_fault_injection( if not has_artifacts: print( - f" ✓ Deployment spec verified clean after {(attempt+1)*5}s" + f" ✓ Deployment spec verified clean after {(attempt + 1) * 5}s" ) spec_cleaned = True break else: print( - f" ... {(attempt+1)*5}s: Artifacts: {', '.join(artifact_details)}" + f" ... {(attempt + 1) * 5}s: Artifacts: {', '.join(artifact_details)}" ) except Exception as e: - print(f" ... {(attempt+1)*5}s: Error checking spec: {e}") + print(f" ... {(attempt + 1) * 5}s: Error checking spec: {e}") if not spec_cleaned: print(" ⚠ Could not verify spec is clean, continuing anyway...") diff --git a/tests/fault_tolerance/migration/test_vllm.py b/tests/fault_tolerance/migration/test_vllm.py index 8c3761244672..9430f71a8b52 100644 --- a/tests/fault_tolerance/migration/test_vllm.py +++ b/tests/fault_tolerance/migration/test_vllm.py @@ -129,9 +129,9 @@ def __init__( env["DYN_REQUEST_PLANE"] = request.getfixturevalue("request_plane") # All workers need unique NIXL side channel ports for KV transfer - env[ - "VLLM_NIXL_SIDE_CHANNEL_PORT" - ] = f"560{worker_id[-1]}" # TODO: use dynamic port allocation + env["VLLM_NIXL_SIDE_CHANNEL_PORT"] = ( + f"560{worker_id[-1]}" # TODO: use dynamic port allocation + ) env["DYN_LOG"] = "debug" # Disable canary health check - these tests expect full control over requests diff --git a/tests/fault_tolerance/migration/utils.py b/tests/fault_tolerance/migration/utils.py index 78afa9df3b44..60f0dda540c7 100644 --- a/tests/fault_tolerance/migration/utils.py +++ b/tests/fault_tolerance/migration/utils.py @@ -442,12 +442,12 @@ def verify_migration_occurred(frontend_process: DynamoFrontendProcess) -> None: break time.sleep(0.005) - assert ( - "Stream disconnected... recreating stream..." in log_content - ), "'Stream disconnected... recreating stream...' message not found in logs" - assert ( - "Cannot recreate stream: " not in log_content - ), "'Cannot recreate stream: ...' error found in logs" + assert "Stream disconnected... recreating stream..." in log_content, ( + "'Stream disconnected... recreating stream...' message not found in logs" + ) + assert "Cannot recreate stream: " not in log_content, ( + "'Cannot recreate stream: ...' error found in logs" + ) def _parse_migration_metric( diff --git a/tests/frontend/grpc/test_tensor_parameters.py b/tests/frontend/grpc/test_tensor_parameters.py index 5c0801f38a80..0c53a1f56da8 100644 --- a/tests/frontend/grpc/test_tensor_parameters.py +++ b/tests/frontend/grpc/test_tensor_parameters.py @@ -138,6 +138,6 @@ def test_request_parameters( for key, expected_value in request_params.items(): assert key in resp_params, f"Parameter '{key}' not echoed" actual = resp_params[key] - assert ( - actual == expected_value - ), f"{key}: expected {expected_value}, got {actual}" + assert actual == expected_value, ( + f"{key}: expected {expected_value}, got {actual}" + ) diff --git a/tests/frontend/grpc/triton_echo_client.py b/tests/frontend/grpc/triton_echo_client.py index c85823c73350..a9a2f73bf1a7 100644 --- a/tests/frontend/grpc/triton_echo_client.py +++ b/tests/frontend/grpc/triton_echo_client.py @@ -57,12 +57,12 @@ def run_infer(self) -> None: output0_data = results.as_numpy("INPUT0") output1_data = results.as_numpy("INPUT1") - assert ( - output0_data is not None - ), "Expected response to include output tensor 'INPUT0'" - assert ( - output1_data is not None - ), "Expected response to include output tensor 'INPUT1'" + assert output0_data is not None, ( + "Expected response to include output tensor 'INPUT0'" + ) + assert output1_data is not None, ( + "Expected response to include output tensor 'INPUT1'" + ) assert np.array_equal(input0_data, output0_data) assert np.array_equal(input1_data, output1_data) @@ -111,19 +111,19 @@ def callback(user_data, result, error): ) data_item = user_data._completed_requests.get(timeout=5) - assert ( - isinstance(data_item, Exception) is False - ), f"Stream inference failed: {data_item}" + assert isinstance(data_item, Exception) is False, ( + f"Stream inference failed: {data_item}" + ) output0_data = data_item.as_numpy("INPUT0") output1_data = data_item.as_numpy("INPUT1") - assert ( - output0_data is not None - ), "Expected response to include output tensor 'INPUT0'" - assert ( - output1_data is not None - ), "Expected response to include output tensor 'INPUT1'" + assert output0_data is not None, ( + "Expected response to include output tensor 'INPUT0'" + ) + assert output1_data is not None, ( + "Expected response to include output tensor 'INPUT1'" + ) assert np.array_equal(input0_data, output0_data) assert np.array_equal(input1_data, output1_data) diff --git a/tests/frontend/test_completion_mocker_engine.py b/tests/frontend/test_completion_mocker_engine.py index a4f1e8ede58a..6cc22a6a1663 100644 --- a/tests/frontend/test_completion_mocker_engine.py +++ b/tests/frontend/test_completion_mocker_engine.py @@ -62,8 +62,7 @@ def test_completion_string_prompt(start_services_with_mocker) -> None: response = _send_completion_request(payload, frontend_port) assert response.status_code == 200, ( - f"Completion request failed with status " - f"{response.status_code}: {response.text}" + f"Completion request failed with status {response.status_code}: {response.text}" ) @@ -94,8 +93,7 @@ def test_completion_single_element_array_prompt(start_services_with_mocker) -> N response = _send_completion_request(payload, frontend_port) assert response.status_code == 200, ( - f"Completion request failed with status " - f"{response.status_code}: {response.text}" + f"Completion request failed with status {response.status_code}: {response.text}" ) @@ -115,13 +113,12 @@ def test_completion_multi_element_array_prompt(start_services_with_mocker) -> No response_data = response.json() assert response.status_code == 200, ( - f"Completion request failed with status " - f"{response.status_code}: {response.text}" + f"Completion request failed with status {response.status_code}: {response.text}" ) expected_choices = len(payload.get("prompt")) # type: ignore choices = len(response_data.get("choices", [])) - assert ( - expected_choices == choices - ), f"Expected {expected_choices} choices, got {choices}" + assert expected_choices == choices, ( + f"Expected {expected_choices} choices, got {choices}" + ) diff --git a/tests/frontend/test_prepost.py b/tests/frontend/test_prepost.py index 3eae02399179..60137d4d3750 100644 --- a/tests/frontend/test_prepost.py +++ b/tests/frontend/test_prepost.py @@ -1663,9 +1663,9 @@ def test_stream_interval_1(processor): if delta.get("content") is not None: seen_content = True if seen_content: - assert ( - delta.get("reasoning_content") is None - ), "reasoning_content appeared after regular content started" + assert delta.get("reasoning_content") is None, ( + "reasoning_content appeared after regular content started" + ) for r in results: delta = r.get("delta", {}) @@ -1716,9 +1716,9 @@ def test_stream_interval_20(tokenizer, request_for_sampling, sampling_params): # -- no markup should appear in content --------------------- all_content = "".join(r.get("delta", {}).get("content", "") for r in results) - assert ( - "" not in all_content - ), f"Raw markup leaked into content: {all_content!r}" + assert "" not in all_content, ( + f"Raw markup leaked into content: {all_content!r}" + ) assert "" not in all_content # -- finish reason ------------------------------------------------------ @@ -1835,9 +1835,9 @@ def test_stream_terminal_single_chunk(tokenizer, request_for_sampling, sampling_ # -- no markup should appear in content --------------------- all_content = "".join(r.get("delta", {}).get("content", "") for r in results) - assert ( - "" not in all_content - ), f"Raw markup leaked into content: {all_content!r}" + assert "" not in all_content, ( + f"Raw markup leaked into content: {all_content!r}" + ) assert "" not in all_content # -- finish reason ------------------------------------------------------ @@ -1875,9 +1875,9 @@ def test_no_tool_call(tokenizer, request_for_sampling, sampling_params): # -- content must include the actual response ---------------------------- all_content = "".join(r.get("delta", {}).get("content", "") for r in results) - assert ( - "The capital of Tuvalu is **Haka**." in all_content - ), f"Post-reasoning content was lost. Got content: {all_content!r}" + assert "The capital of Tuvalu is **Haka**." in all_content, ( + f"Post-reasoning content was lost. Got content: {all_content!r}" + ) # -- no tool calls should be present ------------------------------------ tool_calls = _collect_tool_calls(results) diff --git a/tests/frontend/test_prepost_mistral.py b/tests/frontend/test_prepost_mistral.py index 13b66fd4718e..d6dc0a5d98b6 100644 --- a/tests/frontend/test_prepost_mistral.py +++ b/tests/frontend/test_prepost_mistral.py @@ -653,9 +653,9 @@ def test_mistral_tool_call(processor): # -- [TOOL_CALLS] markup should not appear in content ------------------- all_content = "".join(r.get("delta", {}).get("content", "") for r in results) - assert ( - "[TOOL_CALLS]" not in all_content - ), f"Raw [TOOL_CALLS] markup leaked into content: {all_content!r}" + assert "[TOOL_CALLS]" not in all_content, ( + f"Raw [TOOL_CALLS] markup leaked into content: {all_content!r}" + ) # -- finish reason ------------------------------------------------------ finish_reasons = [r["finish_reason"] for r in results if r.get("finish_reason")] @@ -709,9 +709,9 @@ def test_mistral_tool_call_interval_20( # -- [TOOL_CALLS] markup should not appear in content ------------------- all_content = "".join(r.get("delta", {}).get("content", "") for r in results) - assert ( - "[TOOL_CALLS]" not in all_content - ), f"Raw [TOOL_CALLS] markup leaked into content: {all_content!r}" + assert "[TOOL_CALLS]" not in all_content, ( + f"Raw [TOOL_CALLS] markup leaked into content: {all_content!r}" + ) # -- finish reason ------------------------------------------------------ finish_reasons = [r["finish_reason"] for r in results if r.get("finish_reason")] diff --git a/tests/frontend/test_prompt_embeds.py b/tests/frontend/test_prompt_embeds.py index d2ae24982e51..4dc4aab431da 100644 --- a/tests/frontend/test_prompt_embeds.py +++ b/tests/frontend/test_prompt_embeds.py @@ -258,12 +258,12 @@ def test_usage_prompt_tokens_not_zero(self, dynamo_client): ) assert response.usage is not None, "Should have usage statistics" - assert ( - response.usage.prompt_tokens != 0 - ), "BUG REGRESSION: prompt_tokens is 0! This was the bug in v2.0.3." - assert ( - response.usage.prompt_tokens == sequence_length - ), f"Expected prompt_tokens={sequence_length}, got {response.usage.prompt_tokens}" + assert response.usage.prompt_tokens != 0, ( + "BUG REGRESSION: prompt_tokens is 0! This was the bug in v2.0.3." + ) + assert response.usage.prompt_tokens == sequence_length, ( + f"Expected prompt_tokens={sequence_length}, got {response.usage.prompt_tokens}" + ) assert response.usage.total_tokens == ( response.usage.prompt_tokens + response.usage.completion_tokens ), "total_tokens should equal prompt_tokens + completion_tokens" @@ -286,7 +286,7 @@ def test_large_embeddings_through_nats(self, dynamo_client): large_base64 = base64.b64encode(large_bytes).decode("utf-8") logger.info( - f"Testing large embeddings: {len(large_bytes)/1024/1024:.2f}MB decoded" + f"Testing large embeddings: {len(large_bytes) / 1024 / 1024:.2f}MB decoded" ) response = dynamo_client.completions.create( diff --git a/tests/frontend/test_vllm.py b/tests/frontend/test_vllm.py index ac8f471cd701..a5fcfdf58489 100644 --- a/tests/frontend/test_vllm.py +++ b/tests/frontend/test_vllm.py @@ -226,9 +226,9 @@ def _extract_reasoning_metrics(data: Dict[str, Any]) -> Tuple[str, Optional[int] def _validate_chat_response(response: requests.Response) -> Dict[str, Any]: """Ensure the chat completion response is well-formed and return its payload.""" - assert ( - response.status_code == 200 - ), f"Chat request failed with status {response.status_code}: {response.text}" + assert response.status_code == 200, ( + f"Chat request failed with status {response.status_code}: {response.text}" + ) response_json = response.json() if "choices" not in response_json: raise AssertionError(f"Chat response missing 'choices': {response_json}") @@ -413,7 +413,9 @@ def test_tool_calling_second_round( assert "20" in content and any( temp_word in content.lower() for temp_word in ["celsius", "temperature", "degrees", "°c", "20°"] - ), "Expected response to include temperature information from tool call result (20°C)" + ), ( + "Expected response to include temperature information from tool call result (20°C)" + ) # Measured using: tests/utils/profile_pytest.py tests/frontend/test_vllm.py::test_reasoning @@ -452,6 +454,6 @@ def test_reasoning(request, start_services: ServicePorts, predownload_models) -> content = message.get("content", "").strip() assert content, "Expected model to generate a response with content" - assert any( - char.isdigit() for char in content - ), "Expected response to contain numerical calculations" + assert any(char.isdigit() for char in content), ( + "Expected response to contain numerical calculations" + ) diff --git a/tests/gms/integration/test_external_weight_mgr.py b/tests/gms/integration/test_external_weight_mgr.py index d68da264e4b8..c396eba0e74c 100644 --- a/tests/gms/integration/test_external_weight_mgr.py +++ b/tests/gms/integration/test_external_weight_mgr.py @@ -38,17 +38,13 @@ class _SleepWakeEngine(Protocol): - def __enter__(self): - ... + def __enter__(self): ... - def __exit__(self, exc_type, exc_val, exc_tb) -> None: - ... + def __exit__(self, exc_type, exc_val, exc_tb) -> None: ... - def sleep(self) -> dict: - ... + def sleep(self) -> dict: ... - def wake(self) -> dict: - ... + def wake(self) -> dict: ... def _list_committed_weight_allocations( @@ -94,9 +90,9 @@ def _run_external_weight_mgr_test( # The read-only engine must stall until some external writer # publishes the first committed weights layout. time.sleep(2.0) - assert ( - not start_future.done() - ), "read-only engine should still be waiting for committed weights" + assert not start_future.done(), ( + "read-only engine should still be waiting for committed weights" + ) assert weights_gms.get_runtime_state().state == ServerState.EMPTY assert kv_cache_gms.get_runtime_state().state == ServerState.EMPTY diff --git a/tests/gms/integration/test_gms_shadow_failover.py b/tests/gms/integration/test_gms_shadow_failover.py index acfa1f459538..484bc9a6d222 100644 --- a/tests/gms/integration/test_gms_shadow_failover.py +++ b/tests/gms/integration/test_gms_shadow_failover.py @@ -214,9 +214,8 @@ def _run_shadow_failover_test( ) assert primary_memory_in_use > sleeping_memory_after_sleep assert ( - (primary_memory_in_use - sleeping_memory_after_sleep) - >= shadow_a_released_bytes * MIN_EXPECTED_MEMORY_RETURN_FRACTION - ) + primary_memory_in_use - sleeping_memory_after_sleep + ) >= shadow_a_released_bytes * MIN_EXPECTED_MEMORY_RETURN_FRACTION deadline = time.monotonic() + 30.0 while True: @@ -329,12 +328,12 @@ def _run_shadow_failover_test( assert [ event.kind for event in kv_events_while_lingering ] == expected_kv_kinds_while_blocked - assert _is_process_alive( - primary - ), "primary died before the linger window completed" - assert ( - not wake_future.done() - ), "shadow wake completed while the primary was still alive" + assert _is_process_alive(primary), ( + "primary died before the linger window completed" + ) + assert not wake_future.done(), ( + "shadow wake completed while the primary was still alive" + ) time.sleep(0.2) primary_memory_before_kill = get_gpu_memory_used() diff --git a/tests/kvbm_integration/common.py b/tests/kvbm_integration/common.py index 0a731f440d78..68b740984674 100755 --- a/tests/kvbm_integration/common.py +++ b/tests/kvbm_integration/common.py @@ -644,17 +644,19 @@ def test_example(llm_server_kvbm): # SAFETY: Do NOT use terminate_all_matching_process_names=True or stragglers=["vllm"] here. # Those kill ALL vLLM processes system-wide, breaking parallel test execution. # Port-based cleanup above is targeted and xdist-safe. - with ManagedProcess( - command=command, - env=env, - health_check_ports=[port, metrics_port], # vLLM server + KVBM metrics - timeout=timeout, - display_output=True, - terminate_all_matching_process_names=False, # Port-based cleanup done above instead - stragglers=[], # Empty - we handle cleanup manually per port - straggler_commands=[], # Empty - we handle cleanup manually per port - log_dir=log_dir, - ) as proc: + with ( + ManagedProcess( + command=command, + env=env, + health_check_ports=[port, metrics_port], # vLLM server + KVBM metrics + timeout=timeout, + display_output=True, + terminate_all_matching_process_names=False, # Port-based cleanup done above instead + stragglers=[], # Empty - we handle cleanup manually per port + straggler_commands=[], # Empty - we handle cleanup manually per port + log_dir=log_dir, + ) as proc + ): # Give KVBM connector extra time to fully initialize print("Waiting 5 seconds for KVBM connector to fully initialize...") time.sleep(5) @@ -888,19 +890,19 @@ def _report_results( print("=" * 70) print(f"Total requests: {num_requests}") print( - f"Exact matches: {exact_matches}/{num_requests} ({exact_matches/num_requests:.1%})" + f"Exact matches: {exact_matches}/{num_requests} ({exact_matches / num_requests:.1%})" ) print( - f"Semantic matches: {semantic_matches}/{num_requests} ({semantic_matches/num_requests:.1%})" + f"Semantic matches: {semantic_matches}/{num_requests} ({semantic_matches / num_requests:.1%})" ) print( - f"Semantic divergence: {len(mismatches)}/{num_requests} ({len(mismatches)/num_requests:.1%})" + f"Semantic divergence: {len(mismatches)}/{num_requests} ({len(mismatches) / num_requests:.1%})" ) if mismatches: - print(f"\n{'='*70}") + print(f"\n{'=' * 70}") print(f"NON-DETERMINISTIC RESPONSES ({len(mismatches)} total):") - print(f"{'='*70}") + print(f"{'=' * 70}") for mismatch in mismatches: req_num = mismatch["request_num"] if "error" in mismatch: @@ -915,9 +917,9 @@ def _report_results( semantic_success_rate = (semantic_matches / num_requests) * 100 min_success_rate = 80.0 - print(f"\n{'='*70}") + print(f"\n{'=' * 70}") print(f"SEMANTIC SUCCESS RATE: {semantic_success_rate:.1f}%") - print(f"{'='*70}") + print(f"{'=' * 70}") print(f"Failed requests: {[m['request_num'] for m in mismatches]}") if semantic_success_rate < min_success_rate: @@ -931,11 +933,11 @@ def _report_results( f"TEST PASSED - SEMANTICALLY DETERMINISTIC (>= {min_success_rate:.0f}%)" ) else: - print(f"\n{'='*70}") + print(f"\n{'=' * 70}") print("TEST PASSED - ALL RESPONSES SEMANTICALLY EQUIVALENT") - print(f"{'='*70}") + print(f"{'=' * 70}") print( - f"Exact matches: {exact_matches}/{num_requests} ({exact_matches/num_requests:.1%})" + f"Exact matches: {exact_matches}/{num_requests} ({exact_matches / num_requests:.1%})" ) def _show_final_kvbm_stats(self, metrics_port: int, initial_offload: int): @@ -948,9 +950,9 @@ def _show_final_kvbm_stats(self, metrics_port: int, initial_offload: int): Raises: pytest.fail: If no offload activity was detected during the test """ - print(f"\n{'='*70}") + print(f"\n{'=' * 70}") print("FINAL KVBM STATS") - print(f"{'='*70}") + print(f"{'=' * 70}") try: final_metrics = fetch_kvbm_metrics(port=metrics_port) final_offload = final_metrics.get("kvbm_offload_blocks_d2h", 0) @@ -1055,9 +1057,9 @@ def base_test_spanish_prompt_determinism_under_load( time.sleep(10) # Send requests and track results - print(f"\n{'='*70}") + print(f"\n{'=' * 70}") print(f"SENDING {num_requests} REQUESTS (comparing against baseline)") - print(f"{'='*70}") + print(f"{'=' * 70}") responses = [] mismatches = [] @@ -1065,7 +1067,7 @@ def base_test_spanish_prompt_determinism_under_load( semantic_matches = 0 for i in range(num_requests): - print(f"\n--- Request {i+1}/{num_requests} ---") + print(f"\n--- Request {i + 1}/{num_requests} ---") try: response = tester.make_request( @@ -1260,9 +1262,9 @@ def base_test_determinism_with_cache_reset( if total_passed + total_failed == 0: pytest.skip("No tests were completed - insufficient data") - assert ( - success_rate >= success_rate_threshold - ), f"Model is not deterministic across cache reset: {total_failed} comparisons failed, success rate {success_rate:.1%} lower than expected {success_rate_threshold*100}%" + assert success_rate >= success_rate_threshold, ( + f"Model is not deterministic across cache reset: {total_failed} comparisons failed, success rate {success_rate:.1%} lower than expected {success_rate_threshold * 100}%" + ) # ============================================================================ diff --git a/tests/kvbm_integration/test_chunked_prefill.py b/tests/kvbm_integration/test_chunked_prefill.py index 1b257de58bd0..e27a5d8b1ab2 100755 --- a/tests/kvbm_integration/test_chunked_prefill.py +++ b/tests/kvbm_integration/test_chunked_prefill.py @@ -192,9 +192,9 @@ def test_chunked_prefill_offload(tester, llm_server_kvbm): # noqa: F811 # Verify onboarding occurred onboarded_blocks = metrics_p3["kvbm_onboard_blocks_h2d"] - assert ( - onboarded_blocks > 0 - ), "Phase 3: No blocks onboarded. Expected CPU→GPU transfer after cache eviction." + assert onboarded_blocks > 0, ( + "Phase 3: No blocks onboarded. Expected CPU→GPU transfer after cache eviction." + ) print(f"✓ Phase 3: {onboarded_blocks} blocks onboarded from CPU") diff --git a/tests/kvbm_integration/test_consolidator_router_e2e.py b/tests/kvbm_integration/test_consolidator_router_e2e.py index 9cf4f4065157..c56ea9982343 100755 --- a/tests/kvbm_integration/test_consolidator_router_e2e.py +++ b/tests/kvbm_integration/test_consolidator_router_e2e.py @@ -305,16 +305,18 @@ def frontend_server(test_directory, runtime_services): frontend_log_dir.mkdir(parents=True, exist_ok=True) # Create managed process and start via context manager - with ManagedProcess( - command=command, - env=env, - health_check_urls=[f"http://localhost:{FRONTEND_PORT}/health"], - timeout=120, # Increased timeout for frontend+router initialization - working_dir=str(test_directory), - display_output=False, - log_dir=str(frontend_log_dir), # Absolute path keeps logs in test directory - terminate_all_matching_process_names=False, # Don't kill nats-server/etcd started by runtime_services - ) as frontend_process: + with ( + ManagedProcess( + command=command, + env=env, + health_check_urls=[f"http://localhost:{FRONTEND_PORT}/health"], + timeout=120, # Increased timeout for frontend+router initialization + working_dir=str(test_directory), + display_output=False, + log_dir=str(frontend_log_dir), # Absolute path keeps logs in test directory + terminate_all_matching_process_names=False, # Don't kill nats-server/etcd started by runtime_services + ) as frontend_process + ): # Get actual log file path from ManagedProcess (it may modify log_dir to use temp directory) log_file = Path(frontend_process._log_path) logger.info(f"Frontend started on port {FRONTEND_PORT}, log file: {log_file}") @@ -474,9 +476,9 @@ def assert_no_errors_in_logs( worker_errors = check_logs_for_patterns( worker_log, self.ERROR_PATTERNS, f"{engine_name} Worker" ) - assert ( - not worker_errors - ), f"Errors in {engine_name} Worker logs: {worker_errors}" + assert not worker_errors, ( + f"Errors in {engine_name} Worker logs: {worker_errors}" + ) frontend_errors = check_logs_for_patterns( frontend_log, self.ERROR_PATTERNS, "Frontend/Router" @@ -531,7 +533,7 @@ def test_basic_consolidator_flow(self, tester, llm_worker, frontend_server): response = tester.send_chat_request(messages) content = response["choices"][0]["message"]["content"] logger.info( - f"Request {i+1}/3: {messages[0]['content'][:30]}... => {content[:40]}..." + f"Request {i + 1}/3: {messages[0]['content'][:30]}... => {content[:40]}..." ) # Wait for logs to flush @@ -579,9 +581,9 @@ def test_consolidator_handles_concurrent_requests( time.sleep(5) # Assertions - assert ( - successes >= num_requests * 0.9 - ), f"Too many failed requests: {num_requests - successes}" + assert successes >= num_requests * 0.9, ( + f"Too many failed requests: {num_requests - successes}" + ) # Check for errors in logs self.assert_no_errors_in_logs( @@ -624,9 +626,9 @@ def test_store_deduplication_across_sources( successes, _ = self.send_concurrent_requests( tester, num_requests, max_tokens=50 ) - assert ( - successes >= num_requests * 0.9 - ), f"Too many failed requests: {num_requests - successes}" + assert successes >= num_requests * 0.9, ( + f"Too many failed requests: {num_requests - successes}" + ) # Wait for events to be processed time.sleep(5) @@ -667,23 +669,23 @@ def test_store_deduplication_across_sources( # Assertions: # 1. We should receive STORE events from both sources (order doesn't matter) - assert ( - first_source_stores > 0 - ), f"Expected STORE events from first source (could be {engine.upper()} or KVBM)" - assert ( - dedup_stores > 0 - ), "Expected DEDUP STORE events from second source (proves deduplication working)" + assert first_source_stores > 0, ( + f"Expected STORE events from first source (could be {engine.upper()} or KVBM)" + ) + assert dedup_stores > 0, ( + "Expected DEDUP STORE events from second source (proves deduplication working)" + ) # 2. Published stores should equal first source stores # (each unique block is published once when first stored, regardless of which source) - assert ( - published_stores == first_source_stores - ), f"Expected published events ({published_stores}) to equal first-source stores ({first_source_stores})" + assert published_stores == first_source_stores, ( + f"Expected published events ({published_stores}) to equal first-source stores ({first_source_stores})" + ) # 3. Total stores should be first source + second source (each block stored in both) - assert ( - total_stores_received == first_source_stores + dedup_stores - ), f"Total should be first-source ({first_source_stores}) + second-source ({dedup_stores})" + assert total_stores_received == first_source_stores + dedup_stores, ( + f"Total should be first-source ({first_source_stores}) + second-source ({dedup_stores})" + ) # 4. Check for errors in logs self.assert_no_errors_in_logs(worker_log, frontend_server["log_file"], engine) @@ -740,16 +742,20 @@ def test_remove_deduplication_across_sources( frontend_log_dir = Path(os.path.join(test_directory, "frontend")).absolute() frontend_log_dir.mkdir(parents=True, exist_ok=True) - with ManagedProcess( - command=frontend_command, - env=frontend_env, - health_check_urls=[f"http://localhost:{FRONTEND_PORT}/health"], - timeout=120, - working_dir=str(test_directory), - display_output=False, - log_dir=str(frontend_log_dir), # Absolute path keeps logs in test directory - terminate_all_matching_process_names=False, # Don't kill nats-server/etcd started by runtime_services - ) as _frontend_process: + with ( + ManagedProcess( + command=frontend_command, + env=frontend_env, + health_check_urls=[f"http://localhost:{FRONTEND_PORT}/health"], + timeout=120, + working_dir=str(test_directory), + display_output=False, + log_dir=str( + frontend_log_dir + ), # Absolute path keeps logs in test directory + terminate_all_matching_process_names=False, # Don't kill nats-server/etcd started by runtime_services + ) as _frontend_process + ): # Get actual log file path from ManagedProcess frontend_log = Path(_frontend_process._log_path) logger.info(f"Frontend started on port {FRONTEND_PORT}") @@ -965,20 +971,20 @@ def send_request(request_idx: int) -> tuple[int, bool]: # 1. We should see removals where blocks still exist in another source # This proves deduplication is working (REMOVE not sent to router yet) # Order doesn't matter - could be engine→KVBM or KVBM→engine - assert ( - removes_but_still_in_other_source > 0 - ), f"Expected removals where blocks still exist in another source (deduplication working) for {engine.upper()}" + assert removes_but_still_in_other_source > 0, ( + f"Expected removals where blocks still exist in another source (deduplication working) for {engine.upper()}" + ) # 2. REMOVE events should be published for last-source removals # Order doesn't matter - could be engine or KVBM as last source - assert ( - published_removes > 0 - ), "Expected REMOVE events to be published for last-source removals" + assert published_removes > 0, ( + "Expected REMOVE events to be published for last-source removals" + ) # 3. Published removes should equal removes from last source - assert ( - published_removes == removes_from_last_source - ), f"Expected published REMOVE events ({published_removes}) to equal last-source removals ({removes_from_last_source})" + assert published_removes == removes_from_last_source, ( + f"Expected published REMOVE events ({published_removes}) to equal last-source removals ({removes_from_last_source})" + ) # 3. Check for errors in logs self.assert_no_errors_in_logs(worker_log, frontend_log, engine) diff --git a/tests/kvbm_integration/test_cuda_graph.py b/tests/kvbm_integration/test_cuda_graph.py index 83aabc11515f..5cc2e4d042fd 100755 --- a/tests/kvbm_integration/test_cuda_graph.py +++ b/tests/kvbm_integration/test_cuda_graph.py @@ -184,9 +184,9 @@ def test_kvbm_without_cuda_graph_enabled(request, runtime_services): logger.info(f"Worker PID: {worker.get_pid()}") response = send_completion_request(PROMPT, 100, timeout=10) - assert ( - response.ok - ), f"Expected successful status, got {response.status_code}" + assert response.ok, ( + f"Expected successful status, got {response.status_code}" + ) logger.info(f"Completion request succeeded: {response.status_code}") @@ -221,7 +221,7 @@ def test_kvbm_with_cuda_graph_enabled(request, runtime_services): logger.info(f"Worker PID: {worker.get_pid()}") response = send_completion_request(PROMPT, 100, timeout=10) - assert ( - response.ok - ), f"Expected successful status, got {response.status_code}" + assert response.ok, ( + f"Expected successful status, got {response.status_code}" + ) logger.info(f"Completion request succeeded: {response.status_code}") diff --git a/tests/kvbm_integration/test_determinism_agg.py b/tests/kvbm_integration/test_determinism_agg.py index 1201ff0c0c36..3cb5958e4b83 100755 --- a/tests/kvbm_integration/test_determinism_agg.py +++ b/tests/kvbm_integration/test_determinism_agg.py @@ -168,9 +168,9 @@ def _set_up_trtllm_config(self, gpu_cache_blocks): "KVBM_TRTLLM_LLMAPI_CONFIG_PATH", "/tmp/kvbm_llm_api_config.yaml" ) llm_api_config: Dict[str, Any] = {} - llm_api_config[ - "cuda_graph_config" - ] = None # explicitly disable CUDA graph since Connector API doesn't support CUDA graph yet in TRTLLM + llm_api_config["cuda_graph_config"] = ( + None # explicitly disable CUDA graph since Connector API doesn't support CUDA graph yet in TRTLLM + ) llm_api_config["kv_cache_config"] = { "enable_partial_reuse": False, "free_gpu_memory_fraction": 0.10, # Set a small GPU fraction so that we can evict/reset the on-device kv cache faster diff --git a/tests/kvbm_integration/test_kvbm.py b/tests/kvbm_integration/test_kvbm.py index ebfd3552b06e..c934940d3582 100755 --- a/tests/kvbm_integration/test_kvbm.py +++ b/tests/kvbm_integration/test_kvbm.py @@ -138,12 +138,12 @@ def test_offload_and_onboard(tester, llm_server_kvbm): # noqa: F811 print(f"Response 1: {response_1}") metrics = check_kvbm_metrics("Phase 1", llm_server_kvbm.metrics_port) - assert ( - metrics["kvbm_offload_blocks_d2h"] > 0 - ), "Phase 1: No blocks offloaded. KVBM may not be triggering offloads." - assert ( - metrics["kvbm_onboard_blocks_h2d"] == 0 - ), f"Phase 1: Expected 0 onboarded blocks, got {metrics['kvbm_onboard_blocks_h2d']}" + assert metrics["kvbm_offload_blocks_d2h"] > 0, ( + "Phase 1: No blocks offloaded. KVBM may not be triggering offloads." + ) + assert metrics["kvbm_onboard_blocks_h2d"] == 0, ( + f"Phase 1: Expected 0 onboarded blocks, got {metrics['kvbm_onboard_blocks_h2d']}" + ) print(f"✓ Phase 1: {metrics['kvbm_offload_blocks_d2h']} blocks offloaded") # Phase 2: Reset GPU cache @@ -158,9 +158,9 @@ def test_offload_and_onboard(tester, llm_server_kvbm): # noqa: F811 print(f"Response 2: {response_2}") metrics = check_kvbm_metrics("Phase 3", llm_server_kvbm.metrics_port) - assert ( - metrics["kvbm_onboard_blocks_h2d"] > 0 - ), "Phase 3: No blocks onboarded. Expected CPU→GPU transfer after cache reset." + assert metrics["kvbm_onboard_blocks_h2d"] > 0, ( + "Phase 3: No blocks onboarded. Expected CPU→GPU transfer after cache reset." + ) print(f"✓ Phase 3: {metrics['kvbm_onboard_blocks_h2d']} blocks onboarded from CPU") # Verify determinism @@ -212,9 +212,9 @@ def test_gpu_cache_eviction(tester, llm_server_kvbm): # noqa: F811 f"Phase 1: Expected >= {MIN_OFFLOAD_BLOCKS} blocks offloaded, " f"got {metrics_p1['kvbm_offload_blocks_d2h']}" ) - assert ( - metrics_p1["kvbm_onboard_blocks_h2d"] == 0 - ), f"Phase 1: Expected 0 onboarded, got {metrics_p1['kvbm_onboard_blocks_h2d']}" + assert metrics_p1["kvbm_onboard_blocks_h2d"] == 0, ( + f"Phase 1: Expected 0 onboarded, got {metrics_p1['kvbm_onboard_blocks_h2d']}" + ) print(f"✓ Phase 1: {metrics_p1['kvbm_offload_blocks_d2h']} blocks offloaded") # Phase 2: Second request may evict first from GPU @@ -242,9 +242,9 @@ def test_gpu_cache_eviction(tester, llm_server_kvbm): # noqa: F811 tester.make_request(prompt_1, max_tokens=MAX_TOKENS) metrics_p3 = check_kvbm_metrics("Phase 3", llm_server_kvbm.metrics_port) - assert ( - metrics_p3["kvbm_onboard_blocks_h2d"] > 0 - ), "Phase 3: No blocks onboarded. Expected CPU→GPU retrieval after eviction." + assert metrics_p3["kvbm_onboard_blocks_h2d"] > 0, ( + "Phase 3: No blocks onboarded. Expected CPU→GPU retrieval after eviction." + ) print(f"✓ Phase 3: {metrics_p3['kvbm_onboard_blocks_h2d']} blocks onboarded") print("✓ Eviction mechanics verified: offload → eviction → onboard") diff --git a/tests/kvbm_integration/test_kvbm_vllm_integration.py b/tests/kvbm_integration/test_kvbm_vllm_integration.py index c582f1984e88..de9008ca52ae 100755 --- a/tests/kvbm_integration/test_kvbm_vllm_integration.py +++ b/tests/kvbm_integration/test_kvbm_vllm_integration.py @@ -205,7 +205,7 @@ def assumes( def _assert_interface( - checks: list[tuple[Any, str] | tuple[Any, str, dict[str, Any]]] + checks: list[tuple[Any, str] | tuple[Any, str, dict[str, Any]]], ) -> None: """Run assumes() for each (obj, attr) or (obj, attr, kwargs); pytest.fail if any fail.""" errors = [] diff --git a/tests/mm_router/test_mm_router_e2e.py b/tests/mm_router/test_mm_router_e2e.py index b324ad1e5230..78379aed3887 100644 --- a/tests/mm_router/test_mm_router_e2e.py +++ b/tests/mm_router/test_mm_router_e2e.py @@ -581,9 +581,9 @@ def test_trtllm_mm_overlap_diff_images_less_than_same( overlap_probe, total_probe, segment_probe = _send_request_get_overlap( frontend_port, router_proc, probe_payload, "probe_different_images_req1" ) - assert ( - total_probe > 0 - ), f"No routing score found.\nRecent logs:\n{segment_probe[-4000:]}" + assert total_probe > 0, ( + f"No routing score found.\nRecent logs:\n{segment_probe[-4000:]}" + ) assert abs(total_probe - total_baseline) <= 4, ( f"Expected different-images total blocks to stay near baseline, " f"got different={total_probe}, baseline={total_baseline}" @@ -643,9 +643,9 @@ def test_trtllm_mm_overlap_same_images_different_prompt_less_than_same_prompt( overlap_probe, total_probe, segment_probe = _send_request_get_overlap( frontend_port, router_proc, probe_payload, "probe_same_images_prompt_b_req1" ) - assert ( - total_probe > 0 - ), f"No routing score found.\nRecent logs:\n{segment_probe[-4000:]}" + assert total_probe > 0, ( + f"No routing score found.\nRecent logs:\n{segment_probe[-4000:]}" + ) assert abs(total_probe - total_baseline) <= 4, ( f"Expected different-prompt total blocks to stay near baseline, " f"got different_prompt={total_probe}, baseline={total_baseline}" diff --git a/tests/mm_router/test_vllm_mm_router_e2e.py b/tests/mm_router/test_vllm_mm_router_e2e.py index 4ccdff9dc979..77635da84805 100644 --- a/tests/mm_router/test_vllm_mm_router_e2e.py +++ b/tests/mm_router/test_vllm_mm_router_e2e.py @@ -568,9 +568,9 @@ def test_vllm_mm_overlap_diff_images_less_than_same( overlap_probe, total_probe, segment_probe = _send_request_get_overlap( frontend_port, router_proc, probe_payload, "probe_different_images_req1" ) - assert ( - total_probe > 0 - ), f"No routing score found.\nRecent logs:\n{segment_probe[-4000:]}" + assert total_probe > 0, ( + f"No routing score found.\nRecent logs:\n{segment_probe[-4000:]}" + ) assert abs(total_probe - total_baseline) <= 4, ( f"Expected different-images total blocks to stay near baseline, " f"got different={total_probe}, baseline={total_baseline}" @@ -628,9 +628,9 @@ def test_vllm_mm_overlap_same_images_different_prompt_less_than_same_prompt( overlap_probe, total_probe, segment_probe = _send_request_get_overlap( frontend_port, router_proc, probe_payload, "probe_same_images_prompt_b_req1" ) - assert ( - total_probe > 0 - ), f"No routing score found.\nRecent logs:\n{segment_probe[-4000:]}" + assert total_probe > 0, ( + f"No routing score found.\nRecent logs:\n{segment_probe[-4000:]}" + ) assert abs(total_probe - total_baseline) <= 4, ( f"Expected different-prompt total blocks to stay near baseline, " f"got different_prompt={total_probe}, baseline={total_baseline}" diff --git a/tests/router/common.py b/tests/router/common.py index 46765164a1ec..fd9bb89f08b0 100644 --- a/tests/router/common.py +++ b/tests/router/common.py @@ -245,9 +245,9 @@ async def verify_consumer_lifecycle(): consumer_names = [info.name for info in consumer_infos] logger.info(f"Found {len(consumer_names)} consumers: {consumer_names}") - assert ( - len(consumer_names) == 2 - ), f"Expected 2 durable consumers (one per router), found {len(consumer_names)}: {consumer_names}" + assert len(consumer_names) == 2, ( + f"Expected 2 durable consumers (one per router), found {len(consumer_names)}: {consumer_names}" + ) logger.info("✓ Verified 2 durable consumers exist (one per router)") # Kill the first router process @@ -267,9 +267,9 @@ async def verify_consumer_lifecycle(): f"After killing router1, found {len(consumer_names)} consumers: {consumer_names}" ) - assert ( - len(consumer_names) == 1 - ), f"Expected 1 durable consumer after killing router1, found {len(consumer_names)}: {consumer_names}" + assert len(consumer_names) == 1, ( + f"Expected 1 durable consumer after killing router1, found {len(consumer_names)}: {consumer_names}" + ) logger.info( "✓ Verified 1 durable consumer remains after killing first router" ) @@ -290,9 +290,9 @@ async def verify_consumer_lifecycle(): f"After killing router2, found {len(consumer_names)} consumers: {consumer_names}" ) - assert ( - len(consumer_names) == 0 - ), f"Expected 0 durable consumers after killing both routers, found {len(consumer_names)}: {consumer_names}" + assert len(consumer_names) == 0, ( + f"Expected 0 durable consumers after killing both routers, found {len(consumer_names)}: {consumer_names}" + ) logger.info( "✓ Verified 0 durable consumers remain after killing both routers" ) @@ -502,9 +502,9 @@ async def test_annotation_response(): logger.info("Sending request with query_instance_id annotation...") async with session.post(url, json=annotated_payload) as response: - assert ( - response.status == 200 - ), f"Expected 200 but got {response.status}" + assert response.status == 200, ( + f"Expected 200 but got {response.status}" + ) # Collect all response chunks response_chunks = [] @@ -554,36 +554,36 @@ async def test_annotation_response(): continue # Validate worker_id info - assert ( - worker_id_info is not None - ), f"Missing worker_id in nvext. Response: {full_response}" + assert worker_id_info is not None, ( + f"Missing worker_id in nvext. Response: {full_response}" + ) # For aggregated mode, both prefill and decode should be the same prefill_worker_id = worker_id_info.get("prefill_worker_id") decode_worker_id = worker_id_info.get("decode_worker_id") - assert ( - prefill_worker_id is not None - ), f"Missing prefill_worker_id in worker_id: {worker_id_info}" - assert ( - decode_worker_id is not None - ), f"Missing decode_worker_id in worker_id: {worker_id_info}" - assert ( - prefill_worker_id == decode_worker_id - ), f"For aggregated mode, prefill and decode worker should be same: {worker_id_info}" + assert prefill_worker_id is not None, ( + f"Missing prefill_worker_id in worker_id: {worker_id_info}" + ) + assert decode_worker_id is not None, ( + f"Missing decode_worker_id in worker_id: {worker_id_info}" + ) + assert prefill_worker_id == decode_worker_id, ( + f"For aggregated mode, prefill and decode worker should be same: {worker_id_info}" + ) # Validate token_ids - assert ( - token_list is not None - ), f"Missing token_ids in nvext. Response: {full_response}" - assert isinstance( - token_list, list - ), f"token_ids should be a list, got: {type(token_list)}" - assert ( - len(token_list) > 0 - ), f"token_ids should not be empty: {token_list}" - assert all( - isinstance(token, int) for token in token_list - ), f"All tokens should be integers: {token_list}" + assert token_list is not None, ( + f"Missing token_ids in nvext. Response: {full_response}" + ) + assert isinstance(token_list, list), ( + f"token_ids should be a list, got: {type(token_list)}" + ) + assert len(token_list) > 0, ( + f"token_ids should not be empty: {token_list}" + ) + assert all(isinstance(token, int) for token in token_list), ( + f"All tokens should be integers: {token_list}" + ) logger.info( f"Valid token_ids with {len(token_list)} tokens: {token_list[:10]}{'...' if len(token_list) > 10 else ''}" @@ -745,7 +745,9 @@ async def send_request(req_id, payload): assert ( "Service temporarily unavailable" in error_msg or "All workers are busy" in error_msg - ), f"Expected service overload error message, got: {overload_response['body']}" + ), ( + f"Expected service overload error message, got: {overload_response['body']}" + ) return True # Run the test @@ -777,17 +779,17 @@ async def _zmq_replay_cycle( f"{indexer_url}/test/pause_listener", json={"instance_id": wid, "dp_rank": dp_rank}, ) as resp: - assert ( - resp.status == 200 - ), f"Pause {wid}:{dp_rank} failed: {await resp.text()}" + assert resp.status == 200, ( + f"Pause {wid}:{dp_rank} failed: {await resp.text()}" + ) logger.info("Sending 10 requests while indexer listeners are paused") successful_gap = await send_requests_to_router( router, 10, f"{router_name} (indexer paused)", endpoint ) - assert ( - successful_gap == 10 - ), f"Expected 10 requests while paused, got {successful_gap}" + assert successful_gap == 10, ( + f"Expected 10 requests while paused, got {successful_gap}" + ) async with aiohttp.ClientSession() as session: for wid in worker_ids: @@ -796,17 +798,17 @@ async def _zmq_replay_cycle( f"{indexer_url}/test/resume_listener", json={"instance_id": wid, "dp_rank": dp_rank}, ) as resp: - assert ( - resp.status == 200 - ), f"Resume {wid}:{dp_rank} failed: {await resp.text()}" + assert resp.status == 200, ( + f"Resume {wid}:{dp_rank} failed: {await resp.text()}" + ) logger.info("Sending 5 requests after resume (triggers gap detection + replay)") successful_post = await send_requests_to_router( router, 5, f"{router_name} (post-resume)", endpoint ) - assert ( - successful_post == 5 - ), f"Expected 5 requests post-resume, got {successful_post}" + assert successful_post == 5, ( + f"Expected 5 requests post-resume, got {successful_post}" + ) await asyncio.sleep(2) @@ -939,9 +941,9 @@ async def send_requests_to_router(router, num_requests, router_name, endpoint): successful1 = await send_requests_to_router( kv_router1, 25, "Router 1", endpoint1 ) - assert ( - successful1 == 25 - ), f"Expected 25 successful requests to router 1, got {successful1}" + assert successful1 == 25, ( + f"Expected 25 successful requests to router 1, got {successful1}" + ) # NATS interruption test: stop NATS, send requests, restart if test_nats_interruption: @@ -956,9 +958,9 @@ async def send_requests_to_router(router, num_requests, router_name, endpoint): successful_offline1 = await send_requests_to_router( kv_router1, 10, "Router 1 (NATS down)", endpoint1 ) - assert ( - successful_offline1 == 10 - ), f"Expected 10 successful requests while NATS down, got {successful_offline1}" + assert successful_offline1 == 10, ( + f"Expected 10 successful requests while NATS down, got {successful_offline1}" + ) logger.info("Restarting NATS server (fresh state)") nats_server.start() @@ -1038,9 +1040,9 @@ async def send_requests_to_router(router, num_requests, router_name, endpoint): successful2 = await send_requests_to_router( kv_router2, 25, "Router 2", endpoint2 ) - assert ( - successful2 == 25 - ), f"Expected 25 successful requests to router 2, got {successful2}" + assert successful2 == 25, ( + f"Expected 25 successful requests to router 2, got {successful2}" + ) # NATS interruption test: stop NATS again, send requests, restart, send more if test_nats_interruption: @@ -1055,9 +1057,9 @@ async def send_requests_to_router(router, num_requests, router_name, endpoint): successful_offline2 = await send_requests_to_router( kv_router2, 10, "Router 2 (NATS down)", endpoint2 ) - assert ( - successful_offline2 == 10 - ), f"Expected 10 successful requests while NATS down, got {successful_offline2}" + assert successful_offline2 == 10, ( + f"Expected 10 successful requests while NATS down, got {successful_offline2}" + ) logger.info("Restarting NATS server (fresh state)") nats_server.start() @@ -1067,9 +1069,9 @@ async def send_requests_to_router(router, num_requests, router_name, endpoint): successful_recovery = await send_requests_to_router( kv_router1, 5, "Router 1 (post-recovery)", endpoint1 ) - assert ( - successful_recovery == 5 - ), f"Expected 5 successful requests post-recovery, got {successful_recovery}" + assert successful_recovery == 5, ( + f"Expected 5 successful requests post-recovery, got {successful_recovery}" + ) if test_zmq_replay and standalone_indexer_url: await _zmq_replay_cycle( @@ -1172,13 +1174,12 @@ def sort_key(event): # /dump returns {model:tenant -> {"block_size": N, "events": [...]}} expected_key = f"{model_name}:default" assert expected_key in dump_a, ( - f"Expected dump key '{expected_key}', " - f"got keys={list(dump_a.keys())}" + f"Expected dump key '{expected_key}', got keys={list(dump_a.keys())}" ) for k, v in dump_a.items(): - assert ( - isinstance(v, dict) and "events" in v - ), f"Dump key '{k}' returned unexpected format: {v}" + assert isinstance(v, dict) and "events" in v, ( + f"Dump key '{k}' returned unexpected format: {v}" + ) sorted_standalone_a = sorted(dump_a[expected_key]["events"], key=sort_key) logger.info(f"Standalone Indexer A has {len(sorted_standalone_a)} events") @@ -1190,9 +1191,9 @@ def sort_key(event): if standalone_indexer_b_url: async with aiohttp.ClientSession() as session: async with session.get(f"{standalone_indexer_b_url}/dump") as resp: - assert ( - resp.status == 200 - ), f"GET /dump from Indexer B failed: {resp.status}" + assert resp.status == 200, ( + f"GET /dump from Indexer B failed: {resp.status}" + ) dump_b = await resp.json() assert expected_key in dump_b, ( @@ -1213,8 +1214,7 @@ def sort_key(event): "Standalone B", ) logger.info( - "All 4 dumps match: Router 1, Router 2, " - "Standalone A, Standalone B" + "All 4 dumps match: Router 1, Router 2, Standalone A, Standalone B" ) # Verify NATS consumers are created (while routers are still alive) @@ -1352,9 +1352,9 @@ async def send_progressive_requests(): ) async with session.post(chat_url, json=payload) as response: - assert ( - response.status == 200 - ), f"Request {i + 1} failed with status {response.status}" + assert response.status == 200, ( + f"Request {i + 1} failed with status {response.status}" + ) # Collect all chunks and look for nvext with worker_id and timing prefill_wid = None @@ -1406,9 +1406,9 @@ async def send_progressive_requests(): decode_worker_ids.append(decode_wid) # Verify timing info is present and valid - assert ( - timing_info is not None - ), f"Request {i + 1}: Expected timing info in final chunk, got None" + assert timing_info is not None, ( + f"Request {i + 1}: Expected timing info in final chunk, got None" + ) verify_response_timing(timing_info) # Small delay between requests @@ -1657,9 +1657,9 @@ async def test_sync(): f"got {req4['prefill_worker_id']}" ) if test_dp_rank: - assert ( - req4["prefill_dp_rank"] == dp_rank_a - ), f"Request 4: expected prefill_dp_rank={dp_rank_a}, got {req4['prefill_dp_rank']}" + assert req4["prefill_dp_rank"] == dp_rank_a, ( + f"Request 4: expected prefill_dp_rank={dp_rank_a}, got {req4['prefill_dp_rank']}" + ) # Verify request 5 routed to worker b (tiebreak by smaller tree) req5 = response_worker_ids[4] @@ -1668,9 +1668,9 @@ async def test_sync(): f"got {req5['prefill_worker_id']}" ) if test_dp_rank: - assert ( - req5["prefill_dp_rank"] == dp_rank_b - ), f"Request 5: expected prefill_dp_rank={dp_rank_b}, got {req5['prefill_dp_rank']}" + assert req5["prefill_dp_rank"] == dp_rank_b, ( + f"Request 5: expected prefill_dp_rank={dp_rank_b}, got {req5['prefill_dp_rank']}" + ) logger.info( f"Response routing verified: req4 → worker_a (id={worker_a_id}, dp_rank={dp_rank_a}), " @@ -1819,13 +1819,13 @@ async def test_busy_threshold_api(): # Test 1: GET /busy_threshold - list all thresholds logger.info("Testing GET /busy_threshold (list all)") async with session.get(busy_threshold_url) as response: - assert ( - response.status == 200 - ), f"GET /busy_threshold failed with status {response.status}" + assert response.status == 200, ( + f"GET /busy_threshold failed with status {response.status}" + ) data = await response.json() - assert ( - "thresholds" in data - ), f"Expected 'thresholds' key in response: {data}" + assert "thresholds" in data, ( + f"Expected 'thresholds' key in response: {data}" + ) logger.info(f"GET /busy_threshold response: {data}") # Test 2: POST /busy_threshold with model only (get thresholds) @@ -1836,18 +1836,22 @@ async def test_busy_threshold_api(): busy_threshold_url, json={"model": model_name}, ) as response: - assert ( - response.status == 200 - ), f"POST /busy_threshold (get) failed with status {response.status}" + assert response.status == 200, ( + f"POST /busy_threshold (get) failed with status {response.status}" + ) data = await response.json() assert ( data.get("active_decode_blocks_threshold") == initial_active_decode_blocks_threshold - ), f"Expected initial active_decode_blocks_threshold={initial_active_decode_blocks_threshold}: {data}" + ), ( + f"Expected initial active_decode_blocks_threshold={initial_active_decode_blocks_threshold}: {data}" + ) assert ( data.get("active_prefill_tokens_threshold") == initial_active_prefill_tokens_threshold - ), f"Expected initial active_prefill_tokens_threshold={initial_active_prefill_tokens_threshold}: {data}" + ), ( + f"Expected initial active_prefill_tokens_threshold={initial_active_prefill_tokens_threshold}: {data}" + ) logger.info( f"POST /busy_threshold (get) response: status={response.status}, data={data}" ) @@ -1864,17 +1868,19 @@ async def test_busy_threshold_api(): "active_decode_blocks_threshold": test_active_decode_blocks_threshold, }, ) as response: - assert ( - response.status == 200 - ), f"POST /busy_threshold (set blocks) failed with status {response.status}" + assert response.status == 200, ( + f"POST /busy_threshold (set blocks) failed with status {response.status}" + ) data = await response.json() - assert ( - data.get("model") == model_name - ), f"Expected model={model_name}: {data}" + assert data.get("model") == model_name, ( + f"Expected model={model_name}: {data}" + ) assert ( data.get("active_decode_blocks_threshold") == test_active_decode_blocks_threshold - ), f"Expected active_decode_blocks_threshold={test_active_decode_blocks_threshold}: {data}" + ), ( + f"Expected active_decode_blocks_threshold={test_active_decode_blocks_threshold}: {data}" + ) logger.info(f"POST /busy_threshold (set blocks) response: {data}") # Test 4: POST /busy_threshold to set active_prefill_tokens_threshold only @@ -1891,14 +1897,16 @@ async def test_busy_threshold_api(): "active_prefill_tokens_threshold": test_active_prefill_tokens_threshold, }, ) as response: - assert ( - response.status == 200 - ), f"POST /busy_threshold (set tokens) failed with status {response.status}" + assert response.status == 200, ( + f"POST /busy_threshold (set tokens) failed with status {response.status}" + ) data = await response.json() assert ( data.get("active_prefill_tokens_threshold") == test_active_prefill_tokens_threshold - ), f"Expected active_prefill_tokens_threshold={test_active_prefill_tokens_threshold}: {data}" + ), ( + f"Expected active_prefill_tokens_threshold={test_active_prefill_tokens_threshold}: {data}" + ) logger.info(f"POST /busy_threshold (set tokens) response: {data}") # Test 5: POST /busy_threshold to set both thresholds @@ -1918,42 +1926,50 @@ async def test_busy_threshold_api(): "active_prefill_tokens_threshold": new_active_prefill_tokens_threshold, }, ) as response: - assert ( - response.status == 200 - ), f"POST /busy_threshold (set both) failed with status {response.status}" + assert response.status == 200, ( + f"POST /busy_threshold (set both) failed with status {response.status}" + ) data = await response.json() assert ( data.get("active_decode_blocks_threshold") == new_active_decode_blocks_threshold - ), f"Expected active_decode_blocks_threshold={new_active_decode_blocks_threshold}: {data}" + ), ( + f"Expected active_decode_blocks_threshold={new_active_decode_blocks_threshold}: {data}" + ) assert ( data.get("active_prefill_tokens_threshold") == new_active_prefill_tokens_threshold - ), f"Expected active_prefill_tokens_threshold={new_active_prefill_tokens_threshold}: {data}" + ), ( + f"Expected active_prefill_tokens_threshold={new_active_prefill_tokens_threshold}: {data}" + ) logger.info(f"POST /busy_threshold (set both) response: {data}") # Test 6: GET /busy_threshold - verify thresholds appear in list logger.info("Testing GET /busy_threshold to verify thresholds in list") async with session.get(busy_threshold_url) as response: - assert ( - response.status == 200 - ), f"GET /busy_threshold failed with status {response.status}" + assert response.status == 200, ( + f"GET /busy_threshold failed with status {response.status}" + ) data = await response.json() thresholds = data.get("thresholds", []) model_entry = next( (t for t in thresholds if t["model"] == model_name), None ) - assert ( - model_entry is not None - ), f"Expected model '{model_name}' in thresholds: {data}" + assert model_entry is not None, ( + f"Expected model '{model_name}' in thresholds: {data}" + ) assert ( model_entry.get("active_decode_blocks_threshold") == new_active_decode_blocks_threshold - ), f"Expected active_decode_blocks_threshold={new_active_decode_blocks_threshold}: {data}" + ), ( + f"Expected active_decode_blocks_threshold={new_active_decode_blocks_threshold}: {data}" + ) assert ( model_entry.get("active_prefill_tokens_threshold") == new_active_prefill_tokens_threshold - ), f"Expected active_prefill_tokens_threshold={new_active_prefill_tokens_threshold}: {data}" + ), ( + f"Expected active_prefill_tokens_threshold={new_active_prefill_tokens_threshold}: {data}" + ) logger.info(f"GET /busy_threshold (after set) response: {data}") # Test 7: Invalid active_decode_blocks_threshold value (should fail validation) @@ -1964,9 +1980,9 @@ async def test_busy_threshold_api(): busy_threshold_url, json={"model": model_name, "active_decode_blocks_threshold": 1.5}, ) as response: - assert ( - response.status == 400 - ), f"Expected 400 for invalid active_decode_blocks_threshold, got {response.status}" + assert response.status == 400, ( + f"Expected 400 for invalid active_decode_blocks_threshold, got {response.status}" + ) data = await response.json() logger.info( f"POST /busy_threshold (invalid blocks) response: {data}" @@ -1980,13 +1996,13 @@ async def test_busy_threshold_api(): busy_threshold_url, json={"model": model_name, "active_prefill_tokens_threshold": 5000}, ) as response: - assert ( - response.status == 200 - ), f"Expected 200 for large active_prefill_tokens_threshold, got {response.status}" + assert response.status == 200, ( + f"Expected 200 for large active_prefill_tokens_threshold, got {response.status}" + ) data = await response.json() - assert ( - data.get("active_prefill_tokens_threshold") == 5000 - ), f"Expected active_prefill_tokens_threshold=5000: {data}" + assert data.get("active_prefill_tokens_threshold") == 5000, ( + f"Expected active_prefill_tokens_threshold=5000: {data}" + ) logger.info( f"POST /busy_threshold (large tokens threshold) response: {data}" ) @@ -2001,9 +2017,9 @@ async def test_busy_threshold_api(): busy_threshold_url, json={"model": model_name, "active_prefill_tokens_threshold": -1.0}, ) as response: - assert ( - response.status == 422 - ), f"Expected 422 for negative active_prefill_tokens_threshold, got {response.status}" + assert response.status == 422, ( + f"Expected 422 for negative active_prefill_tokens_threshold, got {response.status}" + ) data = await response.json() logger.info( f"POST /busy_threshold (invalid tokens) response: {data}" @@ -2021,14 +2037,16 @@ async def test_busy_threshold_api(): "active_prefill_tokens_threshold_frac": test_frac_threshold, }, ) as response: - assert ( - response.status == 200 - ), f"POST /busy_threshold (set frac) failed with status {response.status}" + assert response.status == 200, ( + f"POST /busy_threshold (set frac) failed with status {response.status}" + ) data = await response.json() assert ( data.get("active_prefill_tokens_threshold_frac") == test_frac_threshold - ), f"Expected active_prefill_tokens_threshold_frac={test_frac_threshold}: {data}" + ), ( + f"Expected active_prefill_tokens_threshold_frac={test_frac_threshold}: {data}" + ) logger.info(f"POST /busy_threshold (set frac) response: {data}") # Test 11: Verify frac threshold appears in GET /busy_threshold list @@ -2036,21 +2054,23 @@ async def test_busy_threshold_api(): "Testing GET /busy_threshold to verify frac threshold in list" ) async with session.get(busy_threshold_url) as response: - assert ( - response.status == 200 - ), f"GET /busy_threshold failed with status {response.status}" + assert response.status == 200, ( + f"GET /busy_threshold failed with status {response.status}" + ) data = await response.json() thresholds = data.get("thresholds", []) model_entry = next( (t for t in thresholds if t["model"] == model_name), None ) - assert ( - model_entry is not None - ), f"Expected model '{model_name}' in thresholds: {data}" + assert model_entry is not None, ( + f"Expected model '{model_name}' in thresholds: {data}" + ) assert ( model_entry.get("active_prefill_tokens_threshold_frac") == test_frac_threshold - ), f"Expected active_prefill_tokens_threshold_frac={test_frac_threshold}: {data}" + ), ( + f"Expected active_prefill_tokens_threshold_frac={test_frac_threshold}: {data}" + ) logger.info( f"GET /busy_threshold (after set frac) response: {data}" ) @@ -2177,12 +2197,12 @@ async def run_direct_mode_tests(): f"Direct-mode response (attempt {attempt + 1}): " f"status=200, model={data.get('model')}" ) - assert ( - "choices" in data - ), "Expected 'choices' in response data" - assert ( - len(data["choices"]) > 0 - ), "Expected at least one choice in response" + assert "choices" in data, ( + "Expected 'choices' in response data" + ) + assert len(data["choices"]) > 0, ( + "Expected at least one choice in response" + ) break else: logger.info( diff --git a/tests/router/helper.py b/tests/router/helper.py index d3adffdf2fa2..e879f7e9279e 100644 --- a/tests/router/helper.py +++ b/tests/router/helper.py @@ -96,9 +96,9 @@ def verify_response_worker_ids( logger.info(f"Response {key}s: {worker_ids}") # All responses should have the key - assert all( - wid is not None for wid in worker_ids - ), f"Expected all {len(response_worker_ids)} responses to have {key}, got: {worker_ids}" + assert all(wid is not None for wid in worker_ids), ( + f"Expected all {len(response_worker_ids)} responses to have {key}, got: {worker_ids}" + ) # All values should be the same (due to prefix reuse routing) unique_ids = set(worker_ids) @@ -124,12 +124,12 @@ def verify_response_timing(timing_info: dict[str, Any]) -> None: total_time_ms = timing_info.get("total_time_ms") assert ttft_ms is not None and ttft_ms > 0, f"Expected ttft_ms > 0, got: {ttft_ms}" - assert ( - total_time_ms is not None and total_time_ms > 0 - ), f"Expected total_time_ms > 0, got: {total_time_ms}" - assert ( - total_time_ms >= ttft_ms - ), f"Expected total_time_ms >= ttft_ms, got {total_time_ms} < {ttft_ms}" + assert total_time_ms is not None and total_time_ms > 0, ( + f"Expected total_time_ms > 0, got: {total_time_ms}" + ) + assert total_time_ms >= ttft_ms, ( + f"Expected total_time_ms >= ttft_ms, got {total_time_ms} < {ttft_ms}" + ) logger.info( f"✓ Verified timing: ttft_ms={ttft_ms:.2f}, total_time_ms={total_time_ms:.2f}" ) @@ -458,9 +458,9 @@ async def check_nats_consumers(namespace: str, expected_count: Optional[int] = N ) if expected_count is not None: - assert ( - len(consumer_names) == expected_count - ), f"Expected {expected_count} durable consumers, found {len(consumer_names)}: {consumer_names}" + assert len(consumer_names) == expected_count, ( + f"Expected {expected_count} durable consumers, found {len(consumer_names)}: {consumer_names}" + ) logger.info(f"✓ Verified {expected_count} durable consumers exist") return consumer_names @@ -517,9 +517,9 @@ async def send_single_request(session: aiohttp.ClientSession, request_id: int): logger.info(f"Completed all requests: {successful} successful, {failed} failed") - assert ( - successful == num_requests - ), f"Expected {num_requests} successful requests, got {successful}" + assert successful == num_requests, ( + f"Expected {num_requests} successful requests, got {successful}" + ) logger.info(f"All {num_requests} requests completed successfully") diff --git a/tests/router/test_router_e2e_with_mockers.py b/tests/router/test_router_e2e_with_mockers.py index 12033f81d598..7cb011133dee 100644 --- a/tests/router/test_router_e2e_with_mockers.py +++ b/tests/router/test_router_e2e_with_mockers.py @@ -443,9 +443,9 @@ async def launch_mockers_with_indexer(self, endpoint): ), } if replay_base is not None: - payload[ - "replay_endpoint" - ] = f"tcp://127.0.0.1:{replay_base + dp_rank}" + payload["replay_endpoint"] = ( + f"tcp://127.0.0.1:{replay_base + dp_rank}" + ) async with session.post(register_url, json=payload) as resp: if resp.status != 201: body = await resp.text() @@ -457,8 +457,7 @@ async def launch_mockers_with_indexer(self, endpoint): self.worker_id_to_zmq_ports[new_worker_id] = zmq_addresses logger.info( - f"Mocker {i}: worker_id={new_worker_id}, " - f"zmq_addresses={zmq_addresses}" + f"Mocker {i}: worker_id={new_worker_id}, zmq_addresses={zmq_addresses}" ) await wait_for_indexer_workers_active( diff --git a/tests/serve/common.py b/tests/serve/common.py index d3c22d625532..32ee09f49a6a 100644 --- a/tests/serve/common.py +++ b/tests/serve/common.py @@ -42,9 +42,9 @@ def run_serve_deployment( logger = logging.getLogger(request.node.name) logger.info("Starting %s test_deployment", config.name) - assert ( - config.request_payloads is not None and len(config.request_payloads) > 0 - ), "request_payloads must be provided on EngineConfig" + assert config.request_payloads is not None and len(config.request_payloads) > 0, ( + "request_payloads must be provided on EngineConfig" + ) logger.info("Using model: %s", config.model) logger.info("Script: %s", config.script_name) diff --git a/tests/serve/test_sglang.py b/tests/serve/test_sglang.py index c66db22d2fdf..da4be91a5813 100644 --- a/tests/serve/test_sglang.py +++ b/tests/serve/test_sglang.py @@ -418,9 +418,9 @@ def test_sglang_deployment( predownload_models, ): """Test SGLang deployment scenarios using common helpers""" - assert ( - num_system_ports >= 2 - ), "serve tests require at least SYSTEM_PORT1 + SYSTEM_PORT2" + assert num_system_ports >= 2, ( + "serve tests require at least SYSTEM_PORT1 + SYSTEM_PORT2" + ) config = dataclasses.replace( sglang_config_test, frontend_port=dynamo_dynamic_ports.frontend_port ) diff --git a/tests/serve/test_trtllm.py b/tests/serve/test_trtllm.py index 340a6ec4f4b8..79d1f969ff60 100644 --- a/tests/serve/test_trtllm.py +++ b/tests/serve/test_trtllm.py @@ -45,9 +45,9 @@ def response_handler(self, response: Any) -> str: f"Video generation not completed. Status: {result.get('status')}, " f"Error: {result.get('error', 'none')}" ) - assert ( - "data" in result - ), f"Missing 'data' in response. Keys: {list(result.keys())}" + assert "data" in result, ( + f"Missing 'data' in response. Keys: {list(result.keys())}" + ) assert len(result["data"]) > 0, "Empty data in video response" entry = result["data"][0] if "url" in entry: @@ -437,9 +437,9 @@ def test_deployment( """ Test dynamo deployments with different configurations. """ - assert ( - num_system_ports >= 2 - ), "serve tests require at least SYSTEM_PORT1 + SYSTEM_PORT2" + assert num_system_ports >= 2, ( + "serve tests require at least SYSTEM_PORT1 + SYSTEM_PORT2" + ) # Use per-test ports so tests can run safely under pytest-xdist. config = dataclasses.replace( trtllm_config_test, frontend_port=dynamo_dynamic_ports.frontend_port diff --git a/tests/serve/test_vllm.py b/tests/serve/test_vllm.py index 6a895de23878..e7e42288d170 100644 --- a/tests/serve/test_vllm.py +++ b/tests/serve/test_vllm.py @@ -1031,9 +1031,9 @@ def test_lora_aggregated_router( 3. Loads the LoRA adapter on both workers via system API 4. Runs inference with the LoRA model, verifying KV cache routing """ - assert ( - num_system_ports >= 2 - ), "serve tests require at least SYSTEM_PORT1 + SYSTEM_PORT2" + assert num_system_ports >= 2, ( + "serve tests require at least SYSTEM_PORT1 + SYSTEM_PORT2" + ) minio_config: MinioLoraConfig = minio_lora_service # Create payloads that load LoRA on both workers and test inference diff --git a/tests/serve/test_vllm_omni.py b/tests/serve/test_vllm_omni.py index bdd07201e500..ce231b95783a 100644 --- a/tests/serve/test_vllm_omni.py +++ b/tests/serve/test_vllm_omni.py @@ -40,9 +40,9 @@ class ImageGenerationPayload(BasePayload): def response_handler(self, response: Any) -> str: response.raise_for_status() result = response.json() - assert ( - "data" in result - ), f"Missing 'data' in response. Keys: {list(result.keys())}" + assert "data" in result, ( + f"Missing 'data' in response. Keys: {list(result.keys())}" + ) assert len(result["data"]) > 0, "Empty data in image response" entry = result["data"][0] if "url" in entry: @@ -66,9 +66,9 @@ def response_handler(self, response: Any) -> str: f"Video generation not completed. Status: {result.get('status')}, " f"Error: {result.get('error', 'none')}" ) - assert ( - "data" in result - ), f"Missing 'data' in response. Keys: {list(result.keys())}" + assert "data" in result, ( + f"Missing 'data' in response. Keys: {list(result.keys())}" + ) assert len(result["data"]) > 0, "Empty data in video response" entry = result["data"][0] if "url" in entry: @@ -122,12 +122,12 @@ def response_handler(self, response: Any) -> str: return f"binary_audio_{len(audio_bytes)}_bytes" # JSON response (error or url format) result = response.json() - assert ( - result.get("status") != "failed" - ), f"Audio generation failed: {result.get('error', 'unknown')}" - assert ( - "data" in result - ), f"Missing 'data' in response. Keys: {list(result.keys())}" + assert result.get("status") != "failed", ( + f"Audio generation failed: {result.get('error', 'unknown')}" + ) + assert "data" in result, ( + f"Missing 'data' in response. Keys: {list(result.keys())}" + ) assert len(result["data"]) > 0, "Empty data in audio response" entry = result["data"][0] if "url" in entry and entry["url"]: diff --git a/tests/serve/test_vllm_xpu.py b/tests/serve/test_vllm_xpu.py index 93a7c82355fb..6eb12bcb7e73 100644 --- a/tests/serve/test_vllm_xpu.py +++ b/tests/serve/test_vllm_xpu.py @@ -668,9 +668,9 @@ def test_lora_aggregated_router( 3. Loads the LoRA adapter on both workers via system API 4. Runs inference with the LoRA model, verifying KV cache routing """ - assert ( - num_system_ports >= 2 - ), "serve tests require at least SYSTEM_PORT1 + SYSTEM_PORT2" + assert num_system_ports >= 2, ( + "serve tests require at least SYSTEM_PORT1 + SYSTEM_PORT2" + ) minio_config: MinioLoraConfig = minio_lora_service # Create payloads that load LoRA on both workers and test inference diff --git a/tests/utils/client.py b/tests/utils/client.py index 355c7f008de7..a0be791f5ee6 100644 --- a/tests/utils/client.py +++ b/tests/utils/client.py @@ -196,7 +196,7 @@ def wait_for_model_availability( timeout_val = attempt_timeouts[min(attempt, len(attempt_timeouts) - 1)] logger.debug( - f"Testing model availability at {test_url} (attempt {attempt+1}/{max_attempts}, timeout={timeout_val}s)" + f"Testing model availability at {test_url} (attempt {attempt + 1}/{max_attempts}, timeout={timeout_val}s)" ) response = requests.post( test_url, json=test_payload, timeout=timeout_val, headers=headers @@ -221,10 +221,12 @@ def wait_for_model_availability( except requests.Timeout as e: logger.warning( - f"Model availability test timed out (attempt {attempt+1}): {e}" + f"Model availability test timed out (attempt {attempt + 1}): {e}" ) except Exception as e: - logger.warning(f"Model availability test failed (attempt {attempt+1}): {e}") + logger.warning( + f"Model availability test failed (attempt {attempt + 1}): {e}" + ) if attempt < max_attempts - 1: wait_time = 10 if attempt < 5 else 5 diff --git a/tests/utils/engine_process.py b/tests/utils/engine_process.py index 24c3bbfee66f..7d7f011c7a0c 100644 --- a/tests/utils/engine_process.py +++ b/tests/utils/engine_process.py @@ -198,9 +198,9 @@ def from_config( @classmethod def _build_script_command(cls, config: EngineConfig) -> List[str]: """Build command from script configuration.""" - assert ( - config.script_name - ), "Must provide script_name to run fn _build_script_command" + assert config.script_name, ( + "Must provide script_name to run fn _build_script_command" + ) directory = config.directory script_path = os.path.join(directory, "launch", config.script_name) diff --git a/tests/utils/managed_deployment.py b/tests/utils/managed_deployment.py index 0354ca0103f9..0a4f80aca560 100644 --- a/tests/utils/managed_deployment.py +++ b/tests/utils/managed_deployment.py @@ -224,9 +224,9 @@ def namespace(self, value: str): def disable_grove(self): if "annotations" not in self._deployment_spec["metadata"]: self._deployment_spec["metadata"]["annotations"] = {} - self._deployment_spec["metadata"]["annotations"][ - "nvidia.com/enable-grove" - ] = "false" + self._deployment_spec["metadata"]["annotations"]["nvidia.com/enable-grove"] = ( + "false" + ) def set_model(self, model: str, service_name: Optional[str] = None): if service_name is None: @@ -757,9 +757,9 @@ async def _get_pod_status_details(self) -> List[PodStatusDetail]: continue for cs in container_statuses: - state: Literal[ - "Waiting", "Terminated", "Running", "Unknown" - ] = "Unknown" + state: Literal["Waiting", "Terminated", "Running", "Unknown"] = ( + "Unknown" + ) reason = "" message = "" exit_code: Optional[int] = None @@ -1057,7 +1057,7 @@ def port_forward( # Check if port is assigned if port_forward.local_port == 0: self._logger.debug( - f"Port not yet assigned for pod {pod.name} (attempt {attempt+1}/{max_connection_attempts})" + f"Port not yet assigned for pod {pod.name} (attempt {attempt + 1}/{max_connection_attempts})" ) continue @@ -1071,7 +1071,7 @@ def port_forward( return port_forward except (requests.ConnectionError, requests.Timeout) as e: self._logger.warning( - f"Connection test failed for pod {pod.name} (attempt {attempt+1}/{max_connection_attempts}): {e}" + f"Connection test failed for pod {pod.name} (attempt {attempt + 1}/{max_connection_attempts}): {e}" ) # Restart port-forward for next attempt (except on last attempt) diff --git a/tests/utils/payloads.py b/tests/utils/payloads.py index cb2d838fa3ce..018585223bd2 100644 --- a/tests/utils/payloads.py +++ b/tests/utils/payloads.py @@ -121,13 +121,13 @@ def extract_content(response): response.raise_for_status() result = response.json() - assert ( - "choices" in result - ), f"Missing 'choices' in response. Response keys: {list(result.keys())}" + assert "choices" in result, ( + f"Missing 'choices' in response. Response keys: {list(result.keys())}" + ) assert len(result["choices"]) > 0, "Empty choices in response" - assert ( - "message" in result["choices"][0] - ), f"Missing 'message' in first choice. Choice keys: {list(result['choices'][0].keys())}" + assert "message" in result["choices"][0], ( + f"Missing 'message' in first choice. Choice keys: {list(result['choices'][0].keys())}" + ) # Check for content in all possible fields where parsers might put output: # 1. content - standard message content @@ -188,44 +188,44 @@ def validate(self, response: Any, content: str) -> None: for item in content_logprobs: assert "token" in item, "Missing 'token' in logprobs content" assert "logprob" in item, "Missing 'logprob' in logprobs content" - assert ( - "top_logprobs" in item - ), "Missing 'top_logprobs' in logprobs content" + assert "top_logprobs" in item, ( + "Missing 'top_logprobs' in logprobs content" + ) # Sanity check: logprob should be valid (not nan/inf/positive) logprob_val = item["logprob"] assert not math.isnan(logprob_val), "logprob is NaN" assert not math.isinf(logprob_val), "logprob is infinite" - assert ( - logprob_val <= 0 - ), f"logprob should be <= 0, got {logprob_val}" + assert logprob_val <= 0, ( + f"logprob should be <= 0, got {logprob_val}" + ) # Validate bytes field is populated for the selected token assert "bytes" in item, "Missing 'bytes' in logprobs content item" token_str = item["token"] if token_str: - assert ( - item["bytes"] is not None - ), f"'bytes' should be populated for non-empty token {token_str!r}" - assert isinstance( - item["bytes"], list - ), f"'bytes' should be a list, got {type(item['bytes'])}" + assert item["bytes"] is not None, ( + f"'bytes' should be populated for non-empty token {token_str!r}" + ) + assert isinstance(item["bytes"], list), ( + f"'bytes' should be a list, got {type(item['bytes'])}" + ) # Validate top_logprobs entries have token, logprob, and bytes for top_lp in item["top_logprobs"]: - assert ( - "token" in top_lp - ), "Missing 'token' in top_logprobs entry" - assert ( - "logprob" in top_lp - ), "Missing 'logprob' in top_logprobs entry" - assert ( - "bytes" in top_lp - ), "Missing 'bytes' in top_logprobs entry" + assert "token" in top_lp, ( + "Missing 'token' in top_logprobs entry" + ) + assert "logprob" in top_lp, ( + "Missing 'logprob' in top_logprobs entry" + ) + assert "bytes" in top_lp, ( + "Missing 'bytes' in top_logprobs entry" + ) if top_lp["token"]: - assert ( - top_lp["bytes"] is not None - ), f"'bytes' should be populated for top_logprob token {top_lp['token']!r}" + assert top_lp["bytes"] is not None, ( + f"'bytes' should be populated for top_logprob token {top_lp['token']!r}" + ) logger.info( f"✓ Logprobs validation passed: found {len(content_logprobs)} tokens with logprobs" @@ -269,9 +269,9 @@ def validate(self, response, content: str) -> None: # If expected tool name is provided, validate it if self.expected_tool_name: tool_names = [tc.get("function", {}).get("name") for tc in tool_calls] - assert ( - self.expected_tool_name in tool_names - ), f"Expected tool '{self.expected_tool_name}' not found. Available tools: {tool_names}" + assert self.expected_tool_name in tool_names, ( + f"Expected tool '{self.expected_tool_name}' not found. Available tools: {tool_names}" + ) logger.info(f"Expected tool '{self.expected_tool_name}' was called") @@ -483,31 +483,31 @@ def validate(self, response: Any, content: str) -> None: logprobs_data = choice["logprobs"] if logprobs_data is not None: - assert ( - "token_logprobs" in logprobs_data - ), "Missing 'token_logprobs' in logprobs" + assert "token_logprobs" in logprobs_data, ( + "Missing 'token_logprobs' in logprobs" + ) assert "tokens" in logprobs_data, "Missing 'tokens' in logprobs" token_logprobs = logprobs_data["token_logprobs"] tokens = logprobs_data["tokens"] if token_logprobs: - assert len(token_logprobs) == len( - tokens - ), "Mismatch between token_logprobs and tokens length" + assert len(token_logprobs) == len(tokens), ( + "Mismatch between token_logprobs and tokens length" + ) # Sanity check: each logprob should be valid (not nan/inf/positive) for i, logprob_val in enumerate(token_logprobs): if logprob_val is not None: # First token can be None - assert not math.isnan( - logprob_val - ), f"logprob at index {i} is NaN" - assert not math.isinf( - logprob_val - ), f"logprob at index {i} is infinite" - assert ( - logprob_val <= 0 - ), f"logprob at index {i} should be <= 0, got {logprob_val}" + assert not math.isnan(logprob_val), ( + f"logprob at index {i} is NaN" + ) + assert not math.isinf(logprob_val), ( + f"logprob at index {i} is infinite" + ) + assert logprob_val <= 0, ( + f"logprob at index {i} should be <= 0, got {logprob_val}" + ) # Validate top_logprobs entries have token, logprob, and bytes when present top_logprobs_list = logprobs_data.get("top_logprobs", []) @@ -515,19 +515,19 @@ def validate(self, response: Any, content: str) -> None: if not token_top_lps: continue for top_lp in token_top_lps: - assert ( - "token" in top_lp - ), f"Missing 'token' in top_logprobs[{i}] entry" - assert ( - "logprob" in top_lp - ), f"Missing 'logprob' in top_logprobs[{i}] entry" - assert ( - "bytes" in top_lp - ), f"Missing 'bytes' in top_logprobs[{i}] entry" + assert "token" in top_lp, ( + f"Missing 'token' in top_logprobs[{i}] entry" + ) + assert "logprob" in top_lp, ( + f"Missing 'logprob' in top_logprobs[{i}] entry" + ) + assert "bytes" in top_lp, ( + f"Missing 'bytes' in top_logprobs[{i}] entry" + ) if top_lp["token"]: - assert ( - top_lp["bytes"] is not None - ), f"'bytes' should be populated for top_logprob token {top_lp['token']!r}" + assert top_lp["bytes"] is not None, ( + f"'bytes' should be populated for top_logprob token {top_lp['token']!r}" + ) logger.info( f"✓ Logprobs validation passed: found {len(token_logprobs)} tokens with logprobs" @@ -551,32 +551,32 @@ def extract_content(response): response.raise_for_status() result = response.json() - assert ( - result.get("object") == "response" - ), f"Expected object='response', got {result.get('object')}" - assert result.get("id", "").startswith( - "resp_" - ), f"Expected id to start with 'resp_', got {result.get('id')}" - assert ( - result.get("status") == "completed" - ), f"Expected status='completed', got {result.get('status')}" + assert result.get("object") == "response", ( + f"Expected object='response', got {result.get('object')}" + ) + assert result.get("id", "").startswith("resp_"), ( + f"Expected id to start with 'resp_', got {result.get('id')}" + ) + assert result.get("status") == "completed", ( + f"Expected status='completed', got {result.get('status')}" + ) output = result.get("output", []) assert len(output) > 0, "Response output is empty" msg = output[0] - assert ( - msg.get("type") == "message" - ), f"Expected output[0].type='message', got {msg.get('type')}" - assert ( - msg.get("role") == "assistant" - ), f"Expected role='assistant', got {msg.get('role')}" + assert msg.get("type") == "message", ( + f"Expected output[0].type='message', got {msg.get('type')}" + ) + assert msg.get("role") == "assistant", ( + f"Expected role='assistant', got {msg.get('role')}" + ) content_parts = msg.get("content", []) assert len(content_parts) > 0, "Message content is empty" - assert ( - content_parts[0].get("type") == "output_text" - ), f"Expected content[0].type='output_text', got {content_parts[0].get('type')}" + assert content_parts[0].get("type") == "output_text", ( + f"Expected content[0].type='output_text', got {content_parts[0].get('type')}" + ) return content_parts[0].get("text", "") @@ -619,23 +619,23 @@ def extract_content(response): # Validate lifecycle event ordering assert len(event_types) >= 2, f"Too few events: {event_types}" - assert ( - event_types[0] == "response.created" - ), f"First event should be response.created, got {event_types[0]}" - assert ( - event_types[1] == "response.in_progress" - ), f"Second event should be response.in_progress, got {event_types[1]}" + assert event_types[0] == "response.created", ( + f"First event should be response.created, got {event_types[0]}" + ) + assert event_types[1] == "response.in_progress", ( + f"Second event should be response.in_progress, got {event_types[1]}" + ) non_done = [e for e in event_types if e != "done"] - assert ( - non_done[-1] == "response.completed" - ), f"Last real event should be response.completed, got {non_done[-1]}" + assert non_done[-1] == "response.completed", ( + f"Last real event should be response.completed, got {non_done[-1]}" + ) # Validate text content events assert "response.output_item.added" in event_types, "Missing output_item.added" - assert ( - "response.content_part.added" in event_types - ), "Missing content_part.added" + assert "response.content_part.added" in event_types, ( + "Missing content_part.added" + ) assert "response.output_text.delta" in event_types, "Missing output_text.delta" assert "response.output_text.done" in event_types, "Missing output_text.done" assert "response.content_part.done" in event_types, "Missing content_part.done" @@ -644,13 +644,13 @@ def extract_content(response): # Verify text deltas concatenate to the final text deltas = [e[1]["delta"] for e in events if e[0] == "response.output_text.delta"] done_events = [e for e in events if e[0] == "response.output_text.done"] - assert ( - len(done_events) == 1 - ), f"Expected 1 output_text.done, got {len(done_events)}" + assert len(done_events) == 1, ( + f"Expected 1 output_text.done, got {len(done_events)}" + ) full_text = "".join(deltas) - assert ( - done_events[0][1]["text"] == full_text - ), "Concatenated deltas don't match output_text.done text" + assert done_events[0][1]["text"] == full_text, ( + "Concatenated deltas don't match output_text.done text" + ) return full_text @@ -670,15 +670,15 @@ def extract_content(response): response.raise_for_status() result = response.json() - assert ( - result.get("type") == "message" - ), f"Expected type='message', got {result.get('type')}" - assert result.get("id", "").startswith( - "msg_" - ), f"Expected id to start with 'msg_', got {result.get('id')}" - assert ( - result.get("role") == "assistant" - ), f"Expected role='assistant', got {result.get('role')}" + assert result.get("type") == "message", ( + f"Expected type='message', got {result.get('type')}" + ) + assert result.get("id", "").startswith("msg_"), ( + f"Expected id to start with 'msg_', got {result.get('id')}" + ) + assert result.get("role") == "assistant", ( + f"Expected role='assistant', got {result.get('role')}" + ) assert result.get("stop_reason") in ( "end_turn", "max_tokens", @@ -688,9 +688,9 @@ def extract_content(response): content = result.get("content", []) assert len(content) > 0, "Response content is empty" - assert ( - content[0].get("type") == "text" - ), f"Expected content[0].type='text', got {content[0].get('type')}" + assert content[0].get("type") == "text", ( + f"Expected content[0].type='text', got {content[0].get('type')}" + ) usage = result.get("usage", {}) assert "input_tokens" in usage, "Missing input_tokens in usage" @@ -734,20 +734,20 @@ def extract_content(response): # Validate lifecycle event ordering assert len(event_types) >= 3, f"Too few events: {event_types}" - assert ( - event_types[0] == "message_start" - ), f"First event should be message_start, got {event_types[0]}" - assert ( - event_types[-1] == "message_stop" - ), f"Last event should be message_stop, got {event_types[-1]}" + assert event_types[0] == "message_start", ( + f"First event should be message_start, got {event_types[0]}" + ) + assert event_types[-1] == "message_stop", ( + f"Last event should be message_stop, got {event_types[-1]}" + ) # Validate message_start structure msg_start = events[0][1] assert msg_start.get("type") == "message_start", "message_start missing type" message = msg_start.get("message", {}) - assert message.get("id", "").startswith( - "msg_" - ), "message id should start with msg_" + assert message.get("id", "").startswith("msg_"), ( + "message id should start with msg_" + ) assert message.get("role") == "assistant", "message role should be assistant" # Validate required event types @@ -758,9 +758,9 @@ def extract_content(response): # Validate message_delta has stop_reason delta_events = [e for e in events if e[0] == "message_delta"] - assert ( - len(delta_events) == 1 - ), f"Expected 1 message_delta, got {len(delta_events)}" + assert len(delta_events) == 1, ( + f"Expected 1 message_delta, got {len(delta_events)}" + ) delta_body = delta_events[0][1].get("delta", {}) assert delta_body.get("stop_reason") in ( "end_turn", @@ -797,9 +797,9 @@ def extract_embeddings(response): response.raise_for_status() result = response.json() assert "object" in result, "Missing 'object' in response" - assert ( - result["object"] == "list" - ), f"Expected object='list', got {result['object']}" + assert result["object"] == "list", ( + f"Expected object='list', got {result['object']}" + ) assert "data" in result, "Missing 'data' in response" assert len(result["data"]) > 0, "Empty data in response" @@ -807,13 +807,13 @@ def extract_embeddings(response): embeddings = [] for item in result["data"]: assert "object" in item, "Missing 'object' in embedding item" - assert ( - item["object"] == "embedding" - ), f"Expected object='embedding', got {item['object']}" + assert item["object"] == "embedding", ( + f"Expected object='embedding', got {item['object']}" + ) assert "embedding" in item, "Missing 'embedding' vector in item" - assert isinstance( - item["embedding"], list - ), "Embedding should be a list of floats" + assert isinstance(item["embedding"], list), ( + "Embedding should be a list of floats" + ) assert len(item["embedding"]) > 0, "Embedding vector should not be empty" embeddings.append(item["embedding"]) diff --git a/tests/utils/profile_pytest.py b/tests/utils/profile_pytest.py index 630c4f0f019e..7d09ce1555d8 100755 --- a/tests/utils/profile_pytest.py +++ b/tests/utils/profile_pytest.py @@ -802,8 +802,7 @@ def _find_min_vram( print(f"\n--- FIND MINIMUM {mode_label} (binary search) ---") print(f" GPU total : {total_gib:.1f} GiB") print( - f" GPU free : {free_mib / 1024:.1f} GiB " - f"(in use: {used_mib / 1024:.1f} GiB)" + f" GPU free : {free_mib / 1024:.1f} GiB (in use: {used_mib / 1024:.1f} GiB)" ) print(f" Test : {' '.join(pytest_args)}") if model_name: diff --git a/tests/utils/pytest_parallel_gpu.py b/tests/utils/pytest_parallel_gpu.py index 5c363b9cf8c0..b633d96b9cd1 100755 --- a/tests/utils/pytest_parallel_gpu.py +++ b/tests/utils/pytest_parallel_gpu.py @@ -316,8 +316,7 @@ def run_parallel( for gi in gpu_indices: if gi not in gpu_by_idx: _print( - f"ERROR: GPU{gi} not found " - f"(available: {[g['index'] for g in gpus]})" + f"ERROR: GPU{gi} not found (available: {[g['index'] for g in gpus]})" ) return 1 total = gpu_by_idx[gi]["total_mib"] / 1024.0 @@ -368,8 +367,7 @@ def run_parallel( for t in no_kv: _print(f" {t.name}") _print( - "\nAdd the appropriate marker via profile_pytest.py --kv-bytes, " - "then rerun." + "\nAdd the appropriate marker via profile_pytest.py --kv-bytes, then rerun." ) return 1 @@ -443,7 +441,7 @@ def run_parallel( # --- Report skip-marked tests immediately (like xdist SKIPPED) --- completed: list[_CompletedTest] = [] for test in skipped_tests: - _print(f"[w{test.w_id}] {test.name} SKIPPED" f" - {test.skip_reason}") + _print(f"[w{test.w_id}] {test.name} SKIPPED - {test.skip_reason}") completed.append( _CompletedTest( test=test, @@ -610,7 +608,7 @@ def _launch_test(test: _TestEntry, env_base: dict) -> _RunningTest: status = "FAILED" if skipped: - _print(f"[w{w_id}] {test.name} SKIPPED" f" - {skip_reason}") + _print(f"[w{w_id}] {test.name} SKIPPED - {skip_reason}") else: _print(f"[w{w_id}] {test.name} {status} [{duration:.0f}s]") @@ -770,9 +768,7 @@ def _launch_test(test: _TestEntry, env_base: dict) -> _RunningTest: timeout = int(test.timeout) retries = test.retries retry_str = f" ({retries} retries)" if retries else "" - _print( - f"PASSED [w{w_id}] {test.name} " f"[{duration}s/{timeout}s]{retry_str}" - ) + _print(f"PASSED [w{w_id}] {test.name} [{duration}s/{timeout}s]{retry_str}") else: duration = int(c.duration) timeout = int(test.timeout) diff --git a/tests/utils/test_managed_process_teardown.py b/tests/utils/test_managed_process_teardown.py index 0e0445cfef93..78f0129c32b4 100644 --- a/tests/utils/test_managed_process_teardown.py +++ b/tests/utils/test_managed_process_teardown.py @@ -119,9 +119,9 @@ def test_parent_and_children_killed(self, tmp_path): assert len(tree_pids) >= 2, f"Expected parent + children, got {tree_pids}" for pid in tree_pids: - assert _wait_for_pid_death( - pid, timeout=10 - ), f"PID {pid} still alive after teardown" + assert _wait_for_pid_death(pid, timeout=10), ( + f"PID {pid} still alive after teardown" + ) # --------------------------------------------------------------------------- @@ -147,14 +147,14 @@ def test_grandchildren_killed(self, tmp_path): assert mp.proc is not None root_pid = mp.proc.pid tree_pids = _wait_for_tree(root_pid, min_count=3) - assert ( - len(tree_pids) >= 3 - ), f"Expected parent + child + grandchild, got {tree_pids}" + assert len(tree_pids) >= 3, ( + f"Expected parent + child + grandchild, got {tree_pids}" + ) for pid in tree_pids: - assert _wait_for_pid_death( - pid, timeout=10 - ), f"PID {pid} still alive after teardown" + assert _wait_for_pid_death(pid, timeout=10), ( + f"PID {pid} still alive after teardown" + ) # --------------------------------------------------------------------------- @@ -198,9 +198,9 @@ def test_child_in_own_pgid_killed(self, tmp_path): pytest.skip("Child didn't get a separate pgid (OS-dependent)") for pid in tree_pids: - assert _wait_for_pid_death( - pid, timeout=10 - ), f"PID {pid} still alive after teardown (separate pgid scenario)" + assert _wait_for_pid_death(pid, timeout=10), ( + f"PID {pid} still alive after teardown (separate pgid scenario)" + ) # --------------------------------------------------------------------------- @@ -230,9 +230,9 @@ def test_stragglers_not_killed_in_xdist_mode(self, tmp_path): with mp: pass - assert _pid_alive( - bystander_pid - ), "Bystander was killed even though xdist-safe mode was on" + assert _pid_alive(bystander_pid), ( + "Bystander was killed even though xdist-safe mode was on" + ) finally: try: os.killpg(os.getpgid(bystander_pid), signal.SIGKILL) @@ -275,9 +275,9 @@ def test_stragglers_killed_when_not_xdist_mode(self, tmp_path): with mp: time.sleep(0.5) - assert _wait_for_pid_death( - bystander_pid, timeout=10 - ), "Bystander with matching straggler command should have been killed" + assert _wait_for_pid_death(bystander_pid, timeout=10), ( + "Bystander with matching straggler command should have been killed" + ) finally: try: os.killpg(os.getpgid(bystander_pid), signal.SIGKILL) @@ -350,6 +350,6 @@ def test_process_gets_sigterm_grace_before_sigkill(self, tmp_path): assert time.monotonic() < deadline, "Child never became ready" time.sleep(0.05) - assert os.path.exists( - marker_file - ), "Process was SIGKILLed before SIGTERM handler could run" + assert os.path.exists(marker_file), ( + "Process was SIGKILLed before SIGTERM handler could run" + )