diff --git a/tests/dfx/perf/scripts/run_diffusion_benchmark.py b/tests/dfx/perf/scripts/run_diffusion_benchmark.py index 078eaaea33f..1bd9bf1a143 100644 --- a/tests/dfx/perf/scripts/run_diffusion_benchmark.py +++ b/tests/dfx/perf/scripts/run_diffusion_benchmark.py @@ -317,6 +317,7 @@ def _unique_server_params(configs: list[dict[str, Any]]) -> list[dict[str, Any]] "model": cfg["server_params"]["model"], "serve_args": _build_serve_args(cfg["server_params"].get("serve_args", {})), "benchmark_backend": "vllm-omni", + "server_params": cfg["server_params"], } ) return result @@ -363,6 +364,7 @@ def diffusion_server(request): print(f"\nStarting {server_type} server for test: {test_name}") with _make_server(server_cfg) as server: server.test_name = test_name + server.server_params = server_cfg["server_params"] print(f"{server_type} server started successfully") yield server print(f"{server_type} server stopping…") @@ -400,6 +402,7 @@ def run_benchmark( params: dict[str, Any], test_name: str, backend: str = "vllm-omni", + server_params: dict[str, Any] | None = None, ) -> dict[str, Any]: """Run diffusion_benchmark_serving.py as a subprocess and return parsed metrics. @@ -496,6 +499,7 @@ def run_benchmark( "test_name": test_name, "backend": backend, "timestamp": timestamp, + "server_params": server_params, "benchmark_params": params, "result": metrics, "log_file": str(log_file), @@ -561,6 +565,7 @@ def test_diffusion_performance_benchmark(diffusion_server, benchmark_params): params=params, test_name=test_name, backend=backend, + server_params=diffusion_server.server_params, ) print(f"\n{'=' * 60}") diff --git a/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json index 66478e73bb7..eea180ad03f 100644 --- a/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json @@ -43,27 +43,6 @@ "peak_memory_mb_max": 74000, "peak_memory_mb_mean": 74000 } - }, - { - "name": "mixed_resolution", - "dataset": "random", - "task": "t2i", - "num-inference-steps": 20, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "random-request-config": [ - {"width": 512, "height": 512, "num_inference_steps": 20, "weight": 0.15}, - {"width": 768, "height": 768, "num_inference_steps": 20, "weight": 0.25}, - {"width": 1024, "height": 1024, "num_inference_steps": 25, "weight": 0.45}, - {"width": 1536, "height": 1536, "num_inference_steps": 35, "weight": 0.15} - ], - "baseline": { - "throughput_qps": 0.14, - "latency_p99": 25.0, - "peak_memory_mb_max": 74000, - "peak_memory_mb_mean": 74000 - } } ] }, @@ -116,27 +95,6 @@ "peak_memory_mb_max": 61000, "peak_memory_mb_mean": 61000 } - }, - { - "name": "mixed_resolution", - "dataset": "random", - "task": "t2i", - "num-inference-steps": 20, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "random-request-config": [ - {"width": 512, "height": 512, "num_inference_steps": 20, "weight": 0.15}, - {"width": 768, "height": 768, "num_inference_steps": 20, "weight": 0.25}, - {"width": 1024, "height": 1024, "num_inference_steps": 25, "weight": 0.45}, - {"width": 1536, "height": 1536, "num_inference_steps": 35, "weight": 0.15} - ], - "baseline": { - "throughput_qps": 0.20, - "latency_p99": 8.5, - "peak_memory_mb_max": 61000, - "peak_memory_mb_mean": 61000 - } } ] }, @@ -199,27 +157,6 @@ "peak_memory_mb_max": 74000, "peak_memory_mb_mean": 74000 } - }, - { - "name": "mixed_resolution", - "dataset": "random", - "task": "t2i", - "num-inference-steps": 20, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "random-request-config": [ - {"width": 512, "height": 512, "num_inference_steps": 20, "weight": 0.15}, - {"width": 768, "height": 768, "num_inference_steps": 20, "weight": 0.25}, - {"width": 1024, "height": 1024, "num_inference_steps": 25, "weight": 0.45}, - {"width": 1536, "height": 1536, "num_inference_steps": 35, "weight": 0.15} - ], - "baseline": { - "throughput_qps": 0.35, - "latency_p99": 5.33, - "peak_memory_mb_max": 74000, - "peak_memory_mb_mean": 74000 - } } ] }