From 5dcd828b723b1d0d8816fb7aa894d274902bcfb3 Mon Sep 17 00:00:00 2001 From: Lizhi Zhou <1432185+reasonsolo@users.noreply.github.com> Date: Tue, 14 Apr 2026 21:54:37 -0700 Subject: [PATCH 1/2] [None][fix] Cap TLLM_BENCHMARK_REQ_QUEUES_SIZE to avoid fill-loop hang When attention DP is enabled, the effective max capacity is max_batch_size * tp_size, which can be smaller than the requested concurrency. Setting TLLM_BENCHMARK_REQ_QUEUES_SIZE to concurrency in that case causes the fill loop to hang waiting for slots that will never free. Cap queue_size to min(max_capacity, concurrency) and emit a warning when capped. Signed-off-by: Lizhi Zhou <1432185+reasonsolo@users.noreply.github.com> --- examples/disaggregated/slurm/benchmark/submit.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/examples/disaggregated/slurm/benchmark/submit.py b/examples/disaggregated/slurm/benchmark/submit.py index cecf30fa9db3..6565b8121181 100644 --- a/examples/disaggregated/slurm/benchmark/submit.py +++ b/examples/disaggregated/slurm/benchmark/submit.py @@ -228,9 +228,21 @@ def build_worker_environment(worker_config, env_config, role, benchmark_mode, 'TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP', 'TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP=1') if role == "GEN": + gen_config = worker_config.get('gen', {}) + max_batch_size = gen_config.get('max_batch_size', concurrency) + enable_attention_dp = gen_config.get('enable_attention_dp', False) + tp_size = gen_config.get('tensor_parallel_size', 1) + max_capacity = max_batch_size * tp_size if enable_attention_dp else max_batch_size + queue_size = min(max_capacity, int(concurrency)) + if queue_size < int(concurrency): + print(f"[WARNING] TLLM_BENCHMARK_REQ_QUEUES_SIZE capped to " + f"{queue_size} (max_batch_size={max_batch_size} x " + f"tp_size={tp_size} with attention_dp={enable_attention_dp}) " + f"which is less than concurrency={concurrency}. " + f"Fill loop would hang if set to {concurrency}.") upsert_env_config(env_config, 'gen_worker_env_var', 'TLLM_BENCHMARK_REQ_QUEUES_SIZE', - f'TLLM_BENCHMARK_REQ_QUEUES_SIZE={concurrency}') + f'TLLM_BENCHMARK_REQ_QUEUES_SIZE={queue_size}') # 2. Add profiling env vars to env_config (conditional) if nsys_on: From 9cce9cf7c42e9c87cd4ea0b4eca83f508abb8642 Mon Sep 17 00:00:00 2001 From: Lizhi Zhou <1432185+reasonsolo@users.noreply.github.com> Date: Wed, 15 Apr 2026 00:01:40 -0700 Subject: [PATCH 2/2] [None][fix] Fix type bug and formatting in queue size cap logic Cast concurrency, max_batch_size, and tp_size to int to prevent string arithmetic when concurrency comes from split(). Add explicit parentheses around the ternary for clarity. Run yapf to satisfy pre-commit formatting checks. Signed-off-by: Lizhi Zhou <1432185+reasonsolo@users.noreply.github.com> --- examples/disaggregated/slurm/benchmark/submit.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/examples/disaggregated/slurm/benchmark/submit.py b/examples/disaggregated/slurm/benchmark/submit.py index 6565b8121181..ba470d71c7c0 100644 --- a/examples/disaggregated/slurm/benchmark/submit.py +++ b/examples/disaggregated/slurm/benchmark/submit.py @@ -229,15 +229,19 @@ def build_worker_environment(worker_config, env_config, role, benchmark_mode, 'TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP=1') if role == "GEN": gen_config = worker_config.get('gen', {}) - max_batch_size = gen_config.get('max_batch_size', concurrency) + concurrency_int = int(concurrency) + max_batch_size = int( + gen_config.get('max_batch_size', concurrency_int)) enable_attention_dp = gen_config.get('enable_attention_dp', False) - tp_size = gen_config.get('tensor_parallel_size', 1) - max_capacity = max_batch_size * tp_size if enable_attention_dp else max_batch_size - queue_size = min(max_capacity, int(concurrency)) - if queue_size < int(concurrency): + tp_size = int(gen_config.get('tensor_parallel_size', 1)) + max_capacity = ((max_batch_size * tp_size) + if enable_attention_dp else max_batch_size) + queue_size = min(max_capacity, concurrency_int) + if queue_size < concurrency_int: print(f"[WARNING] TLLM_BENCHMARK_REQ_QUEUES_SIZE capped to " f"{queue_size} (max_batch_size={max_batch_size} x " - f"tp_size={tp_size} with attention_dp={enable_attention_dp}) " + f"tp_size={tp_size} with " + f"attention_dp={enable_attention_dp}) " f"which is less than concurrency={concurrency}. " f"Fill loop would hang if set to {concurrency}.") upsert_env_config(env_config, 'gen_worker_env_var',