diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c40-b3-k7-mtp-hicache-nightly-20260901.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c40-b3-k7-mtp-hicache-nightly-20260901.yaml new file mode 100644 index 0000000000..d755038bc7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c40-b3-k7-mtp-hicache-nightly-20260901.yaml @@ -0,0 +1,159 @@ +base: + name: agg-b300-tp2-c40-b3-k7-mtp-hicache-nightly-20260901 + model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 + precision: fp4 + slurm: + time_limit: '08:00:00' + identity: + model: { repo: nvidia/Qwen3.5-397B-A17B-NVFP4 } + container: { image: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 } + frameworks: { sglang: nightly-dev-cu13-20260901-07c8f729 } + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: b300 + gpus_per_node: 8 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 2 + infra: + nats_max_payload_mb: 8 + frontend: + type: sglang + enable_multiple_frontends: false + backend: + type: sglang + aggregated_environment: + SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + MC_FORCE_MNNVL: '1' + NVSHMEM_REMOTE_TRANSPORT: none + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c40-mrr3-k7/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c40-mrr3-k7/flashinfer-cache + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c40-mrr3-k7/sglang-cache + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' + SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 1 + moe-dense-tp-size: 2 + enable-dp-attention: false + enable-dp-lm-head: false + moe-a2a-backend: none + load-balance-method: round_robin + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 8192 + mamba-max-states-per-path: 3 + mamba-ssm-dtype: bfloat16 + max-mamba-cache-size: 1536 + context-length: 262144 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: triton + disable-shared-experts-fusion: true + speculative-algorithm: NEXTN + speculative-num-steps: 7 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 8 + speculative-moe-runner-backend: flashinfer_trtllm + speculative-moe-a2a-backend: none + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + prefill-decode-interval: 0 + mem-fraction-static: 0.85 + max-running-requests: 3 + pp-max-micro-batch-size: 3 + prefill-max-requests: 3 + cuda-graph-max-bs-decode: 3 + cuda-graph-bs-decode: + - 1 + - 2 + - 3 + stream-interval: 20 + decode-log-interval: 10 + watchdog-timeout: 1000000 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-size: 128 + enable-metrics: true + enable-linear-replayssm-spec: true + disable-attn-tp-gather: true + sbatch_directives: + mem: '0' + cpus-per-task: '144' + srun_options: + mem: '0' + container-remap-root: '' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# Coupled scheduler/graph caps are zipped to preserve one controlled physical +# batch size per variant. The base remains the already measured B3 arm. +zip_override_running_cap: + name: + - agg-b300-tp2-c40-b4-k7-mtp-hicache-nightly-20260901 + - agg-b300-tp2-c40-b7-k7-mtp-hicache-nightly-20260901 + - agg-b300-tp2-c40-b8-k7-mtp-hicache-nightly-20260901 + backend: + sglang_config: + aggregated: + max-running-requests: [4, 7, 8] + pp-max-micro-batch-size: [4, 7, 8] + prefill-max-requests: [4, 7, 8] + cuda-graph-max-bs-decode: [4, 7, 8] + cuda-graph-bs-decode: + - [1, 2, 3, 4] + - [1, 2, 3, 4, 5, 6, 7] + - [1, 2, 3, 4, 5, 6, 7, 8] diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c44-b2-k5-mtp-hicache-nightly-20260901.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c44-b2-k5-mtp-hicache-nightly-20260901.yaml new file mode 100644 index 0000000000..17280c0585 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c44-b2-k5-mtp-hicache-nightly-20260901.yaml @@ -0,0 +1,138 @@ +name: agg-b300-tp2-c44-b2-k5-mtp-hicache-nightly-20260901 +model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 + precision: fp4 +slurm: + time_limit: '08:00:00' +identity: + model: { repo: nvidia/Qwen3.5-397B-A17B-NVFP4 } + container: { image: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 } + frameworks: { sglang: nightly-dev-cu13-20260901-07c8f729 } +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 2 +infra: + nats_max_payload_mb: 8 +frontend: + type: sglang + enable_multiple_frontends: false +backend: + type: sglang + aggregated_environment: + SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + MC_FORCE_MNNVL: '1' + NVSHMEM_REMOTE_TRANSPORT: none + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2-k5/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2-k5/flashinfer-cache + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2-k5/sglang-cache + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' + SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 1 + moe-dense-tp-size: 2 + enable-dp-attention: false + enable-dp-lm-head: false + moe-a2a-backend: none + load-balance-method: round_robin + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 8192 + mamba-max-states-per-path: 3 + mamba-ssm-dtype: bfloat16 + max-mamba-cache-size: 1536 + context-length: 262144 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: triton + disable-shared-experts-fusion: true + speculative-algorithm: NEXTN + speculative-num-steps: 5 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 6 + speculative-moe-runner-backend: flashinfer_trtllm + speculative-moe-a2a-backend: none + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + prefill-decode-interval: 0 + mem-fraction-static: 0.85 + max-running-requests: 2 + pp-max-micro-batch-size: 2 + prefill-max-requests: 2 + cuda-graph-max-bs-decode: 2 + cuda-graph-bs-decode: + - 1 + - 2 + stream-interval: 20 + decode-log-interval: 10 + watchdog-timeout: 1000000 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-size: 128 + enable-metrics: true + enable-linear-replayssm-spec: true + disable-attn-tp-gather: true +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2ep2-highthroughput-nightly-20260901.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2ep2-highthroughput-nightly-20260901.yaml new file mode 100644 index 0000000000..9a1ca27d71 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2ep2-highthroughput-nightly-20260901.yaml @@ -0,0 +1,130 @@ +name: agg-b300-tp2ep2-highthroughput-nightly-20260901 +model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 + precision: fp4 +slurm: + time_limit: '08:00:00' +identity: + model: { repo: nvidia/Qwen3.5-397B-A17B-NVFP4 } + container: { image: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 } + frameworks: { sglang: nightly-dev-cu13-20260901-07c8f729 } +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 2 +infra: + nats_max_payload_mb: 8 +frontend: + type: sglang + enable_multiple_frontends: false +backend: + type: sglang + aggregated_environment: + SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '2' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + MC_FORCE_MNNVL: '1' + NVSHMEM_REMOTE_TRANSPORT: none + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/b300-tp2ep2-highthroughput/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/b300-tp2ep2-highthroughput/flashinfer-cache + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/b300-tp2ep2-highthroughput/sglang-cache + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' + SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 2 + enable-dp-attention: false + enable-dp-lm-head: false + enable-symm-mem: false + mamba-ssm-dtype: bfloat16 + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 1048576 + mamba-max-states-per-path: 1 + max-mamba-cache-size: 512 + context-length: 262144 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_cutedsl + linear-attn-prefill-backend: flashinfer + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + speculative-moe-runner-backend: flashinfer_trtllm + enable-linear-replayssm-spec: true + linear-replayssm-cache-len: 8 + mem-fraction-static: 0.85 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + max-running-requests: 224 + cuda-graph-max-bs: 224 + tokenizer-worker-num: 6 + stream-interval: 50 + scheduler-recv-interval: 10 + allow-auto-truncate: true + enable-hierarchical-cache: true + hicache-ratio: 1.8 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_back + watchdog-timeout: 1000000 + enable-metrics: true + enable-dynamic-chunking: false + weight-loader-drop-cache-after-load: true +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp4-lowlat-nightly-20260901.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp4-lowlat-nightly-20260901.yaml new file mode 100644 index 0000000000..79df728bd2 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp4-lowlat-nightly-20260901.yaml @@ -0,0 +1,153 @@ +base: + name: agg-b300-tp4-b1-lowlat-nightly-20260901 + model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 + precision: fp4 + slurm: + time_limit: '08:00:00' + identity: + model: { repo: nvidia/Qwen3.5-397B-A17B-NVFP4 } + container: { image: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 } + frameworks: { sglang: nightly-dev-cu13-20260901-07c8f729 } + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: b300 + gpus_per_node: 8 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 4 + infra: + nats_max_payload_mb: 8 + frontend: + type: sglang + enable_multiple_frontends: false + backend: + type: sglang + aggregated_environment: + SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + MC_FORCE_MNNVL: '1' + NVSHMEM_REMOTE_TRANSPORT: none + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/b300-tp4-lowlat/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/b300-tp4-lowlat/flashinfer-cache + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/b300-tp4-lowlat/sglang-cache + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' + SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 1 + moe-dense-tp-size: 4 + enable-dp-attention: false + enable-dp-lm-head: false + moe-a2a-backend: none + load-balance-method: round_robin + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 8192 + mamba-max-states-per-path: 3 + mamba-ssm-dtype: bfloat16 + max-mamba-cache-size: 512 + context-length: 262144 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_cutedsl + linear-attn-decode-backend: triton + disable-shared-experts-fusion: true + speculative-algorithm: NEXTN + speculative-num-steps: 6 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + speculative-moe-runner-backend: flashinfer_cutedsl + speculative-moe-a2a-backend: none + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + mem-fraction-static: 0.85 + max-running-requests: 1 + pp-max-micro-batch-size: 1 + prefill-max-requests: 1 + cuda-graph-max-bs-decode: 1 + cuda-graph-bs-decode: + - 1 + disable-prefill-cuda-graph: true + stream-interval: 20 + decode-log-interval: 10 + watchdog-timeout: 1000000 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-size: 32 + enable-metrics: true + enable-linear-replayssm-spec: true + disable-attn-tp-gather: true + sbatch_directives: + mem: '0' + cpus-per-task: '144' + srun_options: + mem: '0' + container-remap-root: '' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '4' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# The B2 variant is coupled so scheduler residency, microbatch/prefill caps, +# and decode graph coverage always describe the same physical batch size. +zip_override_batch_cap: + name: + - agg-b300-tp4-b2-lowlat-nightly-20260901 + backend: + sglang_config: + aggregated: + max-running-requests: [2] + pp-max-micro-batch-size: [2] + prefill-max-requests: [2] + cuda-graph-max-bs-decode: [2] + cuda-graph-bs-decode: + - [1, 2] diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 526fbec480..7d11dec5eb 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7353,6 +7353,99 @@ qwen3.5-fp4-b300-sglang-agentic-mtp: - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [36, 44, 52] } +# Qwen3.5 NVFP4 B300 Dynamo/SGLang AgentX variants using the pinned +# SGLang image and committed golden acceptance-length curve. +qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto: + image: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 + model: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-prefix: qwen3.5 + runner: cluster:b300-dsxe + precision: fp4 + framework: dynamo-sglang + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.90 + search-space: + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [64] + num-nodes: 1 + worker: + num-worker: 1 + tp: 2 + pp: 1 + ep: 2 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.39" + - "CONFIG_FILE=recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2ep2-highthroughput-nightly-20260901.yaml" + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [44] + num-nodes: 1 + worker: + num-worker: 1 + tp: 2 + pp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=5.04" + - "CONFIG_FILE=recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c40-b3-k7-mtp-hicache-nightly-20260901.yaml:zip_override_running_cap[2]" + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [36] + num-nodes: 1 + worker: + num-worker: 1 + tp: 2 + pp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=5.04" + - "CONFIG_FILE=recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c40-b3-k7-mtp-hicache-nightly-20260901.yaml:zip_override_running_cap[0]" + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [40] + num-nodes: 1 + worker: + num-worker: 1 + tp: 2 + pp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=4.41" + - "CONFIG_FILE=recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c44-b2-k5-mtp-hicache-nightly-20260901.yaml" + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [4] + num-nodes: 1 + worker: + num-worker: 1 + tp: 4 + pp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=4.80" + - "CONFIG_FILE=recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp4-lowlat-nightly-20260901.yaml:base" + + + # Qwen3.8-Flash-Next NVFP4 AgentX on B300 via SGLang with native NEXTN MTP. # Day-zero recipe; mirrors the B200 arm. TP1 per the cookbook's verified # single-node command: 126 GiB of NVFP4 weights fit on one B300. diff --git a/golden_al_distribution/README.md b/golden_al_distribution/README.md index 061851eee9..e717f460d3 100644 --- a/golden_al_distribution/README.md +++ b/golden_al_distribution/README.md @@ -24,6 +24,13 @@ AL is workload-dependent: a draft model's predictions are easier to accept in so Under the AgentX Guidelines, each model, thinking mode, and draft length has one committed golden AL. Once synthetic acceptance is enabled for a benchmark scenario, a submission may choose any supported draft length, but it may not substitute a different acceptance target. Different models keep their own SPEED-Bench-derived curves. All submissions evaluating the same model and mode use the same curve. +For Qwen3.5 srt-slurm launches, `runners/inject_synthetic_acceptance.py` +resolves the committed curve by the recipe's `speculative-num-steps`, even when +`SYNTHETIC_ACCEPTANCE_LENGTH` is explicitly set. The launch fails if that value +does not match the golden row, if `NUM_SPEC_TOKENS` disagrees with the recipe, +or if `speculative-num-draft-tokens` is not `steps + 1`. The verification token +must not be counted as an additional MTP step. + vLLM supports this through synthetic rejection sampling. For example, an EAGLE3 run can inject the selected YAML value through `synthetic_acceptance_length`: ```bash diff --git a/golden_al_distribution/README_zh.md b/golden_al_distribution/README_zh.md index 79474a4787..04462b74a1 100644 --- a/golden_al_distribution/README_zh.md +++ b/golden_al_distribution/README_zh.md @@ -24,6 +24,8 @@ AL 取决于工作负载：草稿模型的预测在某些领域比其他领域 根据 AgentX 指南，每个模型、思考模式和草稿长度都有一个已提交的黄金 AL。当某个基准场景启用合成接受后，提交可以选择任意受支持的草稿长度，但不能替换为其他接受目标。不同模型保留各自基于 SPEED-Bench 测得的曲线。所有评估同一模型和模式的提交都使用同一条曲线。 +对于 Qwen3.5 的 srt-slurm 启动，`runners/inject_synthetic_acceptance.py` 即使在显式设置 `SYNTHETIC_ACCEPTANCE_LENGTH` 时，也会根据 recipe 中的 `speculative-num-steps` 查询已提交曲线。若该值与黄金表对应行不一致、`NUM_SPEC_TOKENS` 与 recipe 不一致，或 `speculative-num-draft-tokens` 不等于 `steps + 1`，启动都会失败。验证 token 不能额外计为一个 MTP 步数。 + vLLM 通过合成拒绝采样支持这一策略。例如，EAGLE3 运行可以通过 `synthetic_acceptance_length` 注入所选 YAML 值： ```bash diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b0dc6a14d4..0ef31de8b1 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7145,3 +7145,12 @@ - "Update SGLang image from lmsysorg/sglang:nightly-dev-cu13-20260907-30705c00 (2026-09-07 cu13 dev nightly, build commit sgl-project/sglang@30705c00) to the v0.5.19 release image lmsysorg/sglang:v0.5.19-cu130 (digest sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9, build commit sgl-project/sglang@0bcd822377da7b5718e674eaf9c870d349424dd1, Docker Hub last pushed 2026-09-04T22:50:19Z)." - "The release image ships the same CUDA 13.0.3, FlashInfer 0.6.18 and sgl-kernel 0.4.6.post1 as the nightly. benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh is unchanged: SGLANG_ENABLE_SPEC_V2 EAGLE MTP at 3 steps, golden acceptance length 3.39, flashinfer attention with allreduce fusion, fp8 quantization and fp8_e4m3 KV, HiCache kernel IO / page_first layout. TP8/EP1 DRAM HiCache concurrency 2 through 24 unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2966 + +- config-keys: + - qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto + scenario-type: + - agentic-coding + description: + - "Add a Qwen3.5 NVFP4 B300 Dynamo-SGLang AgentX configuration with aggregate MTP recipes for TP2/EP2, TP2, and TP4 worker layouts." + - "Use the pinned lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 image and validate each recipe's MTP step count against the committed Qwen3.5 thinking-on golden acceptance-length curve." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2978 diff --git a/runners/inject_synthetic_acceptance.py b/runners/inject_synthetic_acceptance.py index ab10bb1126..3507538af5 100644 --- a/runners/inject_synthetic_acceptance.py +++ b/runners/inject_synthetic_acceptance.py @@ -26,6 +26,7 @@ python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" "${CONFIG_FILE%%:*}" "$FRAMEWORK" """ +import math import os import sys @@ -41,6 +42,16 @@ "dsv4dsparkprob": "deepseek-v4-pro-0813", } +# Committed golden curves are the authority for AgentX synthetic acceptance. +# Keep this mapping explicit: a model prefix must select one unambiguous curve +# (some models have multiple draft heads / sampling methods). +GOLDEN_AL_REFERENCES = { + "qwen3.5": ( + "qwen3.5_mtp.yaml", + "qwen3.5-397b-a17b-nvfp4", + ), +} + def _log(msg): print(f"[Synthetic AR] {msg}") @@ -87,41 +98,107 @@ def _lookup_al(model_block, num_spec_tokens): return None -def _resolve_al(config_text, injector, ref_yaml): +def _load_reference(ref_yaml, yaml_key, num_spec_tokens): + if not os.path.isfile(ref_yaml): + sys.exit(f"ERROR: golden acceptance reference YAML not found: {ref_yaml}") + + import yaml + + with open(ref_yaml) as f: + data = yaml.safe_load(f) + + model_block = data.get(yaml_key) + if model_block is None: + sys.exit(f'ERROR: model key "{yaml_key}" not found in {ref_yaml}') + + al = _lookup_al(model_block, num_spec_tokens) + if al is None: + sys.exit( + f"ERROR: num_spec_tokens={num_spec_tokens} not found for " + f"{yaml_key} in {ref_yaml}" + ) + return float(al) + + +def _resolve_al(config_text, injector, legacy_ref_yaml): + model_prefix = os.environ.get("MODEL_PREFIX", "").strip() explicit = os.environ.get("SYNTHETIC_ACCEPTANCE_LENGTH", "").strip() + + nst_env = os.environ.get("NUM_SPEC_TOKENS", "").strip() + recipe_spec_tokens = injector.spec_tokens_from_recipe(config_text) + + # Curves listed here are fail closed. In particular, an explicit AL must + # not bypass the recipe's actual speculative step count and silently select + # the next row of the golden table. + reference = GOLDEN_AL_REFERENCES.get(model_prefix) + if reference: + if recipe_spec_tokens is None: + sys.exit( + "ERROR: cannot validate golden AL because the recipe does not " + "declare a speculative-token count" + ) + validator = getattr(injector, "validate_speculative_shape", None) + if validator is not None: + validator(config_text) + if nst_env and int(nst_env) != recipe_spec_tokens: + sys.exit( + f"ERROR: NUM_SPEC_TOKENS={nst_env} disagrees with recipe " + f"speculative-num-steps={recipe_spec_tokens}" + ) + + filename, key = reference + ref_yaml = os.path.join( + os.path.dirname(__file__), "..", "golden_al_distribution", filename + ) + golden_al = _load_reference(ref_yaml, key, recipe_spec_tokens) + if explicit and not math.isclose( + float(explicit), golden_al, rel_tol=0.0, abs_tol=1e-9 + ): + sys.exit( + f"ERROR: explicit SYNTHETIC_ACCEPTANCE_LENGTH={explicit} does " + f"not match golden AL={golden_al:g} for MODEL_PREFIX={model_prefix}, " + f"num_spec_tokens={recipe_spec_tokens} ({ref_yaml})" + ) + _log( + f"Verified golden AL={golden_al:g} from {ref_yaml} " + f"(model={key}, num_spec_tokens={recipe_spec_tokens})" + ) + return golden_al + + # Preserve the legacy reference format for model families that have not yet + # been assigned an unambiguous committed curve above. if explicit: return float(explicit) - if not os.path.isfile(ref_yaml): + if not os.path.isfile(legacy_ref_yaml): sys.exit( "ERROR: SYNTHETIC_ACCEPTANCE_LENGTH not set and reference YAML not " - f"found: {ref_yaml}" + f"found: {legacy_ref_yaml}" ) - import yaml # local import: only needed on the auto-lookup path + import yaml - with open(ref_yaml) as f: + with open(legacy_ref_yaml) as f: data = yaml.safe_load(f) - key = _yaml_key(os.environ.get("MODEL_PREFIX", "")) + key = _yaml_key(model_prefix) model_block = data.get(key) if model_block is None: - sys.exit(f'ERROR: model key "{key}" not found in {ref_yaml}') + sys.exit(f'ERROR: model key "{key}" not found in {legacy_ref_yaml}') - nst_env = os.environ.get("NUM_SPEC_TOKENS", "").strip() - if nst_env: - num_spec_tokens = int(nst_env) - else: - num_spec_tokens = injector.spec_tokens_from_recipe(config_text) or 2 + num_spec_tokens = ( + int(nst_env) if nst_env else (recipe_spec_tokens if recipe_spec_tokens else 2) + ) al = _lookup_al(model_block, num_spec_tokens) if al is None: sys.exit( - f"ERROR: num_spec_tokens={num_spec_tokens} not found for {key} in {ref_yaml}" + f"ERROR: num_spec_tokens={num_spec_tokens} not found for {key} in " + f"{legacy_ref_yaml}" ) _log( - f"Auto-resolved AL={al} from {ref_yaml} " + f"Auto-resolved AL={al} from {legacy_ref_yaml} " f"(model={key}, num_spec_tokens={num_spec_tokens})" ) return float(al) diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index 70254deb65..7dd339d4d4 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -63,8 +63,10 @@ declare -A MODEL_ALIASES=( [nvidia/MiniMax-M3-NVFP4]="MiniMax-M3-NVFP4" [minimax-m3-mxfp8]="MiniMax-M3-MXFP8" [MiniMaxAI/MiniMax-M3-MXFP8]="MiniMax-M3-MXFP8" - [qwen3.5-fp4]="Qwen3.5-397B-A17B-NVFP4-V2" + [qwen3.5-fp4]="Qwen3.5-397B-A17B-NVFP4" + [nvidia/Qwen3.5-397B-A17B-NVFP4]="Qwen3.5-397B-A17B-NVFP4" [qwen3.5-fp8]="Qwen3.5-397B-A17B-FP8" + [qwen3.5-fp4-v2]="Qwen3.5-397B-A17B-NVFP4-V2" [nvidia/Qwen3.5-397B-A17B-NVFP4-V2]="Qwen3.5-397B-A17B-NVFP4-V2" ) @@ -280,10 +282,12 @@ fi # Override the job name in the recipe with the runner name. sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH" -if [[ "${EVAL_ONLY:-false}" == "true" ]]; then - python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" \ - "$CONFIG_PATH" "$FRAMEWORK" || exit 1 -fi +# Throughput recipes opt into synthetic acceptance through the master config. +# Eval-only jobs remove those settings so generated tokens use real target-model +# verification. The injector handles both modes and is a no-op for recipes that +# do not opt in. +python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" \ + "$CONFIG_PATH" "$FRAMEWORK" || exit 1 # Weights live on node-local MODEL_ROOT, which this login host cannot stat, so # srtctl's preflight model.path check is always skipped. Runtime loading still diff --git a/runners/synthetic_injectors/sglang.py b/runners/synthetic_injectors/sglang.py index d5cd6e0b16..793f3e9a61 100644 --- a/runners/synthetic_injectors/sglang.py +++ b/runners/synthetic_injectors/sglang.py @@ -6,7 +6,12 @@ from . import register _SPEC_STEPS_RE = re.compile(r"(?m)^\s+speculative-num-steps:\s*([0-9]+)\s*$") -_ENV_BLOCK_RE = re.compile(r"(?m)^( (?:aggregated|prefill|decode)_environment:\s*)$") +_SPEC_DRAFT_TOKENS_RE = re.compile( + r"(?m)^\s+speculative-num-draft-tokens:\s*([0-9]+)\s*$" +) +_ENV_BLOCK_RE = re.compile( + r"(?m)^([ \t]+)((?:aggregated|prefill|decode)_environment:\s*)$" +) _SIMULATED_ACCEPTANCE_ENV_RE = re.compile( r"(?m)^[ \t]+SGLANG_SIMULATE_ACC_(?:LEN|METHOD|TOKEN_MODE):[^\n]*(?:\n|$)" ) @@ -14,8 +19,29 @@ def spec_tokens_from_recipe(text): """Read SGLang's speculative step count from the recipe.""" - match = _SPEC_STEPS_RE.search(text) - return int(match.group(1)) if match else None + values = {int(match) for match in _SPEC_STEPS_RE.findall(text)} + if len(values) > 1: + raise ValueError( + f"recipe declares multiple speculative-num-steps values: {sorted(values)}" + ) + return next(iter(values)) if values else None + + +def validate_speculative_shape(text): + """Fail if SGLang's verification window is not steps + one bonus token.""" + steps = spec_tokens_from_recipe(text) + drafts = {int(match) for match in _SPEC_DRAFT_TOKENS_RE.findall(text)} + if steps is None or len(drafts) != 1: + raise ValueError( + "recipe must declare one unambiguous speculative-num-steps and " + "speculative-num-draft-tokens value" + ) + draft_tokens = next(iter(drafts)) + if draft_tokens != steps + 1: + raise ValueError( + f"speculative-num-draft-tokens={draft_tokens} must equal " + f"speculative-num-steps + 1 ({steps + 1})" + ) def rewrite(content, al, log): @@ -23,13 +49,17 @@ def rewrite(content, al, log): if "SGLANG_SIMULATE_ACC_LEN" in content: raise ValueError("recipe already contains SGLANG_SIMULATE_ACC_* variables") - variables = ( - f'\n SGLANG_SIMULATE_ACC_LEN: "{al:g}"' - '\n SGLANG_SIMULATE_ACC_METHOD: "match-expected"' - '\n SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token"' - ) + def add_variables(match): + child_indent = match.group(1) + " " + variables = ( + f'\n{child_indent}SGLANG_SIMULATE_ACC_LEN: "{al:g}"' + f'\n{child_indent}SGLANG_SIMULATE_ACC_METHOD: "match-expected"' + f'\n{child_indent}SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token"' + ) + return match.group(0) + variables + rewritten, count = _ENV_BLOCK_RE.subn( - lambda match: match.group(1) + variables, + add_variables, content, ) if count: diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py index 778b357327..cd33d6b5c8 100644 --- a/runners/test_slurm_utils.py +++ b/runners/test_slurm_utils.py @@ -1,5 +1,6 @@ import json import os +import re import runpy import subprocess from pathlib import Path @@ -485,6 +486,203 @@ def test_eval_only_removes_sglang_simulated_acceptance(tmp_path: Path) -> None: assert environment == {"KEEP_ME": "unchanged"} +def test_sglang_throughput_injects_nested_override_environment( + tmp_path: Path, +) -> None: + recipe = tmp_path / "recipe.yaml" + recipe.write_text( + "base:\n" + " backend:\n" + " aggregated_environment:\n" + " KEEP_ME: unchanged\n" + " sglang_config:\n" + " aggregated:\n" + " speculative-num-steps: 5\n" + " speculative-num-draft-tokens: 6\n" + ) + + result = subprocess.run( + ["python3", str(INJECT_ACCEPTANCE), str(recipe), "dynamo-sglang"], + env={ + **os.environ, + "SYNTHETIC_ACCEPTANCE": "true", + "SYNTHETIC_ACCEPTANCE_LENGTH": "4.41", + "MODEL_PREFIX": "qwen3.5", + }, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + environment = yaml.safe_load(recipe.read_text())["base"]["backend"][ + "aggregated_environment" + ] + assert environment == { + "KEEP_ME": "unchanged", + "SGLANG_SIMULATE_ACC_LEN": "4.41", + "SGLANG_SIMULATE_ACC_METHOD": "match-expected", + "SGLANG_SIMULATE_ACC_TOKEN_MODE": "real-draft-token", + } + + +def test_qwen35_explicit_acceptance_must_match_recipe_golden_al( + tmp_path: Path, +) -> None: + recipe = tmp_path / "recipe.yaml" + original = ( + "base:\n" + " backend:\n" + " aggregated_environment:\n" + " KEEP_ME: unchanged\n" + " sglang_config:\n" + " aggregated:\n" + " speculative-num-steps: 5\n" + " speculative-num-draft-tokens: 6\n" + ) + recipe.write_text(original) + + result = subprocess.run( + ["python3", str(INJECT_ACCEPTANCE), str(recipe), "dynamo-sglang"], + env={ + **os.environ, + "SYNTHETIC_ACCEPTANCE": "true", + "SYNTHETIC_ACCEPTANCE_LENGTH": "4.80", + "MODEL_PREFIX": "qwen3.5", + }, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode != 0 + assert "does not match golden AL=4.41" in result.stderr + assert "num_spec_tokens=5" in result.stderr + assert recipe.read_text() == original + + +def test_qwen35_auto_lookup_uses_committed_golden_distribution( + tmp_path: Path, +) -> None: + recipe = tmp_path / "recipe.yaml" + recipe.write_text( + "base:\n" + " backend:\n" + " aggregated_environment:\n" + " KEEP_ME: unchanged\n" + " sglang_config:\n" + " aggregated:\n" + " speculative-num-steps: 6\n" + " speculative-num-draft-tokens: 7\n" + ) + + result = subprocess.run( + ["python3", str(INJECT_ACCEPTANCE), str(recipe), "dynamo-sglang"], + env={ + **os.environ, + "SYNTHETIC_ACCEPTANCE": "true", + "MODEL_PREFIX": "qwen3.5", + }, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + assert "golden_al_distribution/qwen3.5_mtp.yaml" in result.stdout + assert 'SGLANG_SIMULATE_ACC_LEN: "4.8"' in recipe.read_text() + + +def test_qwen35_acceptance_rejects_wrong_draft_window(tmp_path: Path) -> None: + recipe = tmp_path / "recipe.yaml" + original = ( + "base:\n" + " backend:\n" + " aggregated_environment:\n" + " KEEP_ME: unchanged\n" + " sglang_config:\n" + " aggregated:\n" + " speculative-num-steps: 6\n" + " speculative-num-draft-tokens: 6\n" + ) + recipe.write_text(original) + + result = subprocess.run( + ["python3", str(INJECT_ACCEPTANCE), str(recipe), "dynamo-sglang"], + env={ + **os.environ, + "SYNTHETIC_ACCEPTANCE": "true", + "SYNTHETIC_ACCEPTANCE_LENGTH": "4.80", + "MODEL_PREFIX": "qwen3.5", + }, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode != 0 + assert "must equal speculative-num-steps + 1 (7)" in result.stderr + assert recipe.read_text() == original + + +def test_b300_qwen35_pareto_points_match_thinking_on_golden_al() -> None: + master = yaml.safe_load((REPO_ROOT / "configs/nvidia-master.yaml").read_text()) + points = master["qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto"][ + "scenarios" + ]["agentic-coding"][0]["search-space"] + golden = yaml.safe_load( + (REPO_ROOT / "golden_al_distribution/qwen3.5_mtp.yaml").read_text() + )["qwen3.5-397b-a17b-nvfp4"]["thinking_on"] + recipe_root = REPO_ROOT / "benchmarks/multi_node/srt-slurm-recipes" + expected = { + 64: (3, 3.39), + 44: (7, 5.04), + 36: (7, 5.04), + 40: (5, 4.41), + 4: (6, 4.80), + } + + actual = {} + for point in points: + settings = dict( + item.split("=", 1) + for item in point["worker"]["additional-settings"] + ) + recipe_rel = settings["CONFIG_FILE"].split(":", 1)[0] + recipe_rel = recipe_rel.removeprefix("recipes/") + recipe_text = (recipe_root / recipe_rel).read_text() + steps = { + int(value) + for value in re.findall(r"speculative-num-steps:\s*(\d+)", recipe_text) + } + drafts = { + int(value) + for value in re.findall( + r"speculative-num-draft-tokens:\s*(\d+)", recipe_text + ) + } + assert len(steps) == 1 + assert len(drafts) == 1 + step = next(iter(steps)) + assert next(iter(drafts)) == step + 1 + al = float(settings["SYNTHETIC_ACCEPTANCE_LENGTH"]) + assert al == float(golden[step]) + actual[point["conc-list"][0]] = (step, al) + + assert actual == expected + + +def test_b300_dsxe_acceptance_driver_is_unconditional_and_fail_closed() -> None: + content = (REPO_ROOT / "runners/launch_b300-dsxe.sh").read_text() + command = 'python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py"' + command_index = content.index(command) + + assert content.rfind("\nfi", 0, command_index) > content.rfind( + "\nif ", 0, command_index + ) + assert "|| exit 1" in content[command_index : command_index + 180] + + def test_sglang_throughput_rejects_existing_simulated_acceptance( tmp_path: Path, ) -> None: