perf(agentx): refresh Qwen3.5 GB300 recipes with correct metrics

benchmarks/benchmark_lib.sh

@@ -2006,6 +2006,36 @@ write_agentic_result_json() {
     "$AIPERF_PYTHON" "$INFMAX_CONTAINER_WORKSPACE/utils/generate_aiperf_plots.py" "$result_dir" 2>&1 || true
 }
 
+validate_required_agentic_server_metrics() {
+    local result_dir="$1"
+    local required_prefix="${AIPERF_REQUIRED_SERVER_METRIC_PREFIX:-}"
+    local metrics_dir="$result_dir/aiperf_artifacts"
+    local metrics_json="$metrics_dir/server_metrics_export.json"
+    local metrics_csv="$metrics_dir/server_metrics_export.csv"
+
+    # Opt-in so existing AgentX configurations retain their current contract.
+    # Recipes that require trace charts set a metric prefix (for example
+    # `sglang:`) and fail loudly instead of publishing a partial trace artifact.
+    if [ -z "$required_prefix" ]; then
+        return 0
+    fi
+
+    if [ ! -s "$metrics_json" ] || [ ! -s "$metrics_csv" ]; then
+        echo "ERROR: required AIPerf server metrics artifacts are missing or empty in $metrics_dir" >&2
+        return 1
+    fi
+
+    # Avoid parsing the potentially multi-GiB JSON into memory. AIPerf writes
+    # metric names as JSON object keys, so a fixed-string scan establishes that
+    # backend engine metrics—not only frontend/router metrics—were captured.
+    if ! grep -F -m 1 -q "\"${required_prefix}" "$metrics_json"; then
+        echo "ERROR: $metrics_json contains no metric with required prefix '$required_prefix'" >&2
+        return 1
+    fi
+
+    echo "Validated required AIPerf server metrics prefix '$required_prefix'"
+}
+
 run_agentic_replay_and_write_outputs() {
     local result_dir="$1"
     local replay_rc
@@ -2044,4 +2074,6 @@ run_agentic_replay_and_write_outputs() {
         echo "ERROR: agentic trace replay produced invalid results after writing available artifacts" >&2
         return "$validation_rc"
     fi
+
+    validate_required_agentic_server_metrics "$result_dir"
 }

benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c1-mtp-hicache-jid2530006.yaml

@@ -0,0 +1,91 @@
+name: agg-gb300-tp2-c1-mtp-hicache-jid2530006
+model:
+  path: qwen3.5-fp4
+  container: dynamo-sglang
+  precision: fp4
+slurm:
+  time_limit: '8:00:00'
+health_check:
+  max_attempts: 1440
+  interval_seconds: 10
+resources:
+  gpu_type: gb300
+  gpus_per_node: 2
+  agg_nodes: 1
+  agg_workers: 1
+  gpus_per_agg: 2
+infra:
+  nats_max_payload_mb: 8
+frontend:
+  type: sglang
+backend:
+  aggregated_environment:
+    NCCL_NVLS_ENABLE: '1'
+    PYTHONNOUSERSITE: '1'
+    SGLANG_ENABLE_FLASHINFER_GEMM: 'true'
+    SGLANG_SIMULATE_ACC_LEN: '3.39'
+    SGLANG_SIMULATE_ACC_METHOD: match-expected
+    SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token
+    SGL_ENABLE_JIT_DEEPGEMM: 'false'
+    TORCH_CUDA_ARCH_LIST: '10.0'
+  sglang_config:
+    aggregated:
+      served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4
+      enable-cache-report: true
+      enable-metrics: true
+      model-path: /model/
+      trust-remote-code: true
+      tensor-parallel-size: 2
+      data-parallel-size: 1
+      expert-parallel-size: 1
+      enable-symm-mem: true
+      quantization: modelopt_fp4
+      kv-cache-dtype: fp8_e4m3
+      mamba-ssm-dtype: bfloat16
+      mamba-scheduler-strategy: extra_buffer
+      mamba-track-interval: 8192
+      attention-backend: trtllm_mha
+      moe-runner-backend: flashinfer_trtllm
+      speculative-algorithm: NEXTN
+      speculative-num-steps: 3
+      speculative-eagle-topk: 1
+      speculative-num-draft-tokens: 4
+      cuda-graph-max-bs: 80
+      max-running-requests: 72
+      max-prefill-tokens: 16384
+      chunked-prefill-size: 16384
+      mem-fraction-static: 0.75
+      max-mamba-cache-size: 360
+      allow-auto-truncate: true
+      stream-interval: 50
+      scheduler-recv-interval: 10
+      tokenizer-worker-num: 6
+      page-size: 64
+      enable-hierarchical-cache: true
+      hicache-ratio: 0.9
+      hicache-io-backend: kernel
+      hicache-mem-layout: page_first_direct
+      hicache-write-policy: write_back
+      mamba-max-states-per-path: 1
+  type: sglang
+sbatch_directives:
+  mem: '0'
+  cpus-per-task: '144'
+srun_options:
+  mem: '0'
+  container-remap-root: ''
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
+    RESULT_DIR: /logs/agentic
+    PORT: '8000'
+    IS_MULTINODE: 'false'
+    TP: '2'
+    WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k
+    AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
+    HF_HUB_CACHE: /hf_hub_cache
+    AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0'
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:'

benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c24-mtp-hicache-jid2530012.yaml

@@ -0,0 +1,91 @@
+name: agg-gb300-tp2-c24-mtp-hicache-jid2530012
+model:
+  path: qwen3.5-fp4
+  container: dynamo-sglang
+  precision: fp4
+slurm:
+  time_limit: '8:00:00'
+health_check:
+  max_attempts: 1440
+  interval_seconds: 10
+resources:
+  gpu_type: gb300
+  gpus_per_node: 2
+  agg_nodes: 1
+  agg_workers: 1
+  gpus_per_agg: 2
+infra:
+  nats_max_payload_mb: 8
+frontend:
+  type: sglang
+backend:
+  aggregated_environment:
+    NCCL_NVLS_ENABLE: '1'
+    PYTHONNOUSERSITE: '1'
+    SGLANG_ENABLE_FLASHINFER_GEMM: 'true'
+    SGLANG_SIMULATE_ACC_LEN: '3.39'
+    SGLANG_SIMULATE_ACC_METHOD: match-expected
+    SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token
+    SGL_ENABLE_JIT_DEEPGEMM: 'false'
+    TORCH_CUDA_ARCH_LIST: '10.0'
+  sglang_config:
+    aggregated:
+      served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4
+      enable-cache-report: true
+      enable-metrics: true
+      model-path: /model/
+      trust-remote-code: true
+      tensor-parallel-size: 2
+      data-parallel-size: 1
+      expert-parallel-size: 1
+      enable-symm-mem: true
+      quantization: modelopt_fp4
+      kv-cache-dtype: fp8_e4m3
+      mamba-ssm-dtype: bfloat16
+      mamba-scheduler-strategy: extra_buffer
+      mamba-track-interval: 8192
+      attention-backend: trtllm_mha
+      moe-runner-backend: flashinfer_trtllm
+      speculative-algorithm: NEXTN
+      speculative-num-steps: 3
+      speculative-eagle-topk: 1
+      speculative-num-draft-tokens: 4
+      cuda-graph-max-bs: 80
+      max-running-requests: 72
+      max-prefill-tokens: 16384
+      chunked-prefill-size: 16384
+      mem-fraction-static: 0.75
+      max-mamba-cache-size: 360
+      allow-auto-truncate: true
+      stream-interval: 50
+      scheduler-recv-interval: 10
+      tokenizer-worker-num: 6
+      page-size: 64
+      enable-hierarchical-cache: true
+      hicache-ratio: 0.9
+      hicache-io-backend: kernel
+      hicache-mem-layout: page_first_direct
+      hicache-write-policy: write_back
+      mamba-max-states-per-path: 1
+  type: sglang
+sbatch_directives:
+  mem: '0'
+  cpus-per-task: '144'
+srun_options:
+  mem: '0'
+  container-remap-root: ''
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
+    RESULT_DIR: /logs/agentic
+    PORT: '8000'
+    IS_MULTINODE: 'false'
+    TP: '2'
+    WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k
+    AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
+    HF_HUB_CACHE: /hf_hub_cache
+    AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0'
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:'

benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c32-mtp-hicache-jid2530013.yaml

@@ -0,0 +1,91 @@
+name: agg-gb300-tp2-c32-mtp-hicache-jid2530013
+model:
+  path: qwen3.5-fp4
+  container: dynamo-sglang
+  precision: fp4
+slurm:
+  time_limit: '8:00:00'
+health_check:
+  max_attempts: 1440
+  interval_seconds: 10
+resources:
+  gpu_type: gb300
+  gpus_per_node: 2
+  agg_nodes: 1
+  agg_workers: 1
+  gpus_per_agg: 2
+infra:
+  nats_max_payload_mb: 8
+frontend:
+  type: sglang
+backend:
+  aggregated_environment:
+    NCCL_NVLS_ENABLE: '1'
+    PYTHONNOUSERSITE: '1'
+    SGLANG_ENABLE_FLASHINFER_GEMM: 'true'
+    SGLANG_SIMULATE_ACC_LEN: '3.39'
+    SGLANG_SIMULATE_ACC_METHOD: match-expected
+    SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token
+    SGL_ENABLE_JIT_DEEPGEMM: 'false'
+    TORCH_CUDA_ARCH_LIST: '10.0'
+  sglang_config:
+    aggregated:
+      served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4
+      enable-cache-report: true
+      enable-metrics: true
+      model-path: /model/
+      trust-remote-code: true
+      tensor-parallel-size: 2
+      data-parallel-size: 1
+      expert-parallel-size: 1
+      enable-symm-mem: true
+      quantization: modelopt_fp4
+      kv-cache-dtype: fp8_e4m3
+      mamba-ssm-dtype: bfloat16
+      mamba-scheduler-strategy: extra_buffer
+      mamba-track-interval: 8192
+      attention-backend: trtllm_mha
+      moe-runner-backend: flashinfer_trtllm
+      speculative-algorithm: NEXTN
+      speculative-num-steps: 3
+      speculative-eagle-topk: 1
+      speculative-num-draft-tokens: 4
+      cuda-graph-max-bs: 80
+      max-running-requests: 72
+      max-prefill-tokens: 16384
+      chunked-prefill-size: 16384
+      mem-fraction-static: 0.75
+      max-mamba-cache-size: 360
+      allow-auto-truncate: true
+      stream-interval: 50
+      scheduler-recv-interval: 10
+      tokenizer-worker-num: 6
+      page-size: 64
+      enable-hierarchical-cache: true
+      hicache-ratio: 0.9
+      hicache-io-backend: kernel
+      hicache-mem-layout: page_first_direct
+      hicache-write-policy: write_back
+      mamba-max-states-per-path: 1
+  type: sglang
+sbatch_directives:
+  mem: '0'
+  cpus-per-task: '144'
+srun_options:
+  mem: '0'
+  container-remap-root: ''
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
+    RESULT_DIR: /logs/agentic
+    PORT: '8000'
+    IS_MULTINODE: 'false'
+    TP: '2'
+    WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k
+    AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
+    HF_HUB_CACHE: /hf_hub_cache
+    AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0'
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:'