Added b300 glm5.2 recipes for agentx

recipes/b300-fp4/glm5.2-agentx-agg.yaml

@@ -0,0 +1,162 @@
+base:
+  name: b300-fp4-glm5.2-agentx-agg
+  model:
+    path: glm-5.2-fp4
+    container: nightly-20260824
+    precision: fp4
+  identity:
+    model:
+      repo: nvidia/GLM-5.2-NVFP4
+      revision: aec724e
+    frameworks:
+      sglang: nightly-dev-cu13-20260824-95f5ecd3
+  resources:
+    gpu_type: b300
+    gpus_per_node: 8
+  dynamo:
+    hash: 71eb001e17fa73c742f0afe1a6ed96836cb135fd
+    install: true
+  backend:
+    type: sglang
+    sglang_config:
+      aggregated:
+        served-model-name: GLM-5.2-FP4
+        trust-remote-code: true
+        quantization: modelopt_fp4
+        kv-cache-dtype: fp8_e4m3
+        tensor-parallel-size: 8
+        data-parallel-size: 1
+        expert-parallel-size: 1
+        enable-dp-attention: false
+        enable-dp-lm-head: false
+        max-running-requests: 10
+        cuda-graph-max-bs: 10
+        chunked-prefill-size: 8192
+        max-prefill-tokens: 8192
+        context-length: 1048576
+        speculative-algorithm: EAGLE
+        speculative-num-steps: 4
+        speculative-eagle-topk: 1
+        speculative-num-draft-tokens: 5
+        nsa-prefill-backend: trtllm
+        nsa-decode-backend: trtllm
+        moe-runner-backend: flashinfer_trtllm
+        fp4-gemm-backend: flashinfer_trtllm
+        disable-shared-experts-fusion: true
+        enable-flashinfer-allreduce-fusion: true
+        weight-loader-prefetch-checkpoints: true
+        model-loader-extra-config: '{"enable_multithread_load": true}'
+        mem-fraction-static: 0.8
+        enable-hierarchical-cache: true
+        hicache-write-policy: write_back
+        hicache-size: 270
+        hicache-io-backend: direct
+        enable-metrics: true
+        enable-cache-report: true
+    aggregated_environment:
+      TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
+      PYTHONUNBUFFERED: '1'
+      DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
+      MC_TE_METRIC: 'true'
+      NCCL_CUMEM_ENABLE: '1'
+      SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
+      SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
+      SGLANG_ENABLE_THINKING: '1'
+      SGLANG_REASONING_EFFORT: max
+      SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1'
+      SGLANG_HICACHE_DEBUG_LOG: '1'
+      SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384'
+      SGLANG_MOE_NVFP4_DISPATCH: '1'
+      SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1'
+      SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8'
+      SGLANG_SIMULATE_ACC_LEN: '3.33'
+      SGLANG_SIMULATE_ACC_METHOD: match-expected
+      SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token
+      PIP_BREAK_SYSTEM_PACKAGES: '1'
+  benchmark:
+    type: custom
+    env:
+      CONCURRENCY: '2'
+      DURATION: '3600'
+      AIPERF_HF_HOME: /hf_home
+      HF_HOME: /hf_home
+      HF_HUB_CACHE: /hf_home/hub
+      HF_DATASETS_CACHE: /hf_home/datasets
+      HF_HUB_OFFLINE: '1'
+      HF_DATASETS_OFFLINE: '1'
+      HF_HUB_DISABLE_XET: '1'
+      XDG_CACHE_HOME: /tmp/cache
+      AIPERF_DATASET_MMAP_CACHE_DIR: /hf_home/mmap_cache
+      AIPERF_DATASET_CONFIGURATION_TIMEOUT: '1800'
+      AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT: '1800'
+      AIPERF_FAILED_REQUEST_THRESHOLD: '0.1'
+      AIPERF_RECORD_STRIP_PAYLOAD_BYTES: 'true'
+      AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'
+      AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:'
+    command: |
+      set -euo pipefail
+      export HF_HOME="${AIPERF_HF_HOME:-/hf_home}"
+      export HF_HUB_CACHE="${HF_HOME}/hub"
+      export HF_DATASETS_CACHE="${HF_HOME}/datasets"
+      mkdir -p "${XDG_CACHE_HOME}" "${AIPERF_DATASET_MMAP_CACHE_DIR}" /logs/aiperf
+      /aiperf-venv/bin/aiperf profile \
+              --scenario inferencex-agentx-mvp \
+              --url http://localhost:8000 \
+              --endpoint /v1/chat/completions \
+              --endpoint-type chat \
+              --streaming \
+              --model GLM-5.2-FP4 \
+              --tokenizer /model \
+              --tokenizer-trust-remote-code \
+              --concurrency "${CONCURRENCY}" \
+              --benchmark-duration "${DURATION}" \
+              --random-seed 42 \
+              --failed-request-threshold "${AIPERF_FAILED_REQUEST_THRESHOLD}" \
+              --stats-interval 30 \
+              --warmup-requests-per-lane 10 \
+              --warmup-grace-period 1800 \
+              --trajectory-start-min-ratio 0.25 \
+              --trajectory-start-max-ratio 0.75 \
+              --use-server-token-count \
+              --no-gpu-telemetry \
+              --num-dataset-entries 393 \
+              --slice-duration 1.0 \
+              --public-dataset semianalysis_cc_traces_weka_062126 \
+              --trace-idle-gap-cap-seconds 300 \
+              --unsafe-override \
+              --ui simple \
+              --server-metrics-formats json \
+              --output-artifact-dir /logs/aiperf
+      rm -f /logs/aiperf/inputs.json
+  health_check:
+    max_attempts: 1440
+    interval_seconds: 10
+  infra:
+    etcd_nats_dedicated_node: false
+    nats_max_payload_mb: 64
+  frontend:
+    type: dynamo
+    env:
+      PIP_BREAK_SYSTEM_PACKAGES: '1'
+    args:
+      router-mode: kv
+      active-decode-blocks-threshold: None
+      active-prefill-tokens-threshold: None
+      active-prefill-tokens-threshold-frac: None
+      router-session-affinity-ttl-secs: 3600
+  sbatch_directives:
+    mem: '0'
+  srun_options:
+    mem: '0'
+    container-remap-root: ''
+# ################# agentx #################
+zip_override_mtp_agentx_lowlat:
+  name: [agentx-agg1_tp8-c2, agentx-agg1_tp8-c4]
+  benchmark:
+    env:
+      CONCURRENCY: ['2', '4']
+  resources:
+    agg_nodes: 1
+    agg_workers: 1
+    gpus_per_decode: 8
+    gpus_per_prefill: 8

recipes/b300-fp4/glm5.2-agentx.yaml

@@ -0,0 +1,233 @@
+base:
+  name: b300-fp4-glm5.2-agentx
+  model:
+    path: glm-5.2-fp4
+    container: nightly-20260824
+    precision: fp4
+  identity:
+    model:
+      repo: nvidia/GLM-5.2-NVFP4
+      revision: aec724e
+    frameworks:
+      dynamo: 1.2.1
+      sglang: nightly-dev-cu13-20260824-95f5ecd3
+  resources:
+    gpu_type: b300
+    gpus_per_node: 8
+  frontend:
+    type: dynamo
+    nginx_session_affinity: true
+    nginx_session_affinity_header: X-Dynamo-Session-ID
+    env:
+      PIP_BREAK_SYSTEM_PACKAGES: '1'
+    args:
+      router-mode: kv
+      router-session-affinity-ttl-secs: '3600'
+      active-decode-blocks-threshold: None
+      active-prefill-tokens-threshold: None
+      active-prefill-tokens-threshold-frac: None
+  dynamo:
+    hash: 71eb001e17fa73c742f0afe1a6ed96836cb135fd
+    install: true
+  backend:
+    type: sglang
+    prefill_environment:
+      TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
+      PYTHONUNBUFFERED: '1'
+      DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
+      SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
+      SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
+      SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
+      MC_TE_METRIC: 'true'
+      NCCL_CUMEM_ENABLE: '1'
+      SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
+      SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
+      SGLANG_ENABLE_THINKING: '1'
+      SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1'
+      SGLANG_HICACHE_DEBUG_LOG: '1'
+      SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384'
+      SGLANG_REASONING_EFFORT: max
+      PIP_BREAK_SYSTEM_PACKAGES: '1'
+      SGLANG_SIMULATE_ACC_LEN: '2.5'
+    decode_environment:
+      TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
+      PYTHONUNBUFFERED: '1'
+      DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
+      SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
+      SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
+      SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
+      MC_TE_METRIC: 'true'
+      NCCL_CUMEM_ENABLE: '1'
+      SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
+      SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
+      SGLANG_MOE_NVFP4_DISPATCH: '1'
+      SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1'
+      SGLANG_ENABLE_THINKING: '1'
+      SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8'
+      SGLANG_REASONING_EFFORT: max
+      PIP_BREAK_SYSTEM_PACKAGES: '1'
+      SGLANG_SIMULATE_ACC_LEN: '2.5'
+      SGLANG_SIMULATE_ACC_METHOD: match-expected
+      SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token
+    sglang_config:
+      prefill:
+        served-model-name: GLM-5.2-FP4
+        trust-remote-code: true
+        quantization: modelopt_fp4
+        kv-cache-dtype: fp8_e4m3
+        disaggregation-mode: prefill
+        disaggregation-transfer-backend: nixl
+        tensor-parallel-size: 4
+        data-parallel-size: 4
+        expert-parallel-size: 4
+        enable-dp-attention: true
+        enable-dp-lm-head: true
+        load-balance-method: total_tokens
+        chunked-prefill-size: 65536
+        max-prefill-tokens: 16384
+        max-running-requests: 16
+        cuda-graph-max-bs: 16
+        disable-cuda-graph: true
+        nsa-prefill-backend: trtllm
+        nsa-decode-backend: trtllm
+        moe-runner-backend: flashinfer_cutlass
+        fp4-gemm-backend: flashinfer_cutlass
+        enable-flashinfer-allreduce-fusion: true
+        weight-loader-prefetch-checkpoints: true
+        model-loader-extra-config: '{"enable_multithread_load": true}'
+        mem-fraction-static: 0.8
+        context-length: 1048576
+        enable-hierarchical-cache: true
+        hicache-write-policy: write_back
+        hicache-size: 270
+        hicache-io-backend: direct
+        speculative-algorithm: EAGLE
+        speculative-num-steps: 1
+        speculative-eagle-topk: 1
+        speculative-num-draft-tokens: 2
+        enable-metrics: true
+        enable-cache-report: true
+      decode:
+        served-model-name: GLM-5.2-FP4
+        trust-remote-code: true
+        quantization: modelopt_fp4
+        kv-cache-dtype: fp8_e4m3
+        disaggregation-mode: decode
+        disaggregation-transfer-backend: nixl
+        disable-radix-cache: true
+        speculative-algorithm: EAGLE
+        speculative-num-steps: 2
+        speculative-eagle-topk: 1
+        speculative-num-draft-tokens: 3
+        tensor-parallel-size: 4
+        data-parallel-size: 1
+        expert-parallel-size: 1
+        enable-dp-attention: false
+        enable-dp-lm-head: false
+        max-running-requests: 16
+        cuda-graph-max-bs: 16
+        chunked-prefill-size: 64
+        context-length: 1048576
+        nsa-prefill-backend: trtllm
+        nsa-decode-backend: trtllm
+        moe-runner-backend: flashinfer_trtllm
+        fp4-gemm-backend: flashinfer_cutlass
+        skip-tokenizer-init: true
+        stream-interval: 30
+        enable-flashinfer-allreduce-fusion: true
+        weight-loader-prefetch-checkpoints: true
+        model-loader-extra-config: '{"enable_multithread_load": true}'
+        mem-fraction-static: 0.92
+        disaggregation-decode-extra-slots: 0
+        enable-metrics: true
+        enable-cache-report: true
+  health_check:
+    max_attempts: 1440
+    interval_seconds: 10
+  benchmark:
+    type: custom
+    env:
+      CONCURRENCY: '63'
+      DURATION: '3600'
+      AIPERF_HF_HOME: /hf_home
+      HF_HOME: /hf_home
+      HF_HUB_CACHE: /hf_home/hub
+      HF_DATASETS_CACHE: /hf_home/datasets
+      HF_HUB_OFFLINE: '1'
+      HF_DATASETS_OFFLINE: '1'
+      HF_HUB_DISABLE_XET: '1'
+      XDG_CACHE_HOME: /tmp/cache
+      AIPERF_DATASET_MMAP_CACHE_DIR: /hf_home/mmap_cache
+      AIPERF_DATASET_CONFIGURATION_TIMEOUT: '1800'
+      AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT: '1800'
+      AIPERF_FAILED_REQUEST_THRESHOLD: '0.1'
+      AIPERF_RECORD_STRIP_PAYLOAD_BYTES: 'true'
+      AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'
+      AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:'
+    command: |
+      set -euo pipefail
+      export HF_HOME="${AIPERF_HF_HOME:-/hf_home}"
+      export HF_HUB_CACHE="${HF_HOME}/hub"
+      export HF_DATASETS_CACHE="${HF_HOME}/datasets"
+      mkdir -p "${XDG_CACHE_HOME}" "${AIPERF_DATASET_MMAP_CACHE_DIR}" /logs/aiperf
+      /aiperf-venv/bin/aiperf profile \
+              --scenario inferencex-agentx-mvp \
+              --url http://localhost:8000 \
+              --endpoint /v1/chat/completions \
+              --endpoint-type chat \
+              --streaming \
+              --model GLM-5.2-FP4 \
+              --tokenizer /model \
+              --tokenizer-trust-remote-code \
+              --concurrency "${CONCURRENCY}" \
+              --benchmark-duration "${DURATION}" \
+              --random-seed 42 \
+              --failed-request-threshold "${AIPERF_FAILED_REQUEST_THRESHOLD}" \
+              --stats-interval 30 \
+              --warmup-requests-per-lane 10 \
+              --warmup-grace-period 1800 \
+              --trajectory-start-min-ratio 0.25 \
+              --trajectory-start-max-ratio 0.75 \
+              --use-server-token-count \
+              --no-gpu-telemetry \
+              --num-dataset-entries 393 \
+              --slice-duration 1.0 \
+              --public-dataset semianalysis_cc_traces_weka_062126 \
+              --trace-idle-gap-cap-seconds 300 \
+              --unsafe-override \
+              --ui simple \
+              --server-metrics-formats json \
+              --output-artifact-dir /logs/aiperf
+      rm -f /logs/aiperf/inputs.json
+  infra:
+    etcd_nats_dedicated_node: false
+    nats_max_payload_mb: 64
+  sbatch_directives:
+    mem: '0'
+  srun_options:
+    mem: '0'
+    container-remap-root: ''
+# ################# agentx #################
+zip_override_mtp_agentx_lowlat:
+  name: [agentx-1p2d_tp4-c63, agentx-1p3d_tp4-c61, agentx-1p4d_tp4-c59, agentx-1p6d_tp4-c58, agentx-2p1d_dtp8-c139]
+  backend:
+    sglang_config:
+      decode:
+        cuda-graph-max-bs: [16, 16, 16, 16, 139]
+        data-parallel-size: [1, 1, 1, 1, 8]
+        enable-dp-attention: [false, false, false, false, true]
+        enable-dp-lm-head: [false, false, false, false, true]
+        max-running-requests: [16, 16, 16, 16, 139]
+        tensor-parallel-size: [4, 4, 4, 4, 8]
+  benchmark:
+    env:
+      CONCURRENCY: ['63', '61', '59', '58', '139']
+  frontend:
+    enable_multiple_frontends: true
+  resources:
+    decode_nodes: [1, 2, 2, 3, 1]
+    decode_workers: [2, 3, 4, 6, 1]
+    gpus_per_decode: [4, 4, 4, 4, 8]
+    gpus_per_prefill: 4
+    prefill_nodes: 1
+    prefill_workers: [1, 1, 1, 1, 2]