fix(recipes): add build tools for ARM recipes on aiperf>=0.8.0

recipes/deepseek-r1/trtllm/disagg/wide_ep/gb200/perf.yaml

@@ -29,7 +29,9 @@ spec:
         - /bin/sh
         - -c
         - |
-          apt-get update && apt-get install -y curl jq procps git && apt-get clean
+          # aiperf>=0.8.0 depends on crick, which has no Linux aarch64 wheel.
+          # GB200 benchmark pods need build tools to compile it from source.
+          apt-get update && apt-get install -y curl jq procps git build-essential && apt-get clean
           pip install aiperf==0.10.0;
           echo "aiperf installation completed";
           sysctl -w net.ipv4.ip_local_port_range="1024 65000"

recipes/deepseek-v32-fp4/trtllm/agg-round-robin/perf.yaml

@@ -47,15 +47,10 @@ spec:
           echo "File descriptor limit set to: $(ulimit -n)"
           echo 2097152 > /proc/sys/fs/inotify/max_user_watches 2>/dev/null || true
           echo 1024 > /proc/sys/fs/inotify/max_user_instances 2>/dev/null || true
-          apt-get update && apt-get install -y curl jq procps git && apt-get clean
-          # pip install git+https://github.com/ai-dynamo/aiperf.git
-          # Pin transformers==4.57.6 (verified to load the deepseek_v32 tokenizer).
-          # The trtllm-runtime base ships transformers 4.55.0; aiperf 0.6.0's
-          # `transformers>=4.56.0` floor would otherwise upgrade to 5.x, which lacks
-          # native support for model_type=deepseek_v32 (huggingface/transformers#41251)
-          # and fails AutoTokenizer.from_pretrained() before reading tokenizer.json.
-          # aiperf surfaces this as "Failed to load tokenizer". DYN-2878.
-          pip install "aiperf==0.10.0" "transformers==4.57.6"
+          # aiperf>=0.8.0 depends on crick, which has no Linux aarch64 wheel.
+          # GB200 benchmark pods need build tools to compile it from source.
+          apt-get update && apt-get install -y curl jq procps git build-essential && apt-get clean
+          pip install "aiperf==0.10.0"
           echo "aiperf installation completed"
           sysctl -w net.ipv4.ip_local_port_range="1024 65000" 2>/dev/null || true
           export COLUMNS=200

recipes/deepseek-v32-fp4/trtllm/disagg-kv-router/perf.yaml

@@ -47,15 +47,10 @@ spec:
           echo "File descriptor limit set to: $(ulimit -n)"
           echo 2097152 > /proc/sys/fs/inotify/max_user_watches 2>/dev/null || true
           echo 1024 > /proc/sys/fs/inotify/max_user_instances 2>/dev/null || true
-          apt-get update && apt-get install -y curl jq procps git && apt-get clean
-          # pip install git+https://github.com/ai-dynamo/aiperf.git
-          # Pin transformers==4.57.6 (verified to load the deepseek_v32 tokenizer).
-          # The trtllm-runtime base ships transformers 4.55.0; aiperf 0.6.0's
-          # `transformers>=4.56.0` floor would otherwise upgrade to 5.x, which lacks
-          # native support for model_type=deepseek_v32 (huggingface/transformers#41251)
-          # and fails AutoTokenizer.from_pretrained() before reading tokenizer.json.
-          # aiperf surfaces this as "Failed to load tokenizer". DYN-2878.
-          pip install "aiperf==0.10.0" "transformers==4.57.6"
+          # aiperf>=0.8.0 depends on crick, which has no Linux aarch64 wheel.
+          # GB200 benchmark pods need build tools to compile it from source.
+          apt-get update && apt-get install -y curl jq procps git build-essential && apt-get clean
+          pip install "aiperf==0.10.0"
           echo "aiperf installation completed"
           sysctl -w net.ipv4.ip_local_port_range="1024 65000" 2>/dev/null || true
           export COLUMNS=200

recipes/deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/perf.yaml

@@ -22,9 +22,6 @@ spec:
     spec:
       restartPolicy: Never
       securityContext:
-        runAsNonRoot: true
-        runAsUser: 1000
-        runAsGroup: 1000
         seccompProfile:
           type: RuntimeDefault
       containers:
@@ -57,13 +54,6 @@ spec:
           value: "1"
         - name: COLUMNS
           value: "200"
-        # HOME defaults to "/" for uid 1000 with no /etc/passwd entry; pip --user
-        # then tries to write /.local (read-only). Point HOME at /tmp (1777 sticky)
-        # so ~/.local resolves under a writable path.
-        - name: HOME
-          value: /tmp
-        - name: PATH
-          value: /tmp/.local/bin:/usr/local/bin:/usr/bin:/bin
         envFrom:
         - secretRef:
             name: hf-token-secret
@@ -72,14 +62,10 @@ spec:
         - -c
         - |
           set -eu
-          # No apt-get: cluster runs us as non-root, and we don't actually need
-          # curl/jq — Python stdlib handles the readiness probe. Install aiperf
-          # into HOME=/tmp/.local via --user so it works under runAsNonRoot.
-          # NOTE: aiperf>=0.8.0 depends on crick, which has no prebuilt arm64 wheel
-          # and requires gcc/libc6-dev to build from source. This pod runs non-root
-          # with no apt-get access, so this install will fail on arm64 nodes.
-          # Ensure the cluster scheduler places this pod on an x86_64 node.
-          pip install --user -q aiperf==0.10.0 transformers==4.57.3 tokenizers==0.22.2
+          # aiperf>=0.8.0 depends on crick, which has no Linux aarch64 wheel.
+          # GB200 benchmark pods need build tools to compile it from source.
+          apt-get update && apt-get install -y --no-install-recommends build-essential && apt-get clean
+          pip install -q aiperf==0.10.0 transformers==4.57.3 tokenizers==0.22.2
 
           echo "Waiting for model at http://$ENDPOINT/v1/models..."
           python3 - <<'PY'

recipes/glm-5-nvfp4/sglang/disagg/efa/perf.yaml

@@ -60,7 +60,9 @@ spec:
         - -c
         - |
           set -eu
-          apt-get update && apt-get install -y --no-install-recommends curl jq git && apt-get clean
+          # aiperf>=0.8.0 depends on crick, which has no Linux aarch64 wheel.
+          # GB200 benchmark pods need build tools to compile it from source.
+          apt-get update && apt-get install -y --no-install-recommends curl jq git build-essential && apt-get clean
           # torch is needed for the GLM-5 tokenizer's auto_map; install CPU-only
           # to save image size and download time (~200MB vs ~2.5GB for the GPU
           # build). Install separately because pytorch's wheel index does not
@@ -69,9 +71,6 @@ spec:
           # GLM-5 NVFP4 tokenizer uses the `TokenizersBackend` auto-tokenizer
           # class introduced in transformers 5.x — pin to match the runtime
           # image (see `pip show transformers` inside the sglang decode pod).
-          # NOTE: aiperf>=0.8.0 depends on crick (no prebuilt arm64 wheel); on arm64
-          # nodes the apt-get step above must include gcc and libc6-dev so pip can
-          # compile crick from source.
           pip install aiperf==0.10.0 transformers==5.3.0 tokenizers==0.22.2
 
           # Pre-save tokenizer to a flat directory once, then point aiperf at

recipes/glm-5-nvfp4/sglang/disagg/perf.yaml

@@ -13,9 +13,6 @@ spec:
     spec:
       restartPolicy: Never
       securityContext:
-        runAsNonRoot: true
-        runAsUser: 1000
-        runAsGroup: 1000
         seccompProfile:
           type: RuntimeDefault
       containers:
@@ -47,13 +44,6 @@ spec:
           value: "1"
         - name: COLUMNS
           value: "200"
-        # HOME defaults to "/" for uid 1000 with no /etc/passwd entry; pip --user
-        # then tries to write /.local (read-only). Point HOME at /tmp (1777 sticky)
-        # so ~/.local resolves under a writable path.
-        - name: HOME
-          value: /tmp
-        - name: PATH
-          value: /tmp/.local/bin:/usr/local/bin:/usr/bin:/bin
         envFrom:
         - secretRef:
             name: hf-token-secret
@@ -62,15 +52,12 @@ spec:
         - -c
         - |
           set -eu
-          # No apt-get: cluster benchmark pods run as non-root, and Python
-          # stdlib is enough for the readiness probe.
+          # aiperf>=0.8.0 depends on crick, which has no Linux aarch64 wheel.
+          # GB200 benchmark pods need build tools to compile it from source.
+          apt-get update && apt-get install -y --no-install-recommends build-essential && apt-get clean
           # GLM-5's tokenizer_config.json uses tokenizer_class=TokenizersBackend,
           # which is supported by Transformers v5.
-          # NOTE: aiperf>=0.8.0 depends on crick, which has no prebuilt arm64 wheel
-          # and requires gcc/libc6-dev to build from source. This pod runs non-root
-          # with no apt-get access, so this install will fail on arm64 nodes.
-          # Ensure the cluster scheduler places this pod on an x86_64 node.
-          pip install --user -q aiperf==0.10.0 transformers==5.8.0 tokenizers==0.22.2
+          pip install -q aiperf==0.10.0 transformers==5.8.0 tokenizers==0.22.2
 
           echo "Waiting for model at http://$ENDPOINT/v1/models..."
           python3 - <<'PY'

recipes/gpt-oss-120b/trtllm/agg/perf.yaml

@@ -28,7 +28,9 @@ spec:
         - /bin/sh
         - -c
         - |
-          apt-get update && apt-get install -y curl jq procps git && apt-get clean
+          # aiperf>=0.8.0 depends on crick, which has no Linux aarch64 wheel.
+          # GB200 benchmark pods need build tools to compile it from source.
+          apt-get update && apt-get install -y curl jq procps git build-essential && apt-get clean
           pip install "aiperf==0.10.0";
           echo "aiperf installation completed";
           sysctl -w net.ipv4.ip_local_port_range="1024 65000"

recipes/gpt-oss-120b/trtllm/disagg/perf.yaml

@@ -28,7 +28,9 @@ spec:
         - /bin/sh
         - -c
         - |
-          apt-get update && apt-get install -y curl jq procps git && apt-get clean
+          # aiperf>=0.8.0 depends on crick, which has no Linux aarch64 wheel.
+          # GB200 benchmark pods need build tools to compile it from source.
+          apt-get update && apt-get install -y curl jq procps git build-essential && apt-get clean
           pip install "aiperf==0.10.0";
           echo "aiperf installation completed";
           sysctl -w net.ipv4.ip_local_port_range="1024 65000"

recipes/kimi-k2.5/trtllm/agg-eagle-kv-router/perf.yaml

@@ -50,7 +50,9 @@ spec:
         - |
           set -euo pipefail
           ulimit -n 600000
-          apt-get update && apt-get install -y curl jq procps git && apt-get clean
+          # aiperf>=0.8.0 depends on crick, which has no Linux aarch64 wheel.
+          # GB200 benchmark pods need build tools to compile it from source.
+          apt-get update && apt-get install -y curl jq procps git build-essential && apt-get clean
           pip install "aiperf==0.10.0" protobuf "transformers==4.57.3"
           sysctl -w net.ipv4.ip_local_port_range="1024 65000" 2>/dev/null || true
           export COLUMNS=200

recipes/kimi-k2.5/trtllm/agg-eagle-round-robin/perf.yaml

@@ -50,7 +50,9 @@ spec:
         - |
           set -euo pipefail
           ulimit -n 600000
-          apt-get update && apt-get install -y curl jq procps git && apt-get clean
+          # aiperf>=0.8.0 depends on crick, which has no Linux aarch64 wheel.
+          # GB200 benchmark pods need build tools to compile it from source.
+          apt-get update && apt-get install -y curl jq procps git build-essential && apt-get clean
           pip install "aiperf==0.10.0" protobuf "transformers==4.57.3"
           sysctl -w net.ipv4.ip_local_port_range="1024 65000" 2>/dev/null || true
           export COLUMNS=200

recipes/kimi-k2.5/trtllm/agg-round-robin/perf.yaml

@@ -50,7 +50,9 @@ spec:
         - |
           set -euo pipefail
           ulimit -n 600000
-          apt-get update && apt-get install -y curl jq procps git && apt-get clean
+          # aiperf>=0.8.0 depends on crick, which has no Linux aarch64 wheel.
+          # GB200 benchmark pods need build tools to compile it from source.
+          apt-get update && apt-get install -y curl jq procps git build-essential && apt-get clean
           pip install "aiperf==0.10.0" protobuf "transformers==4.57.3"
           sysctl -w net.ipv4.ip_local_port_range="1024 65000" 2>/dev/null || true
           export COLUMNS=200

recipes/kimi-k2.5/trtllm/disagg-eagle-kv-router/perf.yaml

@@ -50,7 +50,9 @@ spec:
         - |
           set -euo pipefail
           ulimit -n 600000
-          apt-get update && apt-get install -y curl jq procps git && apt-get clean
+          # aiperf>=0.8.0 depends on crick, which has no Linux aarch64 wheel.
+          # GB200 benchmark pods need build tools to compile it from source.
+          apt-get update && apt-get install -y curl jq procps git build-essential && apt-get clean
           pip install "aiperf==0.10.0" protobuf "transformers==4.57.3"
           sysctl -w net.ipv4.ip_local_port_range="1024 65000" 2>/dev/null || true
           export COLUMNS=200

recipes/kimi-k2.6/perf/perf.yaml

@@ -62,7 +62,7 @@ spec:
         - |
           set -euo pipefail
           ulimit -n 600000
-          apt-get update && apt-get install -y curl jq procps git && apt-get clean
+          apt-get update && apt-get install -y curl jq procps git build-essential && apt-get clean
           pip install "aiperf==0.10.0" "protobuf==7.35.0" "transformers==4.57.3" "tiktoken==0.13.0"
           sysctl -w net.ipv4.ip_local_port_range="1024 65000" 2>/dev/null || true
           export COLUMNS=200

recipes/llama-3-70b/vllm/agg/perf.yaml

@@ -19,7 +19,7 @@ spec:
         - /bin/sh
         - -c
         - |
-          apt-get update && apt-get install -y curl jq procps git && apt-get clean
+          apt-get update && apt-get install -y curl jq procps git build-essential && apt-get clean
           pip install "aiperf==0.10.0";
           echo "aiperf installation completed";
           sysctl -w net.ipv4.ip_local_port_range="1024 65000"

recipes/llama-3-70b/vllm/disagg-multi-node/perf.yaml

@@ -19,7 +19,7 @@ spec:
         - /bin/sh
         - -c
         - |
-          apt-get update && apt-get install -y curl jq procps git && apt-get clean
+          apt-get update && apt-get install -y curl jq procps git build-essential && apt-get clean
           pip install "aiperf==0.10.0";
           echo "aiperf installation completed";
           sysctl -w net.ipv4.ip_local_port_range="1024 65000"

recipes/llama-3-70b/vllm/disagg-single-node/perf.yaml

@@ -19,7 +19,7 @@ spec:
         - /bin/sh
         - -c
         - |
-          apt-get update && apt-get install -y curl jq procps git && apt-get clean
+          apt-get update && apt-get install -y curl jq procps git build-essential && apt-get clean
           pip install "aiperf==0.10.0";
           echo "aiperf installation completed";
           sysctl -w net.ipv4.ip_local_port_range="1024 65000"

recipes/nemotron-3-super/perf/perf.yaml

@@ -64,7 +64,7 @@ spec:
         - |
           set -euo pipefail
           ulimit -n 600000
-          apt-get update && apt-get install -y curl jq procps git && apt-get clean
+          apt-get update && apt-get install -y curl jq procps git build-essential && apt-get clean
           pip install "aiperf==0.10.0" "protobuf==7.35.0" "transformers==4.57.3" "tiktoken==0.13.0"
           sysctl -w net.ipv4.ip_local_port_range="1024 65000" 2>/dev/null || true
           export COLUMNS=200

recipes/nemotron-3-ultra/perf/perf.yaml

@@ -59,7 +59,7 @@ spec:
         - |
           set -euo pipefail
           ulimit -n 600000
-          apt-get update && apt-get install -y curl jq procps git coreutils && apt-get clean
+          apt-get update && apt-get install -y curl jq procps git coreutils build-essential && apt-get clean
           pip install "aiperf==${AIPERF_VERSION}" protobuf "transformers==4.57.3" "tiktoken==0.13.0"
           sysctl -w net.ipv4.ip_local_port_range="1024 65000" 2>/dev/null || true
           export COLUMNS=200

recipes/qwen3-235b-a22b-fp8/trtllm/agg/blackwell/perf.yaml

@@ -28,7 +28,7 @@ spec:
         - /bin/sh
         - -c
         - |
-          apt-get update && apt-get install -y curl jq procps git && apt-get clean
+          apt-get update && apt-get install -y curl jq procps git build-essential && apt-get clean
           pip install "aiperf==0.10.0";
           echo "aiperf installation completed";
           sysctl -w net.ipv4.ip_local_port_range="1024 65000"

recipes/qwen3-235b-a22b-fp8/trtllm/agg/hopper/perf.yaml

@@ -28,7 +28,7 @@ spec:
         - /bin/sh
         - -c
         - |
-          apt-get update && apt-get install -y curl jq procps git && apt-get clean
+          apt-get update && apt-get install -y curl jq procps git build-essential && apt-get clean
           pip install "aiperf==0.10.0";
           echo "aiperf installation completed";
           sysctl -w net.ipv4.ip_local_port_range="1024 65000"

recipes/qwen3-235b-a22b-fp8/trtllm/disagg/blackwell/perf.yaml

@@ -28,7 +28,7 @@ spec:
         - /bin/sh
         - -c
         - |
-          apt-get update && apt-get install -y curl jq procps git && apt-get clean
+          apt-get update && apt-get install -y curl jq procps git build-essential && apt-get clean
           pip install "aiperf==0.10.0";
           echo "aiperf installation completed";
           sysctl -w net.ipv4.ip_local_port_range="1024 65000"

recipes/qwen3-235b-a22b-fp8/trtllm/disagg/hopper/perf.yaml

@@ -28,7 +28,7 @@ spec:
         - /bin/sh
         - -c
         - |
-          apt-get update && apt-get install -y curl jq procps git && apt-get clean
+          apt-get update && apt-get install -y curl jq procps git build-essential && apt-get clean
           pip install "aiperf==0.10.0";
           echo "aiperf installation completed";
           sysctl -w net.ipv4.ip_local_port_range="1024 65000"

recipes/qwen3-32b-fp8/trtllm/agg/perf.yaml

@@ -28,7 +28,7 @@ spec:
         - /bin/sh
         - -c
         - |
-          apt-get update && apt-get install -y curl jq procps git && apt-get clean
+          apt-get update && apt-get install -y curl jq procps git build-essential && apt-get clean
           pip install "aiperf==0.10.0";
           echo "aiperf installation completed";
           sysctl -w net.ipv4.ip_local_port_range="1024 65000"

recipes/qwen3-32b-fp8/trtllm/disagg/perf.yaml

@@ -28,7 +28,7 @@ spec:
         - /bin/sh
         - -c
         - |
-          apt-get update && apt-get install -y curl jq procps git && apt-get clean
+          apt-get update && apt-get install -y curl jq procps git build-essential && apt-get clean
           pip install "aiperf==0.10.0";
           echo "aiperf installation completed";
           sysctl -w net.ipv4.ip_local_port_range="1024 65000"

recipes/qwen3-32b-fp8/vllm/disagg/perf.yaml

@@ -18,7 +18,7 @@ spec:
         - /bin/sh
         - -c
         - |
-          apt-get update && apt-get install -y curl jq procps git && apt-get clean
+          apt-get update && apt-get install -y curl jq procps git build-essential && apt-get clean
           pip install "aiperf==0.10.0";
           echo "aiperf installation completed";
           sysctl -w net.ipv4.ip_local_port_range="1024 65000"

recipes/qwen3-32b/vllm/agg-round-robin/perf.yaml

@@ -17,7 +17,7 @@ spec:
         # Setup
         ulimit -n 1048576
         ulimit -u 65536
-        apt update && apt install tmux wget curl jq -y
+        apt update && apt install tmux wget curl jq build-essential -y
 
         # Install benchmarking tool
         pip install aiperf==0.10.0

recipes/qwen3-32b/vllm/disagg-kv-router/perf.yaml

@@ -17,7 +17,7 @@ spec:
         # Setup
         ulimit -n 1048576
         ulimit -u 65536
-        apt update && apt install tmux wget curl jq -y
+        apt update && apt install tmux wget curl jq build-essential -y
 
         # Install benchmarking tool
         pip install aiperf==0.10.0

recipes/qwen3-vl-30b/vllm/agg-embedding-cache/perf.yaml

@@ -18,7 +18,9 @@ spec:
           ulimit -n 1048576
           ulimit -u 65536
 
-          apt update && apt install -y tmux curl jq
+          # aiperf>=0.8.0 depends on crick, which has no Linux aarch64 wheel.
+          # GB200 benchmark pods need build tools to compile it from source.
+          apt update && apt install -y tmux curl jq build-essential
           pip install aiperf==0.10.0
 
           echo "Waiting for model '${MODEL_NAME}' at http://${FRONTEND}:8000/v1/models..."

recipes/qwen3-vl-32b-fp8/benchmark/perf.yaml

@@ -26,7 +26,7 @@ spec:
         - -c
         - |
           set -e
-          apt-get update && apt-get install -y curl jq procps git && apt-get clean
+          apt-get update && apt-get install -y curl jq procps git build-essential && apt-get clean
 
           pip install aiperf==0.10.0
           echo "aiperf installation completed"

recipes/qwen3.6-35b/perf.yaml

@@ -37,7 +37,9 @@ spec:
           ulimit -n 1048576
           ulimit -u 65536
 
-          apt update && apt install -y --no-install-recommends curl jq git
+          # aiperf>=0.8.0 depends on crick, which has no Linux aarch64 wheel.
+          # GB200 benchmark pods need build tools to compile it from source.
+          apt update && apt install -y --no-install-recommends curl jq git build-essential
 
           pip install --no-cache-dir "aiperf==0.10.0"
           python -c "import aiperf; print('aiperf', aiperf.__version__, 'OK')"