From ef474678d434d2cbf1a4e28a12fb7db41399bb6e Mon Sep 17 00:00:00 2001
From: pvijayakrish <pvijayakrish@nvidia.com>
Date: Mon, 29 Jun 2026 14:23:43 -0700
Subject: [PATCH 1/3] chore: upgrade the aiperf version in dynamo repo

Signed-off-by: pvijayakrish <pvijayakrish@nvidia.com>
---
 benchmarks/frontend/scripts/README.md          |  2 +-
 .../frontend/scripts/sweep_k8s/aiperf.py       |  4 ++--
 benchmarks/pyproject.toml                      |  2 +-
 container/deps/requirements.benchmark.txt      |  2 +-
 container/templates/frontend.Dockerfile        |  2 +-
 .../deepseek-v3-2-wideep-routing.mdx           |  2 +-
 docs/benchmarks/kv-router-ab-testing.md        |  2 +-
 docs/recipes/deepseek-v3-2-nvfp4.mdx           |  2 +-
 .../trtllm/disagg/wide_ep/gb200/perf.yaml      |  2 +-
 .../trtllm/agg-round-robin/perf.yaml           |  2 +-
 .../trtllm/disagg-kv-router/perf.yaml          |  2 +-
 .../vllm/disagg/gb200/perf.yaml                |  2 +-
 .../glm-5-nvfp4/sglang/disagg/efa/perf.yaml    |  2 +-
 recipes/glm-5-nvfp4/sglang/disagg/perf.yaml    |  2 +-
 recipes/gpt-oss-120b/trtllm/agg/perf.yaml      |  2 +-
 recipes/gpt-oss-120b/trtllm/disagg/perf.yaml   |  2 +-
 .../trtllm/agg-eagle-kv-router/perf.yaml       |  2 +-
 .../trtllm/agg-eagle-round-robin/perf.yaml     |  2 +-
 .../kimi-k2.5/trtllm/agg-round-robin/perf.yaml |  2 +-
 .../trtllm/disagg-eagle-kv-router/perf.yaml    |  2 +-
 recipes/kimi-k2.6/perf/perf.yaml               |  2 +-
 recipes/llama-3-70b/vllm/agg/perf.yaml         |  2 +-
 .../vllm/disagg-multi-node/perf.yaml           |  2 +-
 .../vllm/disagg-single-node/perf.yaml          |  2 +-
 recipes/nemotron-3-super/perf/perf.yaml        |  2 +-
 recipes/nemotron-3-ultra/perf/README.md        |  2 +-
 recipes/nemotron-3-ultra/perf/perf.yaml        |  2 +-
 .../trtllm/agg/blackwell/perf.yaml             |  2 +-
 .../trtllm/agg/hopper/perf.yaml                |  2 +-
 .../trtllm/disagg/blackwell/perf.yaml          |  2 +-
 .../trtllm/disagg/hopper/perf.yaml             |  2 +-
 recipes/qwen3-32b-fp8/trtllm/agg/perf.yaml     |  2 +-
 recipes/qwen3-32b-fp8/trtllm/disagg/perf.yaml  |  2 +-
 recipes/qwen3-32b-fp8/vllm/disagg/perf.yaml    |  2 +-
 .../qwen3-32b/vllm/agg-round-robin/perf.yaml   |  2 +-
 .../qwen3-32b/vllm/disagg-kv-router/perf.yaml  |  2 +-
 .../vllm/agg-embedding-cache/perf.yaml         |  2 +-
 recipes/qwen3-vl-32b-fp8/benchmark/perf.yaml   |  2 +-
 recipes/qwen3.6-35b/README.md                  | 18 ++++++++----------
 recipes/qwen3.6-35b/perf.yaml                  | 17 ++---------------
 tests/fault_tolerance/deploy/client.py         |  3 +--
 41 files changed, 50 insertions(+), 66 deletions(-)

diff --git a/benchmarks/frontend/scripts/README.md b/benchmarks/frontend/scripts/README.md
index 1afeda23799d..07319911d9f6 100644
--- a/benchmarks/frontend/scripts/README.md
+++ b/benchmarks/frontend/scripts/README.md
@@ -116,7 +116,7 @@ When `--num-models` is 1, the served model name matches the HF model path (e.g.,
 |------|----------|---------|
 | etcd | Yes | `apt install etcd` or [releases](https://github.com/etcd-io/etcd/releases) |
 | nats-server | Yes | `apt install nats-server` or [nats.io](https://nats.io/download/) |
-| aiperf | Yes | `uv pip install "aiperf==0.8.0"` (in dynamo venv) |
+| aiperf | Yes | `uv pip install "aiperf==0.10.0"` (in dynamo venv) |
 | jq | Yes | `apt install jq` |
 | perf | Optional | `apt install linux-tools-$(uname -r)` |
 | bpftrace | Optional | `apt install bpftrace` (needs root or CAP_BPF + CAP_PERFMON) |
diff --git a/benchmarks/frontend/scripts/sweep_k8s/aiperf.py b/benchmarks/frontend/scripts/sweep_k8s/aiperf.py
index 693c472d3202..533174e31722 100644
--- a/benchmarks/frontend/scripts/sweep_k8s/aiperf.py
+++ b/benchmarks/frontend/scripts/sweep_k8s/aiperf.py
@@ -61,7 +61,7 @@ def _build_aiperf_script(
 
     return f"""set -e
 apt-get update -qq && apt-get install -y -qq curl jq git procps 2>/dev/null
-# aiperf 0.8.0 pulls crick==0.0.8 which has no manylinux aarch64 wheel; on
+# aiperf 0.10.0 pulls crick==0.0.8 which has no manylinux aarch64 wheel; on
 # arm64 nodes pip falls back to sdist and needs a C compiler. python:3.12-slim
 # bundles Python headers at /usr/local/include/python3.12/, but libc headers
 # (stdlib.h, etc.) come from libc6-dev — and on Debian libc6-dev is only a
@@ -71,7 +71,7 @@ def _build_aiperf_script(
 if [ "$(uname -m)" = "aarch64" ]; then
     apt-get install -y -qq gcc libc6-dev 2>/dev/null
 fi
-pip install --quiet aiperf==0.8.0
+pip install --quiet aiperf==0.10.0
 echo "aiperf installed"
 
 # Wait for model
diff --git a/benchmarks/pyproject.toml b/benchmarks/pyproject.toml
index 333279e06204..75cd9848d3e1 100644
--- a/benchmarks/pyproject.toml
+++ b/benchmarks/pyproject.toml
@@ -41,7 +41,7 @@ classifiers = [
 
 dependencies = [
     "aiconfigurator[webapp]>=0.9.0",
-    "aiperf==0.8.0",
+    "aiperf==0.10.0",
     "matplotlib",
     "networkx",
     "numpy",
diff --git a/container/deps/requirements.benchmark.txt b/container/deps/requirements.benchmark.txt
index 754542a7a860..a08b7b5a8333 100644
--- a/container/deps/requirements.benchmark.txt
+++ b/container/deps/requirements.benchmark.txt
@@ -3,4 +3,4 @@
 
 # Benchmark and profiling tool dependencies.
 
-aiperf==0.8.0
+aiperf==0.10.0
diff --git a/container/templates/frontend.Dockerfile b/container/templates/frontend.Dockerfile
index 269521ba5000..5c13c0664c62 100644
--- a/container/templates/frontend.Dockerfile
+++ b/container/templates/frontend.Dockerfile
@@ -9,7 +9,7 @@
 FROM ${EPP_IMAGE} AS epp
 
 # Build `crick` as a wheel in an isolated stage so the C toolchain never
-# reaches the final frontend image. aiperf 0.8.0 depends on crick==0.0.8,
+# reaches the final frontend image. aiperf 0.10.0 depends on crick==0.0.8,
 # which publishes no manylinux aarch64 wheel — without this, arm64 builds
 # fall back to sdist and fail in the final stage where gcc is intentionally
 # absent. amd64 has a prebuilt manylinux_x86_64 wheel on PyPI, so the build
diff --git a/docs/benchmarks/deepseek-v3-2-wideep-routing.mdx b/docs/benchmarks/deepseek-v3-2-wideep-routing.mdx
index 7cc9ace70e13..cface5fe7d40 100644
--- a/docs/benchmarks/deepseek-v3-2-wideep-routing.mdx
+++ b/docs/benchmarks/deepseek-v3-2-wideep-routing.mdx
@@ -93,7 +93,7 @@ The benchmark runs as a Kubernetes Job; tail it with `kubectl logs -f -l job-nam
 ## Notes
 
 - The source publishes the comparison as a results video plus the dataset statistics above, not a numeric results table; run both configurations to produce TTFT/ITL/goodput deltas for your cluster.
-- `perf.yaml` pins `transformers==4.57.6` alongside `aiperf==0.6.0` — older transformers cannot load the `deepseek_v32` tokenizer and AIPerf surfaces it as "Failed to load tokenizer".
+- `perf.yaml` pins `transformers==4.57.6` alongside `aiperf==0.10.0` — the trtllm-runtime base ships transformers 4.55.0, and aiperf's `transformers>=4.56.0` floor would otherwise pull 5.x, which cannot load the `deepseek_v32` tokenizer (surfaces as "Failed to load tokenizer").
 - Multi-node GB200 deployments need the ComputeDomain CR so the DRA scheduler co-locates worker pods on MNNVL-connected nodes; if you rename it, mirror the change in each `deploy.yaml` under `extraPodSpec.resourceClaims` and `resources.claims`.
 - Background on the underlying optimizations: [Optimizing DeepSeek-V3.2 on NVIDIA Blackwell GPUs](https://nvidia.github.io/TensorRT-LLM/blogs/tech_blog/blog15_Optimizing_DeepSeek_V32_on_NVIDIA_Blackwell_GPUs.html).
 - Source: [recipes/deepseek-v32-fp4](https://github.com/ai-dynamo/dynamo/tree/main/recipes/deepseek-v32-fp4)
diff --git a/docs/benchmarks/kv-router-ab-testing.md b/docs/benchmarks/kv-router-ab-testing.md
index 0a6a613c1e3d..77ebf74f7d60 100644
--- a/docs/benchmarks/kv-router-ab-testing.md
+++ b/docs/benchmarks/kv-router-ab-testing.md
@@ -450,7 +450,7 @@ spec:
           - -lc
           - |
             apt-get update -qq && apt-get install -y -qq tmux > /dev/null 2>&1
-            pip install -q aiperf==0.8.0
+            pip install -q aiperf==0.10.0
             echo "Benchmark pod ready (tmux + aiperf installed)."
             sleep infinity
         imagePullPolicy: IfNotPresent
diff --git a/docs/recipes/deepseek-v3-2-nvfp4.mdx b/docs/recipes/deepseek-v3-2-nvfp4.mdx
index 0caa5062c37c..795158a92d6e 100644
--- a/docs/recipes/deepseek-v3-2-nvfp4.mdx
+++ b/docs/recipes/deepseek-v3-2-nvfp4.mdx
@@ -125,7 +125,7 @@ Because the replay is `--fixed-schedule`, throughput is fixed by the trace — t
 ## Notes
 
 - The aggregated round-robin configuration (`trtllm/agg-round-robin/`) is the benchmark control arm; it appears on the Feature Benchmark page, not as a recipe target here.
-- The benchmark job pins `transformers==4.57.6`: the trtllm-runtime base ships 4.55.0, and aiperf 0.6.0's `transformers>=4.56.0` floor would otherwise pull 5.x, which cannot load the `deepseek_v32` tokenizer (surfaces as "Failed to load tokenizer").
+- The benchmark job pins `transformers==4.57.6`: the trtllm-runtime base ships 4.55.0, and aiperf's `transformers>=4.56.0` floor would otherwise pull 5.x, which cannot load the `deepseek_v32` tokenizer (surfaces as "Failed to load tokenizer").
 - Decode workers run the WIDEEP MoE backend with `moe_expert_parallel_size: 8` and FP8 KV cache; prefill enables chunked prefill and KV block reuse (`tokens_per_block: 64`, matching `--kv-block-size 64`).
 - The trace tops out at 109,459 input tokens; both engine configs set `max_seq_len: 121000` to accommodate it.
 
diff --git a/recipes/deepseek-r1/trtllm/disagg/wide_ep/gb200/perf.yaml b/recipes/deepseek-r1/trtllm/disagg/wide_ep/gb200/perf.yaml
index 093165434f67..47cfbfd87b3f 100644
--- a/recipes/deepseek-r1/trtllm/disagg/wide_ep/gb200/perf.yaml
+++ b/recipes/deepseek-r1/trtllm/disagg/wide_ep/gb200/perf.yaml
@@ -30,7 +30,7 @@ spec:
         - -c
         - |
           apt-get update && apt-get install -y curl jq procps git && apt-get clean
-          pip install aiperf==0.6.0;
+          pip install aiperf==0.10.0;
           echo "aiperf installation completed";
           sysctl -w net.ipv4.ip_local_port_range="1024 65000"
           cat /proc/sys/net/ipv4/ip_local_port_range
diff --git a/recipes/deepseek-v32-fp4/trtllm/agg-round-robin/perf.yaml b/recipes/deepseek-v32-fp4/trtllm/agg-round-robin/perf.yaml
index eef7c962efc6..47779d9d40fa 100644
--- a/recipes/deepseek-v32-fp4/trtllm/agg-round-robin/perf.yaml
+++ b/recipes/deepseek-v32-fp4/trtllm/agg-round-robin/perf.yaml
@@ -55,7 +55,7 @@ spec:
           # native support for model_type=deepseek_v32 (huggingface/transformers#41251)
           # and fails AutoTokenizer.from_pretrained() before reading tokenizer.json.
           # aiperf surfaces this as "Failed to load tokenizer". DYN-2878.
-          pip install "aiperf==0.6.0" "transformers==4.57.6"
+          pip install "aiperf==0.10.0" "transformers==4.57.6"
           echo "aiperf installation completed"
           sysctl -w net.ipv4.ip_local_port_range="1024 65000" 2>/dev/null || true
           export COLUMNS=200
diff --git a/recipes/deepseek-v32-fp4/trtllm/disagg-kv-router/perf.yaml b/recipes/deepseek-v32-fp4/trtllm/disagg-kv-router/perf.yaml
index 7515449545fa..dfe703a68052 100644
--- a/recipes/deepseek-v32-fp4/trtllm/disagg-kv-router/perf.yaml
+++ b/recipes/deepseek-v32-fp4/trtllm/disagg-kv-router/perf.yaml
@@ -55,7 +55,7 @@ spec:
           # native support for model_type=deepseek_v32 (huggingface/transformers#41251)
           # and fails AutoTokenizer.from_pretrained() before reading tokenizer.json.
           # aiperf surfaces this as "Failed to load tokenizer". DYN-2878.
-          pip install "aiperf==0.6.0" "transformers==4.57.6"
+          pip install "aiperf==0.10.0" "transformers==4.57.6"
           echo "aiperf installation completed"
           sysctl -w net.ipv4.ip_local_port_range="1024 65000" 2>/dev/null || true
           export COLUMNS=200
diff --git a/recipes/deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/perf.yaml b/recipes/deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/perf.yaml
index d6bd33728c32..27868fc87348 100644
--- a/recipes/deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/perf.yaml
+++ b/recipes/deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/perf.yaml
@@ -75,7 +75,7 @@ spec:
           # No apt-get: cluster runs us as non-root, and we don't actually need
           # curl/jq — Python stdlib handles the readiness probe. Install aiperf
           # into HOME=/tmp/.local via --user so it works under runAsNonRoot.
-          pip install --user -q aiperf==0.6.0 transformers==4.57.3 tokenizers==0.22.2
+          pip install --user -q aiperf==0.10.0 transformers==4.57.3 tokenizers==0.22.2
 
           echo "Waiting for model at http://$ENDPOINT/v1/models..."
           python3 - <<'PY'
diff --git a/recipes/glm-5-nvfp4/sglang/disagg/efa/perf.yaml b/recipes/glm-5-nvfp4/sglang/disagg/efa/perf.yaml
index 47b19229ac7a..9f40ea70dfe7 100644
--- a/recipes/glm-5-nvfp4/sglang/disagg/efa/perf.yaml
+++ b/recipes/glm-5-nvfp4/sglang/disagg/efa/perf.yaml
@@ -69,7 +69,7 @@ spec:
           # GLM-5 NVFP4 tokenizer uses the `TokenizersBackend` auto-tokenizer
           # class introduced in transformers 5.x — pin to match the runtime
           # image (see `pip show transformers` inside the sglang decode pod).
-          pip install aiperf==0.6.0 transformers==5.3.0 tokenizers==0.22.2
+          pip install aiperf==0.10.0 transformers==5.3.0 tokenizers==0.22.2
 
           # Pre-save tokenizer to a flat directory once, then point aiperf at
           # that path. Avoids each of aiperf's N record-processor subprocesses
diff --git a/recipes/glm-5-nvfp4/sglang/disagg/perf.yaml b/recipes/glm-5-nvfp4/sglang/disagg/perf.yaml
index 3383d97153d2..5e53e6fbefae 100644
--- a/recipes/glm-5-nvfp4/sglang/disagg/perf.yaml
+++ b/recipes/glm-5-nvfp4/sglang/disagg/perf.yaml
@@ -66,7 +66,7 @@ spec:
           # stdlib is enough for the readiness probe.
           # GLM-5's tokenizer_config.json uses tokenizer_class=TokenizersBackend,
           # which is supported by Transformers v5.
-          pip install --user -q aiperf==0.6.0 transformers==5.8.0 tokenizers==0.22.2
+          pip install --user -q aiperf==0.10.0 transformers==5.8.0 tokenizers==0.22.2
 
           echo "Waiting for model at http://$ENDPOINT/v1/models..."
           python3 - <<'PY'
diff --git a/recipes/gpt-oss-120b/trtllm/agg/perf.yaml b/recipes/gpt-oss-120b/trtllm/agg/perf.yaml
index 4c323ddcc79c..c83f883acb2e 100644
--- a/recipes/gpt-oss-120b/trtllm/agg/perf.yaml
+++ b/recipes/gpt-oss-120b/trtllm/agg/perf.yaml
@@ -29,7 +29,7 @@ spec:
         - -c
         - |
           apt-get update && apt-get install -y curl jq procps git && apt-get clean
-          pip install git+https://github.com/ai-dynamo/aiperf.git@54cd6dc820bff8bfebc875da104e59d745e14f75;
+          pip install "aiperf==0.10.0";
           echo "aiperf installation completed";
           sysctl -w net.ipv4.ip_local_port_range="1024 65000"
           cat /proc/sys/net/ipv4/ip_local_port_range
diff --git a/recipes/gpt-oss-120b/trtllm/disagg/perf.yaml b/recipes/gpt-oss-120b/trtllm/disagg/perf.yaml
index 38137a2c5e2a..e07b69bc7ec5 100644
--- a/recipes/gpt-oss-120b/trtllm/disagg/perf.yaml
+++ b/recipes/gpt-oss-120b/trtllm/disagg/perf.yaml
@@ -29,7 +29,7 @@ spec:
         - -c
         - |
           apt-get update && apt-get install -y curl jq procps git && apt-get clean
-          pip install git+https://github.com/ai-dynamo/aiperf.git@4d3fa29403c8f75da22a14f1f7b3aeb27db9288f;
+          pip install "aiperf==0.10.0";
           echo "aiperf installation completed";
           sysctl -w net.ipv4.ip_local_port_range="1024 65000"
           cat /proc/sys/net/ipv4/ip_local_port_range
diff --git a/recipes/kimi-k2.5/trtllm/agg-eagle-kv-router/perf.yaml b/recipes/kimi-k2.5/trtllm/agg-eagle-kv-router/perf.yaml
index d2c7abe00da9..75e212598540 100644
--- a/recipes/kimi-k2.5/trtllm/agg-eagle-kv-router/perf.yaml
+++ b/recipes/kimi-k2.5/trtllm/agg-eagle-kv-router/perf.yaml
@@ -51,7 +51,7 @@ spec:
           set -euo pipefail
           ulimit -n 600000
           apt-get update && apt-get install -y curl jq procps git && apt-get clean
-          pip install "aiperf==0.7.0" protobuf "transformers==4.57.3"
+          pip install "aiperf==0.10.0" protobuf "transformers==4.57.3"
           sysctl -w net.ipv4.ip_local_port_range="1024 65000" 2>/dev/null || true
           export COLUMNS=200
           EPOCH=$(date +%s)
diff --git a/recipes/kimi-k2.5/trtllm/agg-eagle-round-robin/perf.yaml b/recipes/kimi-k2.5/trtllm/agg-eagle-round-robin/perf.yaml
index 39e4de105fde..6063cc2f6dc6 100644
--- a/recipes/kimi-k2.5/trtllm/agg-eagle-round-robin/perf.yaml
+++ b/recipes/kimi-k2.5/trtllm/agg-eagle-round-robin/perf.yaml
@@ -51,7 +51,7 @@ spec:
           set -euo pipefail
           ulimit -n 600000
           apt-get update && apt-get install -y curl jq procps git && apt-get clean
-          pip install "aiperf==0.7.0" protobuf "transformers==4.57.3"
+          pip install "aiperf==0.10.0" protobuf "transformers==4.57.3"
           sysctl -w net.ipv4.ip_local_port_range="1024 65000" 2>/dev/null || true
           export COLUMNS=200
           EPOCH=$(date +%s)
diff --git a/recipes/kimi-k2.5/trtllm/agg-round-robin/perf.yaml b/recipes/kimi-k2.5/trtllm/agg-round-robin/perf.yaml
index eb7d757e4095..f6db2d1a6028 100644
--- a/recipes/kimi-k2.5/trtllm/agg-round-robin/perf.yaml
+++ b/recipes/kimi-k2.5/trtllm/agg-round-robin/perf.yaml
@@ -51,7 +51,7 @@ spec:
           set -euo pipefail
           ulimit -n 600000
           apt-get update && apt-get install -y curl jq procps git && apt-get clean
-          pip install "aiperf==0.7.0" protobuf "transformers==4.57.3"
+          pip install "aiperf==0.10.0" protobuf "transformers==4.57.3"
           sysctl -w net.ipv4.ip_local_port_range="1024 65000" 2>/dev/null || true
           export COLUMNS=200
           EPOCH=$(date +%s)
diff --git a/recipes/kimi-k2.5/trtllm/disagg-eagle-kv-router/perf.yaml b/recipes/kimi-k2.5/trtllm/disagg-eagle-kv-router/perf.yaml
index b91149be57b2..6300f0a1a07c 100644
--- a/recipes/kimi-k2.5/trtllm/disagg-eagle-kv-router/perf.yaml
+++ b/recipes/kimi-k2.5/trtllm/disagg-eagle-kv-router/perf.yaml
@@ -51,7 +51,7 @@ spec:
           set -euo pipefail
           ulimit -n 600000
           apt-get update && apt-get install -y curl jq procps git && apt-get clean
-          pip install "aiperf==0.7.0" protobuf "transformers==4.57.3"
+          pip install "aiperf==0.10.0" protobuf "transformers==4.57.3"
           sysctl -w net.ipv4.ip_local_port_range="1024 65000" 2>/dev/null || true
           export COLUMNS=200
           EPOCH=$(date +%s)
diff --git a/recipes/kimi-k2.6/perf/perf.yaml b/recipes/kimi-k2.6/perf/perf.yaml
index 4fa49aa09b1c..73d1ed17b45e 100644
--- a/recipes/kimi-k2.6/perf/perf.yaml
+++ b/recipes/kimi-k2.6/perf/perf.yaml
@@ -63,7 +63,7 @@ spec:
           set -euo pipefail
           ulimit -n 600000
           apt-get update && apt-get install -y curl jq procps git && apt-get clean
-          pip install "aiperf==0.7.0" "protobuf==7.35.0" "transformers==4.57.3" "tiktoken==0.13.0"
+          pip install "aiperf==0.10.0" "protobuf==7.35.0" "transformers==4.57.3" "tiktoken==0.13.0"
           sysctl -w net.ipv4.ip_local_port_range="1024 65000" 2>/dev/null || true
           export COLUMNS=200
           EPOCH=$(date +%s)
diff --git a/recipes/llama-3-70b/vllm/agg/perf.yaml b/recipes/llama-3-70b/vllm/agg/perf.yaml
index 4575c015b705..974eca17e543 100644
--- a/recipes/llama-3-70b/vllm/agg/perf.yaml
+++ b/recipes/llama-3-70b/vllm/agg/perf.yaml
@@ -20,7 +20,7 @@ spec:
         - -c
         - |
           apt-get update && apt-get install -y curl jq procps git && apt-get clean
-          pip install git+https://github.com/ai-dynamo/aiperf.git@54cd6dc820bff8bfebc875da104e59d745e14f75;
+          pip install "aiperf==0.10.0";
           echo "aiperf installation completed";
           sysctl -w net.ipv4.ip_local_port_range="1024 65000"
           cat /proc/sys/net/ipv4/ip_local_port_range
diff --git a/recipes/llama-3-70b/vllm/disagg-multi-node/perf.yaml b/recipes/llama-3-70b/vllm/disagg-multi-node/perf.yaml
index 6bd12171ea73..b0048bd2e9b2 100644
--- a/recipes/llama-3-70b/vllm/disagg-multi-node/perf.yaml
+++ b/recipes/llama-3-70b/vllm/disagg-multi-node/perf.yaml
@@ -20,7 +20,7 @@ spec:
         - -c
         - |
           apt-get update && apt-get install -y curl jq procps git && apt-get clean
-          pip install git+https://github.com/ai-dynamo/aiperf.git@54cd6dc820bff8bfebc875da104e59d745e14f75;
+          pip install "aiperf==0.10.0";
           echo "aiperf installation completed";
           sysctl -w net.ipv4.ip_local_port_range="1024 65000"
           cat /proc/sys/net/ipv4/ip_local_port_range
diff --git a/recipes/llama-3-70b/vllm/disagg-single-node/perf.yaml b/recipes/llama-3-70b/vllm/disagg-single-node/perf.yaml
index a7fd06258310..653eee18a78b 100644
--- a/recipes/llama-3-70b/vllm/disagg-single-node/perf.yaml
+++ b/recipes/llama-3-70b/vllm/disagg-single-node/perf.yaml
@@ -20,7 +20,7 @@ spec:
         - -c
         - |
           apt-get update && apt-get install -y curl jq procps git && apt-get clean
-          pip install git+https://github.com/ai-dynamo/aiperf.git@54cd6dc820bff8bfebc875da104e59d745e14f75;
+          pip install "aiperf==0.10.0";
           echo "aiperf installation completed";
           sysctl -w net.ipv4.ip_local_port_range="1024 65000"
           cat /proc/sys/net/ipv4/ip_local_port_range
diff --git a/recipes/nemotron-3-super/perf/perf.yaml b/recipes/nemotron-3-super/perf/perf.yaml
index a9ad4d0a4764..ef808d3abd47 100644
--- a/recipes/nemotron-3-super/perf/perf.yaml
+++ b/recipes/nemotron-3-super/perf/perf.yaml
@@ -65,7 +65,7 @@ spec:
           set -euo pipefail
           ulimit -n 600000
           apt-get update && apt-get install -y curl jq procps git && apt-get clean
-          pip install "aiperf==0.7.0" "protobuf==7.35.0" "transformers==4.57.3" "tiktoken==0.13.0"
+          pip install "aiperf==0.10.0" "protobuf==7.35.0" "transformers==4.57.3" "tiktoken==0.13.0"
           sysctl -w net.ipv4.ip_local_port_range="1024 65000" 2>/dev/null || true
           export COLUMNS=200
           EPOCH=$(date +%s)
diff --git a/recipes/nemotron-3-ultra/perf/README.md b/recipes/nemotron-3-ultra/perf/README.md
index c3ad06301a9f..4c0903a70c27 100644
--- a/recipes/nemotron-3-ultra/perf/README.md
+++ b/recipes/nemotron-3-ultra/perf/README.md
@@ -147,7 +147,7 @@ Edit the `env` block on the Job:
 | `CONCURRENCY` | `18` | Single value; see [Running a Concurrency Sweep](#running-a-concurrency-sweep) |
 | `TARGET_MODEL` | `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` | Must match the served model name on the DGD frontend |
 | `TOKENIZER_PATH` | `/opt/models/patched/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` | Used by AIPerf tokenization |
-| `AIPERF_VERSION` | `0.8.0` | Matches the saved Ultra benchmark profile exports |
+| `AIPERF_VERSION` | `0.10.0` | aiperf version used for benchmarking |
 | `ROOT_ARTIFACT_DIR` | `/opt/models/perf` | Shared PVC artifact root |
 
 ## Artifacts
diff --git a/recipes/nemotron-3-ultra/perf/perf.yaml b/recipes/nemotron-3-ultra/perf/perf.yaml
index a6998a3f3abe..45420627d345 100644
--- a/recipes/nemotron-3-ultra/perf/perf.yaml
+++ b/recipes/nemotron-3-ultra/perf/perf.yaml
@@ -183,7 +183,7 @@ spec:
         - name: AIPERF_HTTP_CONNECTION_LIMIT
           value: "200"
         - name: AIPERF_VERSION
-          value: "0.8.0"
+          value: "0.10.0"
         - name: AIPERF_HTTP_SO_RCVTIMEO
           value: "120"
         - name: AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT
diff --git a/recipes/qwen3-235b-a22b-fp8/trtllm/agg/blackwell/perf.yaml b/recipes/qwen3-235b-a22b-fp8/trtllm/agg/blackwell/perf.yaml
index 305b0d39f42c..bb151fe56564 100644
--- a/recipes/qwen3-235b-a22b-fp8/trtllm/agg/blackwell/perf.yaml
+++ b/recipes/qwen3-235b-a22b-fp8/trtllm/agg/blackwell/perf.yaml
@@ -29,7 +29,7 @@ spec:
         - -c
         - |
           apt-get update && apt-get install -y curl jq procps git && apt-get clean
-          pip install git+https://github.com/ai-dynamo/aiperf.git@54cd6dc820bff8bfebc875da104e59d745e14f75;
+          pip install "aiperf==0.10.0";
           echo "aiperf installation completed";
           sysctl -w net.ipv4.ip_local_port_range="1024 65000"
           cat /proc/sys/net/ipv4/ip_local_port_range
diff --git a/recipes/qwen3-235b-a22b-fp8/trtllm/agg/hopper/perf.yaml b/recipes/qwen3-235b-a22b-fp8/trtllm/agg/hopper/perf.yaml
index 305b0d39f42c..bb151fe56564 100644
--- a/recipes/qwen3-235b-a22b-fp8/trtllm/agg/hopper/perf.yaml
+++ b/recipes/qwen3-235b-a22b-fp8/trtllm/agg/hopper/perf.yaml
@@ -29,7 +29,7 @@ spec:
         - -c
         - |
           apt-get update && apt-get install -y curl jq procps git && apt-get clean
-          pip install git+https://github.com/ai-dynamo/aiperf.git@54cd6dc820bff8bfebc875da104e59d745e14f75;
+          pip install "aiperf==0.10.0";
           echo "aiperf installation completed";
           sysctl -w net.ipv4.ip_local_port_range="1024 65000"
           cat /proc/sys/net/ipv4/ip_local_port_range
diff --git a/recipes/qwen3-235b-a22b-fp8/trtllm/disagg/blackwell/perf.yaml b/recipes/qwen3-235b-a22b-fp8/trtllm/disagg/blackwell/perf.yaml
index b8d6a7334d1e..55fc8e911ff3 100644
--- a/recipes/qwen3-235b-a22b-fp8/trtllm/disagg/blackwell/perf.yaml
+++ b/recipes/qwen3-235b-a22b-fp8/trtllm/disagg/blackwell/perf.yaml
@@ -29,7 +29,7 @@ spec:
         - -c
         - |
           apt-get update && apt-get install -y curl jq procps git && apt-get clean
-          pip install git+https://github.com/ai-dynamo/aiperf.git@54cd6dc820bff8bfebc875da104e59d745e14f75;
+          pip install "aiperf==0.10.0";
           echo "aiperf installation completed";
           sysctl -w net.ipv4.ip_local_port_range="1024 65000"
           cat /proc/sys/net/ipv4/ip_local_port_range
diff --git a/recipes/qwen3-235b-a22b-fp8/trtllm/disagg/hopper/perf.yaml b/recipes/qwen3-235b-a22b-fp8/trtllm/disagg/hopper/perf.yaml
index b8d6a7334d1e..55fc8e911ff3 100644
--- a/recipes/qwen3-235b-a22b-fp8/trtllm/disagg/hopper/perf.yaml
+++ b/recipes/qwen3-235b-a22b-fp8/trtllm/disagg/hopper/perf.yaml
@@ -29,7 +29,7 @@ spec:
         - -c
         - |
           apt-get update && apt-get install -y curl jq procps git && apt-get clean
-          pip install git+https://github.com/ai-dynamo/aiperf.git@54cd6dc820bff8bfebc875da104e59d745e14f75;
+          pip install "aiperf==0.10.0";
           echo "aiperf installation completed";
           sysctl -w net.ipv4.ip_local_port_range="1024 65000"
           cat /proc/sys/net/ipv4/ip_local_port_range
diff --git a/recipes/qwen3-32b-fp8/trtllm/agg/perf.yaml b/recipes/qwen3-32b-fp8/trtllm/agg/perf.yaml
index 14f832b3a095..16eb0361f071 100644
--- a/recipes/qwen3-32b-fp8/trtllm/agg/perf.yaml
+++ b/recipes/qwen3-32b-fp8/trtllm/agg/perf.yaml
@@ -29,7 +29,7 @@ spec:
         - -c
         - |
           apt-get update && apt-get install -y curl jq procps git && apt-get clean
-          pip install git+https://github.com/ai-dynamo/aiperf.git@54cd6dc820bff8bfebc875da104e59d745e14f75;
+          pip install "aiperf==0.10.0";
           echo "aiperf installation completed";
           sysctl -w net.ipv4.ip_local_port_range="1024 65000"
           cat /proc/sys/net/ipv4/ip_local_port_range
diff --git a/recipes/qwen3-32b-fp8/trtllm/disagg/perf.yaml b/recipes/qwen3-32b-fp8/trtllm/disagg/perf.yaml
index 5c08b0b394c5..f700eed9dba8 100644
--- a/recipes/qwen3-32b-fp8/trtllm/disagg/perf.yaml
+++ b/recipes/qwen3-32b-fp8/trtllm/disagg/perf.yaml
@@ -29,7 +29,7 @@ spec:
         - -c
         - |
           apt-get update && apt-get install -y curl jq procps git && apt-get clean
-          pip install git+https://github.com/ai-dynamo/aiperf.git@54cd6dc820bff8bfebc875da104e59d745e14f75;
+          pip install "aiperf==0.10.0";
           echo "aiperf installation completed";
           sysctl -w net.ipv4.ip_local_port_range="1024 65000"
           cat /proc/sys/net/ipv4/ip_local_port_range
diff --git a/recipes/qwen3-32b-fp8/vllm/disagg/perf.yaml b/recipes/qwen3-32b-fp8/vllm/disagg/perf.yaml
index 414361fbf09d..31ae040b644a 100644
--- a/recipes/qwen3-32b-fp8/vllm/disagg/perf.yaml
+++ b/recipes/qwen3-32b-fp8/vllm/disagg/perf.yaml
@@ -19,7 +19,7 @@ spec:
         - -c
         - |
           apt-get update && apt-get install -y curl jq procps git && apt-get clean
-          pip install git+https://github.com/ai-dynamo/aiperf.git@54cd6dc820bff8bfebc875da104e59d745e14f75;
+          pip install "aiperf==0.10.0";
           echo "aiperf installation completed";
           sysctl -w net.ipv4.ip_local_port_range="1024 65000"
           cat /proc/sys/net/ipv4/ip_local_port_range
diff --git a/recipes/qwen3-32b/vllm/agg-round-robin/perf.yaml b/recipes/qwen3-32b/vllm/agg-round-robin/perf.yaml
index 75b91b2050c3..c2086c812cc7 100644
--- a/recipes/qwen3-32b/vllm/agg-round-robin/perf.yaml
+++ b/recipes/qwen3-32b/vllm/agg-round-robin/perf.yaml
@@ -20,7 +20,7 @@ spec:
         apt update && apt install tmux wget curl jq -y
 
         # Install benchmarking tool
-        pip install aiperf==0.6.0
+        pip install aiperf==0.10.0
 
         # Wait for model to be ready
         echo "Waiting for model '${MODEL_NAME}' at http://${FRONTEND}:8000/v1/models..."
diff --git a/recipes/qwen3-32b/vllm/disagg-kv-router/perf.yaml b/recipes/qwen3-32b/vllm/disagg-kv-router/perf.yaml
index ef637ae7db5d..52f73097c560 100644
--- a/recipes/qwen3-32b/vllm/disagg-kv-router/perf.yaml
+++ b/recipes/qwen3-32b/vllm/disagg-kv-router/perf.yaml
@@ -20,7 +20,7 @@ spec:
         apt update && apt install tmux wget curl jq -y
 
         # Install benchmarking tool
-        pip install aiperf==0.6.0
+        pip install aiperf==0.10.0
 
         # Wait for model to be ready
         echo "Waiting for model '${MODEL_NAME}' at http://${FRONTEND}:8000/v1/models..."
diff --git a/recipes/qwen3-vl-30b/vllm/agg-embedding-cache/perf.yaml b/recipes/qwen3-vl-30b/vllm/agg-embedding-cache/perf.yaml
index 5cb2bb3b972b..275054f57c93 100644
--- a/recipes/qwen3-vl-30b/vllm/agg-embedding-cache/perf.yaml
+++ b/recipes/qwen3-vl-30b/vllm/agg-embedding-cache/perf.yaml
@@ -19,7 +19,7 @@ spec:
           ulimit -u 65536
 
           apt update && apt install -y tmux curl jq
-          pip install aiperf==0.6.0
+          pip install aiperf==0.10.0
 
           echo "Waiting for model '${MODEL_NAME}' at http://${FRONTEND}:8000/v1/models..."
           until curl -s "http://${FRONTEND}:8000/v1/models" | jq -e --arg model "${MODEL_NAME}" '.data[]? | select(.id == $model)' >/dev/null 2>&1; do
diff --git a/recipes/qwen3-vl-32b-fp8/benchmark/perf.yaml b/recipes/qwen3-vl-32b-fp8/benchmark/perf.yaml
index ad256d648bbc..d23862a147bb 100644
--- a/recipes/qwen3-vl-32b-fp8/benchmark/perf.yaml
+++ b/recipes/qwen3-vl-32b-fp8/benchmark/perf.yaml
@@ -28,7 +28,7 @@ spec:
           set -e
           apt-get update && apt-get install -y curl jq procps git && apt-get clean
 
-          pip install aiperf==0.9.0
+          pip install aiperf==0.10.0
           echo "aiperf installation completed"
 
           export COLUMNS=200
diff --git a/recipes/qwen3.6-35b/README.md b/recipes/qwen3.6-35b/README.md
index 5830a3d3621e..8e6c17a8ebdf 100644
--- a/recipes/qwen3.6-35b/README.md
+++ b/recipes/qwen3.6-35b/README.md
@@ -159,16 +159,14 @@ nodes live in a different AZ.
 
 ## aiperf install
 
-We install aiperf from source pinned to a recent `main` SHA that
-includes [PR 824](https://github.com/ai-dynamo/aiperf/pull/824)
-(`feat(dataset): add session_id to single-turn for causal ordering`).
-Our sliding-window dataset writes one row per `(user, turn)` with
-`session_id=user_<N>`; PR 824 is what makes aiperf's `single_turn`
-mode honor that ordering so prefix-cache hits across turns are real.
-
-The pin lives in the shared `perf.yaml` (`AIPERF_GIT_REF` env var) at
-the recipe root, applied identically across all three configs. Bump
-when you want newer aiperf fixes.
+We install `aiperf==0.10.0` from PyPI. This release includes
+[PR 824](https://github.com/ai-dynamo/aiperf/pull/824)
+(`feat(dataset): add session_id to single-turn for causal ordering`),
+which makes `single_turn` mode honor `session_id` ordering so
+prefix-cache hits across the 8 turns of a given user are real.
+
+The version is pinned in `perf.yaml` and applies identically across
+all three configs. Bump the `aiperf==` pin there to roll forward.
 
 ## Naming & ownership
 
diff --git a/recipes/qwen3.6-35b/perf.yaml b/recipes/qwen3.6-35b/perf.yaml
index 00ac110247e2..7ed7bca8bf8b 100644
--- a/recipes/qwen3.6-35b/perf.yaml
+++ b/recipes/qwen3.6-35b/perf.yaml
@@ -7,11 +7,6 @@
 # the per-hardware ones (${HW_NODE_SELECTOR}, ${HW_TOLERATIONS}) and
 # pipes this file through `envsubst` before `kubectl apply`.
 #
-# AIPERF_GIT_REF must include https://github.com/ai-dynamo/aiperf/pull/824
-# ("add session_id to single-turn for causal ordering"). Our
-# sliding-window jsonl writes session_id=user_<N> per row; PR 824 is
-# what makes single_turn mode honor that ordering so prefix-cache
-# hits across the 8 turns of a given user actually land.
 apiVersion: v1
 kind: Pod
 metadata:
@@ -44,11 +39,8 @@ spec:
 
           apt update && apt install -y --no-install-recommends curl jq git
 
-          # Pin to a recent aiperf main SHA that includes PR 824
-          # (session_id for single-turn) + PR 903 (HF image-byte-dict fix).
-          # Bump AIPERF_GIT_REF to roll forward.
-          pip install --no-cache-dir "aiperf @ git+https://github.com/ai-dynamo/aiperf.git@${AIPERF_GIT_REF}"
-          python -c "import aiperf, sys; print('aiperf', aiperf.__version__ if hasattr(aiperf,'__version__') else 'sha-pinned', 'OK')"
+          pip install --no-cache-dir "aiperf==0.10.0"
+          python -c "import aiperf; print('aiperf', aiperf.__version__, 'OK')"
 
           echo "Waiting for model '${MODEL_NAME}' at http://${FRONTEND}:8000/v1/models ..."
           until curl -s "http://${FRONTEND}:8000/v1/models" \
@@ -109,11 +101,6 @@ spec:
           value: ${BENCH_FRONTEND}
         - name: RUN_LABEL
           value: ${BENCH_RUN_LABEL}
-        # Pin to current aiperf main HEAD; includes PR 824 + PR 903.
-        # To roll back to PR 824's exact merge commit:
-        #   fd7c2ad4abc43cc0c73079d27c016513e29309e7
-        - name: AIPERF_GIT_REF
-          value: "79de74ece808"
         - name: MAX_TOKENS
           value: "1024"
         - name: REQUEST_COUNT
diff --git a/tests/fault_tolerance/deploy/client.py b/tests/fault_tolerance/deploy/client.py
index 00bbee007408..75057cd7648a 100644
--- a/tests/fault_tolerance/deploy/client.py
+++ b/tests/fault_tolerance/deploy/client.py
@@ -339,8 +339,7 @@ def run_aiperf(
                 )
                 if success:
                     break  # Success - exit the retry loop
-            ## TODO: bug with aiperf git+https://github.com/ai-dynamo/aiperf.git@54cd6dc820bff8bfebc875da104e59d745e14f75
-            ## where sending a SIGINT on Mac can sometimes have an error code of -9 (SIGABRT) which results in profile_export_aiperf.json not being created
+            ## TODO: aiperf bug where sending a SIGINT on Mac can sometimes have an error code of -9 (SIGABRT) which results in profile_export_aiperf.json not being created
             elif result.returncode == -9 and continuous_load:
                 logger.warning(
                     f"""

From d574eeb96f1b52ee326dfcb5f3e598d7076d08f3 Mon Sep 17 00:00:00 2001
From: pvijayakrish <pvijayakrish@nvidia.com>
Date: Mon, 29 Jun 2026 14:33:27 -0700
Subject: [PATCH 2/3] revert the change in comment

Signed-off-by: pvijayakrish <pvijayakrish@nvidia.com>
---
 tests/fault_tolerance/deploy/client.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/fault_tolerance/deploy/client.py b/tests/fault_tolerance/deploy/client.py
index 75057cd7648a..00bbee007408 100644
--- a/tests/fault_tolerance/deploy/client.py
+++ b/tests/fault_tolerance/deploy/client.py
@@ -339,7 +339,8 @@ def run_aiperf(
                 )
                 if success:
                     break  # Success - exit the retry loop
-            ## TODO: aiperf bug where sending a SIGINT on Mac can sometimes have an error code of -9 (SIGABRT) which results in profile_export_aiperf.json not being created
+            ## TODO: bug with aiperf git+https://github.com/ai-dynamo/aiperf.git@54cd6dc820bff8bfebc875da104e59d745e14f75
+            ## where sending a SIGINT on Mac can sometimes have an error code of -9 (SIGABRT) which results in profile_export_aiperf.json not being created
             elif result.returncode == -9 and continuous_load:
                 logger.warning(
                     f"""

From f2308015c0049ce8aeeaa15f547aa6a92afc83ad Mon Sep 17 00:00:00 2001
From: pvijayakrish <pvijayakrish@nvidia.com>
Date: Mon, 29 Jun 2026 15:20:29 -0700
Subject: [PATCH 3/3] updated comment

Signed-off-by: pvijayakrish <pvijayakrish@nvidia.com>
---
 .../deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/perf.yaml   | 4 ++++
 recipes/glm-5-nvfp4/sglang/disagg/efa/perf.yaml               | 3 +++
 recipes/glm-5-nvfp4/sglang/disagg/perf.yaml                   | 4 ++++
 3 files changed, 11 insertions(+)

diff --git a/recipes/deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/perf.yaml b/recipes/deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/perf.yaml
index 27868fc87348..40265d685b81 100644
--- a/recipes/deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/perf.yaml
+++ b/recipes/deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/perf.yaml
@@ -75,6 +75,10 @@ spec:
           # No apt-get: cluster runs us as non-root, and we don't actually need
           # curl/jq — Python stdlib handles the readiness probe. Install aiperf
           # into HOME=/tmp/.local via --user so it works under runAsNonRoot.
+          # NOTE: aiperf>=0.8.0 depends on crick, which has no prebuilt arm64 wheel
+          # and requires gcc/libc6-dev to build from source. This pod runs non-root
+          # with no apt-get access, so this install will fail on arm64 nodes.
+          # Ensure the cluster scheduler places this pod on an x86_64 node.
           pip install --user -q aiperf==0.10.0 transformers==4.57.3 tokenizers==0.22.2
 
           echo "Waiting for model at http://$ENDPOINT/v1/models..."
diff --git a/recipes/glm-5-nvfp4/sglang/disagg/efa/perf.yaml b/recipes/glm-5-nvfp4/sglang/disagg/efa/perf.yaml
index 9f40ea70dfe7..4bad655b460c 100644
--- a/recipes/glm-5-nvfp4/sglang/disagg/efa/perf.yaml
+++ b/recipes/glm-5-nvfp4/sglang/disagg/efa/perf.yaml
@@ -69,6 +69,9 @@ spec:
           # GLM-5 NVFP4 tokenizer uses the `TokenizersBackend` auto-tokenizer
           # class introduced in transformers 5.x — pin to match the runtime
           # image (see `pip show transformers` inside the sglang decode pod).
+          # NOTE: aiperf>=0.8.0 depends on crick (no prebuilt arm64 wheel); on arm64
+          # nodes the apt-get step above must include gcc and libc6-dev so pip can
+          # compile crick from source.
           pip install aiperf==0.10.0 transformers==5.3.0 tokenizers==0.22.2
 
           # Pre-save tokenizer to a flat directory once, then point aiperf at
diff --git a/recipes/glm-5-nvfp4/sglang/disagg/perf.yaml b/recipes/glm-5-nvfp4/sglang/disagg/perf.yaml
index 5e53e6fbefae..c77a03cf5fd6 100644
--- a/recipes/glm-5-nvfp4/sglang/disagg/perf.yaml
+++ b/recipes/glm-5-nvfp4/sglang/disagg/perf.yaml
@@ -66,6 +66,10 @@ spec:
           # stdlib is enough for the readiness probe.
           # GLM-5's tokenizer_config.json uses tokenizer_class=TokenizersBackend,
           # which is supported by Transformers v5.
+          # NOTE: aiperf>=0.8.0 depends on crick, which has no prebuilt arm64 wheel
+          # and requires gcc/libc6-dev to build from source. This pod runs non-root
+          # with no apt-get access, so this install will fail on arm64 nodes.
+          # Ensure the cluster scheduler places this pod on an x86_64 node.
           pip install --user -q aiperf==0.10.0 transformers==5.8.0 tokenizers==0.22.2
 
           echo "Waiting for model at http://$ENDPOINT/v1/models..."