diff --git a/examples/backends/sglang/deploy/README.md b/examples/backends/sglang/deploy/README.md index 44a94c44484d..6040c4865b32 100644 --- a/examples/backends/sglang/deploy/README.md +++ b/examples/backends/sglang/deploy/README.md @@ -9,6 +9,7 @@ Basic deployment pattern with frontend and a single decode worker. **Architecture:** - `Frontend`: OpenAI-compatible API server +- `ModelExpress`: Shared model caching service across workers - `SGLangDecodeWorker`: Single worker handling both prefill and decode ### 2. **Aggregated Router Deployment** (`agg_router.yaml`) @@ -16,13 +17,15 @@ Enhanced aggregated deployment with KV cache routing capabilities. **Architecture:** - `Frontend`: OpenAI-compatible API server with router mode enabled (`--router-mode kv`) +- `ModelExpress`: Shared model caching service across workers - `SGLangDecodeWorker`: Single worker handling both prefill and decode -### 3. **Disaggregated Deployment** (`disagg.yaml`)** +### 3. **Disaggregated Deployment** (`disagg.yaml`) High-performance deployment with separated prefill and decode workers. **Architecture:** - `Frontend`: HTTP API server coordinating between workers +- `ModelExpress`: Shared model caching service across workers - `SGLangDecodeWorker`: Specialized decode-only worker (`--disaggregation-mode decode`) - `SGLangPrefillWorker`: Specialized prefill-only worker (`--disaggregation-mode prefill`) - Communication via NIXL transfer backend (`--disaggregation-transfer-backend nixl`) @@ -79,6 +82,28 @@ Before using these templates, ensure you have: 3. **Container registry access** for SGLang runtime images 4. **HuggingFace token secret** (referenced as `envFromSecret: hf-token-secret`) +### Persistent Volume Claim (PVC) + +All templates expect a pre-created PVC named `model-cache-pvc` for the shared model cache used by ModelExpress and SGLang workers. + +Apply the shared PVC once per namespace before deploying any graph: + +```bash +kubectl apply -f model_cache_pvc.yaml -n $NAMESPACE +``` + +Note: If your cluster requires a specific storage class, edit `model_cache_pvc.yaml` to set `storageClassName` accordingly. + +### Container Images + +We have public images available on [NGC Catalog](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/ai-dynamo/collections/ai-dynamo/artifacts). If you'd prefer to use your own registry, build and push your own image: + +```bash +./container/build.sh --framework sglang +# Tag and push to your container registry +# Update the image references in the YAML files +``` + ## Usage ### 1. Choose Your Template @@ -118,6 +143,11 @@ Then, deploy the model using the deployment file. ```bash export DEPLOYMENT_FILE=agg.yaml + +# Create the shared model cache PVC (run once per namespace) +kubectl apply -f model_cache_pvc.yaml -n $NAMESPACE + +# Apply the SGLang deployment kubectl apply -f $DEPLOYMENT_FILE -n ${NAMESPACE} ``` @@ -135,7 +165,7 @@ kubectl apply -f $DEPLOYMENT_FILE.generated -n $NAMESPACE ## Model Configuration -All templates use **DeepSeek-R1-Distill-Llama-8B** as the default model. But you can use any sglang argument and configuration. Key parameters: +All templates use **Qwen/Qwen3-0.6B** as the default model. You can use any SGLang arguments and configuration. Key parameters include `--model-path`, `--served-model-name`, and disaggregation flags (see YAMLs for examples). ## Monitoring and Health diff --git a/examples/backends/sglang/deploy/agg.yaml b/examples/backends/sglang/deploy/agg.yaml index c6302906b6b6..eb4fb281e51b 100644 --- a/examples/backends/sglang/deploy/agg.yaml +++ b/examples/backends/sglang/deploy/agg.yaml @@ -6,6 +6,16 @@ kind: DynamoGraphDeployment metadata: name: sglang-agg spec: + pvcs: + - name: model-cache-pvc + create: false + envs: + - name: MODEL_EXPRESS_CACHE_DIRECTORY + value: "/model/.model-express/cache" + - name: HF_HUB_CACHE + value: "/model/.model-express/cache" + - name: MODEL_EXPRESS_URL + value: "http://sglang-agg-modelexpress:8000" services: Frontend: dynamoNamespace: sglang-agg @@ -14,6 +24,58 @@ spec: extraPodSpec: mainContainer: image: my-registry/sglang-runtime:my-tag + ModelExpress: + envFromSecret: hf-token-secret + dynamoNamespace: sglang-agg + componentType: frontend + readinessProbe: + tcpSocket: + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + replicas: 1 + resources: + requests: + cpu: "4" + memory: "16Gi" + limits: + cpu: "4" + memory: "16Gi" + extraPodSpec: + mainContainer: + image: nvcr.io/nvidia/ai-dynamo/modelexpress-server:my-tag + imagePullPolicy: IfNotPresent + env: + - name: MODEL_EXPRESS_SERVER_PORT + value: "8000" + - name: MODEL_EXPRESS_LOGGING_LEVEL + value: "info" + - name: MODEL_EXPRESS_DATABASE_PATH + value: "/model/models.db" + command: + - /bin/sh + - -c + args: + - | + echo "Setting up Model Express configuration..." + + mkdir -p $MODEL_EXPRESS_CACHE_DIRECTORY + cat > $MODEL_EXPRESS_CACHE_DIRECTORY/config.yaml << EOF + local_path: $MODEL_EXPRESS_CACHE_DIRECTORY + server_endpoint: http://localhost:8000 + timeout_secs: null + EOF + + ./modelexpress-server & + + SERVER_PID=$! + echo "Server started with PID: $SERVER_PID" + wait $SERVER_PID + volumeMounts: + - name: model-cache-pvc + mountPoint: /model decode: envFromSecret: hf-token-secret dynamoNamespace: sglang-agg @@ -41,4 +103,7 @@ spec: - "1" - --trust-remote-code - --skip-tokenizer-init + volumeMounts: + - name: model-cache-pvc + mountPoint: /model diff --git a/examples/backends/sglang/deploy/agg_logging.yaml b/examples/backends/sglang/deploy/agg_logging.yaml index a6e576aae932..93f6f90ea29b 100644 --- a/examples/backends/sglang/deploy/agg_logging.yaml +++ b/examples/backends/sglang/deploy/agg_logging.yaml @@ -9,6 +9,15 @@ spec: envs: - name: DYN_LOGGING_JSONL value: "1" + - name: MODEL_EXPRESS_CACHE_DIRECTORY + value: "/model/.model-express/cache" + - name: HF_HUB_CACHE + value: "/model/.model-express/cache" + - name: MODEL_EXPRESS_URL + value: "http://sglang-agg-modelexpress:8000" + pvcs: + - name: model-cache-pvc + create: false services: Frontend: dynamoNamespace: sglang-agg @@ -17,6 +26,58 @@ spec: extraPodSpec: mainContainer: image: my-registry/sglang-runtime:my-tag + ModelExpress: + envFromSecret: hf-token-secret + dynamoNamespace: sglang-agg + componentType: frontend + readinessProbe: + tcpSocket: + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + replicas: 1 + resources: + requests: + cpu: "4" + memory: "16Gi" + limits: + cpu: "4" + memory: "16Gi" + extraPodSpec: + mainContainer: + image: nvcr.io/nvidia/ai-dynamo/modelexpress-server:my-tag + imagePullPolicy: IfNotPresent + env: + - name: MODEL_EXPRESS_SERVER_PORT + value: "8000" + - name: MODEL_EXPRESS_LOGGING_LEVEL + value: "info" + - name: MODEL_EXPRESS_DATABASE_PATH + value: "/model/models.db" + command: + - /bin/sh + - -c + args: + - | + echo "Setting up Model Express configuration..." + + mkdir -p $MODEL_EXPRESS_CACHE_DIRECTORY + cat > $MODEL_EXPRESS_CACHE_DIRECTORY/config.yaml << EOF + local_path: $MODEL_EXPRESS_CACHE_DIRECTORY + server_endpoint: http://localhost:8000 + timeout_secs: null + EOF + + ./modelexpress-server & + + SERVER_PID=$! + echo "Server started with PID: $SERVER_PID" + wait $SERVER_PID + volumeMounts: + - name: model-cache-pvc + mountPoint: /model decode: envFromSecret: hf-token-secret dynamoNamespace: sglang-agg @@ -43,4 +104,7 @@ spec: - --tp - "1" - --trust-remote-code - - --skip-tokenizer-init \ No newline at end of file + - --skip-tokenizer-init + volumeMounts: + - name: model-cache-pvc + mountPoint: /model \ No newline at end of file diff --git a/examples/backends/sglang/deploy/agg_router.yaml b/examples/backends/sglang/deploy/agg_router.yaml index e9e01e8bb81d..e71965b50e2b 100644 --- a/examples/backends/sglang/deploy/agg_router.yaml +++ b/examples/backends/sglang/deploy/agg_router.yaml @@ -6,6 +6,16 @@ kind: DynamoGraphDeployment metadata: name: sglang-agg-router spec: + pvcs: + - name: model-cache-pvc + create: false + envs: + - name: MODEL_EXPRESS_CACHE_DIRECTORY + value: "/model/.model-express/cache" + - name: HF_HUB_CACHE + value: "/model/.model-express/cache" + - name: MODEL_EXPRESS_URL + value: "http://sglang-agg-router-modelexpress:8000" services: Frontend: dynamoNamespace: sglang-agg-router @@ -17,6 +27,58 @@ spec: envs: - name: DYN_ROUTER_MODE value: kv + ModelExpress: + envFromSecret: hf-token-secret + dynamoNamespace: sglang-agg-router + componentType: frontend + readinessProbe: + tcpSocket: + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + replicas: 1 + resources: + requests: + cpu: "4" + memory: "16Gi" + limits: + cpu: "4" + memory: "16Gi" + extraPodSpec: + mainContainer: + image: nvcr.io/nvidia/ai-dynamo/modelexpress-server:my-tag + imagePullPolicy: IfNotPresent + env: + - name: MODEL_EXPRESS_SERVER_PORT + value: "8000" + - name: MODEL_EXPRESS_LOGGING_LEVEL + value: "info" + - name: MODEL_EXPRESS_DATABASE_PATH + value: "/model/models.db" + command: + - /bin/sh + - -c + args: + - | + echo "Setting up Model Express configuration..." + + mkdir -p $MODEL_EXPRESS_CACHE_DIRECTORY + cat > $MODEL_EXPRESS_CACHE_DIRECTORY/config.yaml << EOF + local_path: $MODEL_EXPRESS_CACHE_DIRECTORY + server_endpoint: http://localhost:8000 + timeout_secs: null + EOF + + ./modelexpress-server & + + SERVER_PID=$! + echo "Server started with PID: $SERVER_PID" + wait $SERVER_PID + volumeMounts: + - name: model-cache-pvc + mountPoint: /model decode: envFromSecret: hf-token-secret dynamoNamespace: sglang-agg-router @@ -44,3 +106,6 @@ spec: - "1" - --trust-remote-code - --skip-tokenizer-init + volumeMounts: + - name: model-cache-pvc + mountPoint: /model diff --git a/examples/backends/sglang/deploy/disagg-multinode.yaml b/examples/backends/sglang/deploy/disagg-multinode.yaml index ac161b3159e5..d6e2b749c852 100644 --- a/examples/backends/sglang/deploy/disagg-multinode.yaml +++ b/examples/backends/sglang/deploy/disagg-multinode.yaml @@ -14,7 +14,16 @@ spec: key: HF_TOKEN - name: GLOO_SOCKET_IFNAME value: "eth0" + - name: MODEL_EXPRESS_CACHE_DIRECTORY + value: "/model/.model-express/cache" + - name: HF_HUB_CACHE + value: "/model/.model-express/cache" + - name: MODEL_EXPRESS_URL + value: "http://sglang-disagg-multinode-modelexpress:8000" backendFramework: sglang + pvcs: + - name: model-cache-pvc + create: false services: Frontend: dynamoNamespace: sglang-disagg-multinode @@ -23,6 +32,58 @@ spec: extraPodSpec: mainContainer: image: my-registry/sglang-runtime:my-tag + ModelExpress: + envFromSecret: hf-token-secret + dynamoNamespace: sglang-disagg-multinode + componentType: frontend + readinessProbe: + tcpSocket: + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + replicas: 1 + resources: + requests: + cpu: "4" + memory: "16Gi" + limits: + cpu: "4" + memory: "16Gi" + extraPodSpec: + mainContainer: + image: nvcr.io/nvidia/ai-dynamo/modelexpress-server:my-tag + imagePullPolicy: IfNotPresent + env: + - name: MODEL_EXPRESS_SERVER_PORT + value: "8000" + - name: MODEL_EXPRESS_LOGGING_LEVEL + value: "info" + - name: MODEL_EXPRESS_DATABASE_PATH + value: "/model/models.db" + command: + - /bin/sh + - -c + args: + - | + echo "Setting up Model Express configuration..." + + mkdir -p $MODEL_EXPRESS_CACHE_DIRECTORY + cat > $MODEL_EXPRESS_CACHE_DIRECTORY/config.yaml << EOF + local_path: $MODEL_EXPRESS_CACHE_DIRECTORY + server_endpoint: http://localhost:8000 + timeout_secs: null + EOF + + ./modelexpress-server & + + SERVER_PID=$! + echo "Server started with PID: $SERVER_PID" + wait $SERVER_PID + volumeMounts: + - name: model-cache-pvc + mountPoint: /model decode: multinode: nodeCount: 2 @@ -60,6 +121,9 @@ spec: - "0.0.0.0" - --mem-fraction-static - "0.82" + volumeMounts: + - name: model-cache-pvc + mountPoint: /model prefill: multinode: nodeCount: 2 @@ -96,4 +160,7 @@ spec: - --mem-fraction-static - "0.82" - --host - - "0.0.0.0" \ No newline at end of file + - "0.0.0.0" + volumeMounts: + - name: model-cache-pvc + mountPoint: /model \ No newline at end of file diff --git a/examples/backends/sglang/deploy/disagg.yaml b/examples/backends/sglang/deploy/disagg.yaml index 52866fedf56a..93ac6ad21728 100644 --- a/examples/backends/sglang/deploy/disagg.yaml +++ b/examples/backends/sglang/deploy/disagg.yaml @@ -6,6 +6,16 @@ kind: DynamoGraphDeployment metadata: name: sglang-disagg spec: + pvcs: + - name: model-cache-pvc + create: false + envs: + - name: MODEL_EXPRESS_CACHE_DIRECTORY + value: "/model/.model-express/cache" + - name: HF_HUB_CACHE + value: "/model/.model-express/cache" + - name: MODEL_EXPRESS_URL + value: "http://sglang-disagg-modelexpress:8000" services: Frontend: dynamoNamespace: sglang-disagg @@ -14,6 +24,58 @@ spec: extraPodSpec: mainContainer: image: my-registry/sglang-runtime:my-tag + ModelExpress: + envFromSecret: hf-token-secret + dynamoNamespace: sglang-disagg + componentType: frontend + readinessProbe: + tcpSocket: + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + replicas: 1 + resources: + requests: + cpu: "4" + memory: "16Gi" + limits: + cpu: "4" + memory: "16Gi" + extraPodSpec: + mainContainer: + image: nvcr.io/nvidia/ai-dynamo/modelexpress-server:my-tag + imagePullPolicy: IfNotPresent + env: + - name: MODEL_EXPRESS_SERVER_PORT + value: "8000" + - name: MODEL_EXPRESS_LOGGING_LEVEL + value: "info" + - name: MODEL_EXPRESS_DATABASE_PATH + value: "/model/models.db" + command: + - /bin/sh + - -c + args: + - | + echo "Setting up Model Express configuration..." + + mkdir -p $MODEL_EXPRESS_CACHE_DIRECTORY + cat > $MODEL_EXPRESS_CACHE_DIRECTORY/config.yaml << EOF + local_path: $MODEL_EXPRESS_CACHE_DIRECTORY + server_endpoint: http://localhost:8000 + timeout_secs: null + EOF + + ./modelexpress-server & + + SERVER_PID=$! + echo "Server started with PID: $SERVER_PID" + wait $SERVER_PID + volumeMounts: + - name: model-cache-pvc + mountPoint: /model decode: envFromSecret: hf-token-secret dynamoNamespace: sglang-disagg @@ -50,6 +112,9 @@ spec: - "12345" - --host - "0.0.0.0" + volumeMounts: + - name: model-cache-pvc + mountPoint: /model prefill: envFromSecret: hf-token-secret dynamoNamespace: sglang-disagg @@ -85,4 +150,7 @@ spec: - --disaggregation-bootstrap-port - "12345" - --host - - "0.0.0.0" \ No newline at end of file + - "0.0.0.0" + volumeMounts: + - name: model-cache-pvc + mountPoint: /model \ No newline at end of file diff --git a/examples/backends/sglang/deploy/disagg_planner.yaml b/examples/backends/sglang/deploy/disagg_planner.yaml index 1ed1d195ba49..f9fbe735d280 100644 --- a/examples/backends/sglang/deploy/disagg_planner.yaml +++ b/examples/backends/sglang/deploy/disagg_planner.yaml @@ -9,6 +9,15 @@ spec: pvcs: - name: dynamo-pvc create: false # Must be pre-created before deployment and SLA profiler must have been run + - name: model-cache-pvc + create: false + envs: + - name: MODEL_EXPRESS_CACHE_DIRECTORY + value: "/model/.model-express/cache" + - name: HF_HUB_CACHE + value: "/model/.model-express/cache" + - name: MODEL_EXPRESS_URL + value: "http://sglang-disagg-planner-modelexpress:8000" services: Frontend: dynamoNamespace: dynamo @@ -17,6 +26,58 @@ spec: extraPodSpec: mainContainer: image: my-registry/sglang-runtime:my-tag + ModelExpress: + envFromSecret: hf-token-secret + dynamoNamespace: dynamo + componentType: frontend + readinessProbe: + tcpSocket: + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + replicas: 1 + resources: + requests: + cpu: "4" + memory: "16Gi" + limits: + cpu: "4" + memory: "16Gi" + extraPodSpec: + mainContainer: + image: nvcr.io/nvidia/ai-dynamo/modelexpress-server:my-tag + imagePullPolicy: IfNotPresent + env: + - name: MODEL_EXPRESS_SERVER_PORT + value: "8000" + - name: MODEL_EXPRESS_LOGGING_LEVEL + value: "info" + - name: MODEL_EXPRESS_DATABASE_PATH + value: "/model/models.db" + command: + - /bin/sh + - -c + args: + - | + echo "Setting up Model Express configuration..." + + mkdir -p $MODEL_EXPRESS_CACHE_DIRECTORY + cat > $MODEL_EXPRESS_CACHE_DIRECTORY/config.yaml << EOF + local_path: $MODEL_EXPRESS_CACHE_DIRECTORY + server_endpoint: http://localhost:8000 + timeout_secs: null + EOF + + ./modelexpress-server & + + SERVER_PID=$! + echo "Server started with PID: $SERVER_PID" + wait $SERVER_PID + volumeMounts: + - name: model-cache-pvc + mountPoint: /model Planner: dynamoNamespace: dynamo envFromSecret: hf-token-secret @@ -74,6 +135,9 @@ spec: - "12345" - --host - "0.0.0.0" + volumeMounts: + - name: model-cache-pvc + mountPoint: /model prefill: dynamoNamespace: dynamo envFromSecret: hf-token-secret @@ -110,3 +174,6 @@ spec: - "12345" - --host - "0.0.0.0" + volumeMounts: + - name: model-cache-pvc + mountPoint: /model diff --git a/examples/backends/sglang/deploy/model_cache_pvc.yaml b/examples/backends/sglang/deploy/model_cache_pvc.yaml new file mode 100644 index 000000000000..56ad5f005224 --- /dev/null +++ b/examples/backends/sglang/deploy/model_cache_pvc.yaml @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: model-cache-pvc +spec: + accessModes: + - ReadWriteMany + storageClassName: csi-mounted-fs-path-sc + resources: + requests: + storage: 256Gi + limits: + storage: 256Gi diff --git a/examples/backends/trtllm/deploy/README.md b/examples/backends/trtllm/deploy/README.md index 0f7aecd9f2f0..7ba16d40ae7d 100644 --- a/examples/backends/trtllm/deploy/README.md +++ b/examples/backends/trtllm/deploy/README.md @@ -9,6 +9,7 @@ Basic deployment pattern with frontend and a single worker. **Architecture:** - `Frontend`: OpenAI-compatible API server (with kv router mode disabled) +- `ModelExpress`: Shared model caching service across workers - `TRTLLMWorker`: Single worker handling both prefill and decode ### 2. **Aggregated Router Deployment** (`agg_router.yaml`) @@ -16,6 +17,7 @@ Enhanced aggregated deployment with KV cache routing capabilities. **Architecture:** - `Frontend`: OpenAI-compatible API server (with kv router mode enabled) +- `ModelExpress`: Shared model caching service across workers - `TRTLLMWorker`: Multiple workers handling both prefill and decode (2 replicas for load balancing) ### 3. **Disaggregated Deployment** (`disagg.yaml`) @@ -23,6 +25,7 @@ High-performance deployment with separated prefill and decode workers. **Architecture:** - `Frontend`: HTTP API server coordinating between workers +- `ModelExpress`: Shared model caching service across workers - `TRTLLMDecodeWorker`: Specialized decode-only worker - `TRTLLMPrefillWorker`: Specialized prefill-only worker @@ -31,6 +34,7 @@ Advanced disaggregated deployment with KV cache routing capabilities. **Architecture:** - `Frontend`: HTTP API server (with kv router mode enabled) +- `ModelExpress`: Shared model caching service across workers - `TRTLLMDecodeWorker`: Specialized decode-only worker - `TRTLLMPrefillWorker`: Specialized prefill-only worker (2 replicas for load balancing) @@ -40,6 +44,7 @@ Aggregated deployment with custom configuration. **Architecture:** - `nvidia-config`: ConfigMap containing a custom trtllm configuration - `Frontend`: OpenAI-compatible API server (with kv router mode disabled) +- `ModelExpress`: Shared model caching service across workers - `TRTLLMWorker`: Single worker handling both prefill and decode with custom configuration mounted from the configmap ### 6. **Disaggregated Planner Deployment** (`disagg_planner.yaml`) @@ -47,6 +52,7 @@ Advanced disaggregated deployment with SLA-based automatic scaling. **Architecture:** - `Frontend`: HTTP API server coordinating between workers +- `ModelExpress`: Shared model caching service across workers - `Planner`: SLA-based planner that monitors performance and scales workers automatically - `Prometheus`: Metrics collection and monitoring - `TRTLLMDecodeWorker`: Specialized decode-only worker @@ -107,6 +113,16 @@ Before using these templates, ensure you have: 3. **Container registry access** for TensorRT-LLM runtime images 4. **HuggingFace token secret** (referenced as `envFromSecret: hf-token-secret`) +### Persistent Volume Claim (PVC) + +All templates expect a pre-created PVC named `model-cache-pvc` for the shared model cache used by ModelExpress and TensorRT-LLM workers. + +Apply the shared PVC once per namespace before deploying any graph: + +```bash +kubectl apply -f model_cache_pvc.yaml -n $NAMESPACE +``` + ### Container Images The deployment files currently require access to `my-registry/trtllm-runtime`. If you don't have access, build and push your own image: @@ -170,6 +186,11 @@ Export the NAMESPACE you used in your Dynamo Cloud Installation. ```bash cd dynamo/examples/backends/trtllm/deploy export DEPLOYMENT_FILE=agg.yaml + +# Create the shared model cache PVC (run once per namespace) +kubectl apply -f model_cache_pvc.yaml -n $NAMESPACE + +# Apply the deployment kubectl apply -f $DEPLOYMENT_FILE -n $NAMESPACE ``` diff --git a/examples/backends/trtllm/deploy/agg-with-config.yaml b/examples/backends/trtllm/deploy/agg-with-config.yaml index d18d1b0fb2fb..a263c3bb5de4 100644 --- a/examples/backends/trtllm/deploy/agg-with-config.yaml +++ b/examples/backends/trtllm/deploy/agg-with-config.yaml @@ -27,6 +27,16 @@ kind: DynamoGraphDeployment metadata: name: trtllm-agg spec: + pvcs: + - name: model-cache-pvc + create: false + envs: + - name: MODEL_EXPRESS_CACHE_DIRECTORY + value: "/model/.model-express/cache" + - name: HF_HUB_CACHE + value: "/model/.model-express/cache" + - name: MODEL_EXPRESS_URL + value: "http://trtllm-agg-modelexpress:8000" services: Frontend: dynamoNamespace: trtllm-agg @@ -35,6 +45,58 @@ spec: extraPodSpec: mainContainer: image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:my-tag + ModelExpress: + envFromSecret: hf-token-secret + dynamoNamespace: trtllm-agg + componentType: frontend + readinessProbe: + tcpSocket: + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + replicas: 1 + resources: + requests: + cpu: "4" + memory: "16Gi" + limits: + cpu: "4" + memory: "16Gi" + extraPodSpec: + mainContainer: + image: nvcr.io/nvidia/ai-dynamo/modelexpress-server:my-tag + imagePullPolicy: IfNotPresent + env: + - name: MODEL_EXPRESS_SERVER_PORT + value: "8000" + - name: MODEL_EXPRESS_LOGGING_LEVEL + value: "info" + - name: MODEL_EXPRESS_DATABASE_PATH + value: "/model/models.db" + command: + - /bin/sh + - -c + args: + - | + echo "Setting up Model Express configuration..." + + mkdir -p $MODEL_EXPRESS_CACHE_DIRECTORY + cat > $MODEL_EXPRESS_CACHE_DIRECTORY/config.yaml << EOF + local_path: $MODEL_EXPRESS_CACHE_DIRECTORY + server_endpoint: http://localhost:8000 + timeout_secs: null + EOF + + ./modelexpress-server & + + SERVER_PID=$! + echo "Server started with PID: $SERVER_PID" + wait $SERVER_PID + volumeMounts: + - name: model-cache-pvc + mountPoint: /model TRTLLMWorker: envFromSecret: hf-token-secret dynamoNamespace: trtllm-agg @@ -68,3 +130,6 @@ spec: - Qwen/Qwen3-0.6B - --extra-engine-args - ./recipes/qwen3/trtllm/agg.yaml + volumeMounts: + - name: model-cache-pvc + mountPoint: /model diff --git a/examples/backends/trtllm/deploy/agg.yaml b/examples/backends/trtllm/deploy/agg.yaml index 54412576a258..616fd3dae7c3 100644 --- a/examples/backends/trtllm/deploy/agg.yaml +++ b/examples/backends/trtllm/deploy/agg.yaml @@ -6,6 +6,16 @@ kind: DynamoGraphDeployment metadata: name: trtllm-agg spec: + pvcs: + - name: model-cache-pvc + create: false + envs: + - name: MODEL_EXPRESS_CACHE_DIRECTORY + value: "/model/.model-express/cache" + - name: HF_HUB_CACHE + value: "/model/.model-express/cache" + - name: MODEL_EXPRESS_URL + value: "http://trtllm-agg-modelexpress:8000" services: Frontend: dynamoNamespace: trtllm-agg @@ -14,6 +24,58 @@ spec: extraPodSpec: mainContainer: image: my-registry/trtllm-runtime:my-tag + ModelExpress: + envFromSecret: hf-token-secret + dynamoNamespace: trtllm-agg + componentType: frontend + readinessProbe: + tcpSocket: + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + replicas: 1 + resources: + requests: + cpu: "4" + memory: "16Gi" + limits: + cpu: "4" + memory: "16Gi" + extraPodSpec: + mainContainer: + image: nvcr.io/nvidia/ai-dynamo/modelexpress-server:my-tag + imagePullPolicy: IfNotPresent + env: + - name: MODEL_EXPRESS_SERVER_PORT + value: "8000" + - name: MODEL_EXPRESS_LOGGING_LEVEL + value: "info" + - name: MODEL_EXPRESS_DATABASE_PATH + value: "/model/models.db" + command: + - /bin/sh + - -c + args: + - | + echo "Setting up Model Express configuration..." + + mkdir -p $MODEL_EXPRESS_CACHE_DIRECTORY + cat > $MODEL_EXPRESS_CACHE_DIRECTORY/config.yaml << EOF + local_path: $MODEL_EXPRESS_CACHE_DIRECTORY + server_endpoint: http://localhost:8000 + timeout_secs: null + EOF + + ./modelexpress-server & + + SERVER_PID=$! + echo "Server started with PID: $SERVER_PID" + wait $SERVER_PID + volumeMounts: + - name: model-cache-pvc + mountPoint: /model TRTLLMWorker: envFromSecret: hf-token-secret dynamoNamespace: trtllm-agg @@ -37,3 +99,6 @@ spec: - Qwen/Qwen3-0.6B - --extra-engine-args - ./recipes/qwen3/trtllm/agg.yaml + volumeMounts: + - name: model-cache-pvc + mountPoint: /model diff --git a/examples/backends/trtllm/deploy/agg_router.yaml b/examples/backends/trtllm/deploy/agg_router.yaml index ed42129fb421..7e438764dfcb 100644 --- a/examples/backends/trtllm/deploy/agg_router.yaml +++ b/examples/backends/trtllm/deploy/agg_router.yaml @@ -6,6 +6,16 @@ kind: DynamoGraphDeployment metadata: name: trtllm-agg-router spec: + pvcs: + - name: model-cache-pvc + create: false + envs: + - name: MODEL_EXPRESS_CACHE_DIRECTORY + value: "/model/.model-express/cache" + - name: HF_HUB_CACHE + value: "/model/.model-express/cache" + - name: MODEL_EXPRESS_URL + value: "http://trtllm-agg-router-modelexpress:8000" services: Frontend: dynamoNamespace: trtllm-agg-router @@ -17,6 +27,58 @@ spec: envs: - name: DYN_ROUTER_MODE value: kv + ModelExpress: + envFromSecret: hf-token-secret + dynamoNamespace: trtllm-agg-router + componentType: frontend + readinessProbe: + tcpSocket: + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + replicas: 1 + resources: + requests: + cpu: "4" + memory: "16Gi" + limits: + cpu: "4" + memory: "16Gi" + extraPodSpec: + mainContainer: + image: nvcr.io/nvidia/ai-dynamo/modelexpress-server:my-tag + imagePullPolicy: IfNotPresent + env: + - name: MODEL_EXPRESS_SERVER_PORT + value: "8000" + - name: MODEL_EXPRESS_LOGGING_LEVEL + value: "info" + - name: MODEL_EXPRESS_DATABASE_PATH + value: "/model/models.db" + command: + - /bin/sh + - -c + args: + - | + echo "Setting up Model Express configuration..." + + mkdir -p $MODEL_EXPRESS_CACHE_DIRECTORY + cat > $MODEL_EXPRESS_CACHE_DIRECTORY/config.yaml << EOF + local_path: $MODEL_EXPRESS_CACHE_DIRECTORY + server_endpoint: http://localhost:8000 + timeout_secs: null + EOF + + ./modelexpress-server & + + SERVER_PID=$! + echo "Server started with PID: $SERVER_PID" + wait $SERVER_PID + volumeMounts: + - name: model-cache-pvc + mountPoint: /model TRTLLMWorker: envFromSecret: hf-token-secret dynamoNamespace: trtllm-agg-router @@ -41,3 +103,6 @@ spec: - --extra-engine-args - ./recipes/qwen3/trtllm/agg.yaml - --publish-events-and-metrics + volumeMounts: + - name: model-cache-pvc + mountPoint: /model diff --git a/examples/backends/trtllm/deploy/disagg-multinode.yaml b/examples/backends/trtllm/deploy/disagg-multinode.yaml index 6ba847157379..5dec109e66c3 100644 --- a/examples/backends/trtllm/deploy/disagg-multinode.yaml +++ b/examples/backends/trtllm/deploy/disagg-multinode.yaml @@ -62,17 +62,6 @@ data: cache_transceiver_config: backend: DEFAULT --- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: models -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 100Gi ---- apiVersion: nvidia.com/v1alpha1 kind: DynamoGraphDeployment metadata: @@ -80,14 +69,19 @@ metadata: spec: backendFramework: trtllm pvcs: - - name: models + - name: model-cache-pvc + create: false envs: - name: OMPI_ALLOW_RUN_AS_ROOT value: "1" - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM value: "1" - - name: HF_HOME - value: "/models" + - name: MODEL_EXPRESS_CACHE_DIRECTORY + value: "/model/.model-express/cache" + - name: HF_HUB_CACHE + value: "/model/.model-express/cache" + - name: MODEL_EXPRESS_URL + value: "http://trtllm-disagg-modelexpress:8000" services: Frontend: dynamoNamespace: trtllm-disagg @@ -104,10 +98,62 @@ spec: args: - --http-port - "8000" + ModelExpress: + envFromSecret: hf-token-secret + dynamoNamespace: trtllm-disagg + componentType: frontend + readinessProbe: + tcpSocket: + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + replicas: 1 + resources: + requests: + cpu: "4" + memory: "16Gi" + limits: + cpu: "4" + memory: "16Gi" + extraPodSpec: + mainContainer: + image: nvcr.io/nvidia/ai-dynamo/modelexpress-server:my-tag + imagePullPolicy: IfNotPresent + env: + - name: MODEL_EXPRESS_SERVER_PORT + value: "8000" + - name: MODEL_EXPRESS_LOGGING_LEVEL + value: "info" + - name: MODEL_EXPRESS_DATABASE_PATH + value: "/model/models.db" + command: + - /bin/sh + - -c + args: + - | + echo "Setting up Model Express configuration..." + + mkdir -p $MODEL_EXPRESS_CACHE_DIRECTORY + cat > $MODEL_EXPRESS_CACHE_DIRECTORY/config.yaml << EOF + local_path: $MODEL_EXPRESS_CACHE_DIRECTORY + server_endpoint: http://localhost:8000 + timeout_secs: null + EOF + + ./modelexpress-server & + + SERVER_PID=$! + echo "Server started with PID: $SERVER_PID" + wait $SERVER_PID + volumeMounts: + - name: model-cache-pvc + mountPoint: /model prefill: volumeMounts: - - name: models - mountPoint: /models + - name: model-cache-pvc + mountPoint: /model dynamoNamespace: trtllm-disagg envFromSecret: hf-token-secret componentType: worker @@ -146,8 +192,8 @@ spec: - decode_first decode: volumeMounts: - - name: models - mountPoint: /models + - name: model-cache-pvc + mountPoint: /model dynamoNamespace: trtllm-disagg envFromSecret: hf-token-secret componentType: worker diff --git a/examples/backends/trtllm/deploy/disagg.yaml b/examples/backends/trtllm/deploy/disagg.yaml index 501d2a4c200e..578708c36b77 100644 --- a/examples/backends/trtllm/deploy/disagg.yaml +++ b/examples/backends/trtllm/deploy/disagg.yaml @@ -6,6 +6,16 @@ kind: DynamoGraphDeployment metadata: name: trtllm-disagg spec: + pvcs: + - name: model-cache-pvc + create: false + envs: + - name: MODEL_EXPRESS_CACHE_DIRECTORY + value: "/model/.model-express/cache" + - name: HF_HUB_CACHE + value: "/model/.model-express/cache" + - name: MODEL_EXPRESS_URL + value: "http://trtllm-disagg-modelexpress:8000" services: Frontend: dynamoNamespace: trtllm-disagg @@ -14,6 +24,58 @@ spec: extraPodSpec: mainContainer: image: my-registry/trtllm-runtime:my-tag + ModelExpress: + envFromSecret: hf-token-secret + dynamoNamespace: trtllm-disagg + componentType: frontend + readinessProbe: + tcpSocket: + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + replicas: 1 + resources: + requests: + cpu: "4" + memory: "16Gi" + limits: + cpu: "4" + memory: "16Gi" + extraPodSpec: + mainContainer: + image: nvcr.io/nvidia/ai-dynamo/modelexpress-server:my-tag + imagePullPolicy: IfNotPresent + env: + - name: MODEL_EXPRESS_SERVER_PORT + value: "8000" + - name: MODEL_EXPRESS_LOGGING_LEVEL + value: "info" + - name: MODEL_EXPRESS_DATABASE_PATH + value: "/model/models.db" + command: + - /bin/sh + - -c + args: + - | + echo "Setting up Model Express configuration..." + + mkdir -p $MODEL_EXPRESS_CACHE_DIRECTORY + cat > $MODEL_EXPRESS_CACHE_DIRECTORY/config.yaml << EOF + local_path: $MODEL_EXPRESS_CACHE_DIRECTORY + server_endpoint: http://localhost:8000 + timeout_secs: null + EOF + + ./modelexpress-server & + + SERVER_PID=$! + echo "Server started with PID: $SERVER_PID" + wait $SERVER_PID + volumeMounts: + - name: model-cache-pvc + mountPoint: /model TRTLLMPrefillWorker: dynamoNamespace: trtllm-disagg envFromSecret: hf-token-secret @@ -42,6 +104,9 @@ spec: - prefill - --disaggregation-strategy - decode_first + volumeMounts: + - name: model-cache-pvc + mountPoint: /model TRTLLMDecodeWorker: dynamoNamespace: trtllm-disagg envFromSecret: hf-token-secret @@ -70,3 +135,6 @@ spec: - decode - --disaggregation-strategy - decode_first + volumeMounts: + - name: model-cache-pvc + mountPoint: /model diff --git a/examples/backends/trtllm/deploy/disagg_planner.yaml b/examples/backends/trtllm/deploy/disagg_planner.yaml index 9324bd50896f..e022be59317e 100644 --- a/examples/backends/trtllm/deploy/disagg_planner.yaml +++ b/examples/backends/trtllm/deploy/disagg_planner.yaml @@ -9,6 +9,15 @@ spec: pvcs: - name: dynamo-pvc create: false + - name: model-cache-pvc + create: false + envs: + - name: MODEL_EXPRESS_CACHE_DIRECTORY + value: "/model/.model-express/cache" + - name: HF_HUB_CACHE + value: "/model/.model-express/cache" + - name: MODEL_EXPRESS_URL + value: "http://trtllm-disagg-planner-modelexpress:8000" services: Frontend: dynamoNamespace: trtllm-disagg-planner @@ -34,6 +43,58 @@ spec: - --router-temperature - "0.0" - --no-kv-events + ModelExpress: + envFromSecret: hf-token-secret + dynamoNamespace: trtllm-disagg-planner + componentType: frontend + readinessProbe: + tcpSocket: + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + replicas: 1 + resources: + requests: + cpu: "4" + memory: "16Gi" + limits: + cpu: "4" + memory: "16Gi" + extraPodSpec: + mainContainer: + image: nvcr.io/nvidia/ai-dynamo/modelexpress-server:my-tag + imagePullPolicy: IfNotPresent + env: + - name: MODEL_EXPRESS_SERVER_PORT + value: "8000" + - name: MODEL_EXPRESS_LOGGING_LEVEL + value: "info" + - name: MODEL_EXPRESS_DATABASE_PATH + value: "/model/models.db" + command: + - /bin/sh + - -c + args: + - | + echo "Setting up Model Express configuration..." + + mkdir -p $MODEL_EXPRESS_CACHE_DIRECTORY + cat > $MODEL_EXPRESS_CACHE_DIRECTORY/config.yaml << EOF + local_path: $MODEL_EXPRESS_CACHE_DIRECTORY + server_endpoint: http://localhost:8000 + timeout_secs: null + EOF + + ./modelexpress-server & + + SERVER_PID=$! + echo "Server started with PID: $SERVER_PID" + wait $SERVER_PID + volumeMounts: + - name: model-cache-pvc + mountPoint: /model Planner: dynamoNamespace: trtllm-disagg-planner envFromSecret: hf-token-secret @@ -102,6 +163,9 @@ spec: - decode - --disaggregation-strategy - decode_first + volumeMounts: + - name: model-cache-pvc + mountPoint: /model TRTLLMPrefillWorker: dynamoNamespace: trtllm-disagg-planner envFromSecret: hf-token-secret @@ -131,3 +195,6 @@ spec: - prefill - --disaggregation-strategy - decode_first + volumeMounts: + - name: model-cache-pvc + mountPoint: /model diff --git a/examples/backends/trtllm/deploy/disagg_router.yaml b/examples/backends/trtllm/deploy/disagg_router.yaml index f687354a9546..2c4c3f9ff17f 100644 --- a/examples/backends/trtllm/deploy/disagg_router.yaml +++ b/examples/backends/trtllm/deploy/disagg_router.yaml @@ -6,6 +6,16 @@ kind: DynamoGraphDeployment metadata: name: trtllm-v1-disagg-router spec: + pvcs: + - name: model-cache-pvc + create: false + envs: + - name: MODEL_EXPRESS_CACHE_DIRECTORY + value: "/model/.model-express/cache" + - name: HF_HUB_CACHE + value: "/model/.model-express/cache" + - name: MODEL_EXPRESS_URL + value: "http://trtllm-v1-disagg-router-modelexpress:8000" services: Frontend: dynamoNamespace: trtllm-v1-disagg-router @@ -17,6 +27,58 @@ spec: envs: - name: DYN_ROUTER_MODE value: kv + ModelExpress: + envFromSecret: hf-token-secret + dynamoNamespace: trtllm-v1-disagg-router + componentType: frontend + readinessProbe: + tcpSocket: + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + replicas: 1 + resources: + requests: + cpu: "4" + memory: "16Gi" + limits: + cpu: "4" + memory: "16Gi" + extraPodSpec: + mainContainer: + image: nvcr.io/nvidia/ai-dynamo/modelexpress-server:my-tag + imagePullPolicy: IfNotPresent + env: + - name: MODEL_EXPRESS_SERVER_PORT + value: "8000" + - name: MODEL_EXPRESS_LOGGING_LEVEL + value: "info" + - name: MODEL_EXPRESS_DATABASE_PATH + value: "/model/models.db" + command: + - /bin/sh + - -c + args: + - | + echo "Setting up Model Express configuration..." + + mkdir -p $MODEL_EXPRESS_CACHE_DIRECTORY + cat > $MODEL_EXPRESS_CACHE_DIRECTORY/config.yaml << EOF + local_path: $MODEL_EXPRESS_CACHE_DIRECTORY + server_endpoint: http://localhost:8000 + timeout_secs: null + EOF + + ./modelexpress-server & + + SERVER_PID=$! + echo "Server started with PID: $SERVER_PID" + wait $SERVER_PID + volumeMounts: + - name: model-cache-pvc + mountPoint: /model TRTLLMPrefillWorker: dynamoNamespace: trtllm-v1-disagg-router envFromSecret: hf-token-secret @@ -45,6 +107,9 @@ spec: - --disaggregation-strategy - prefill_first - --publish-events-and-metrics + volumeMounts: + - name: model-cache-pvc + mountPoint: /model TRTLLMDecodeWorker: dynamoNamespace: trtllm-v1-disagg-router envFromSecret: hf-token-secret @@ -72,3 +137,6 @@ spec: - decode - --disaggregation-strategy - prefill_first + volumeMounts: + - name: model-cache-pvc + mountPoint: /model diff --git a/examples/backends/trtllm/deploy/model_cache_pvc.yaml b/examples/backends/trtllm/deploy/model_cache_pvc.yaml new file mode 100644 index 000000000000..7748488f4850 --- /dev/null +++ b/examples/backends/trtllm/deploy/model_cache_pvc.yaml @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: model-cache-pvc +spec: + accessModes: + - ReadWriteMany + storageClassName: csi-mounted-fs-path-sc + resources: + requests: + storage: 256Gi + limits: + storage: 256Gi + + diff --git a/examples/backends/vllm/deploy/README.md b/examples/backends/vllm/deploy/README.md index 5f2c11dc4c38..28f4fa4d2c6a 100644 --- a/examples/backends/vllm/deploy/README.md +++ b/examples/backends/vllm/deploy/README.md @@ -9,6 +9,7 @@ Basic deployment pattern with frontend and a single decode worker. **Architecture:** - `Frontend`: OpenAI-compatible API server (with kv router mode disabled) +- `ModelExpress`: Shared model caching service across workers - `VLLMDecodeWorker`: Single worker handling both prefill and decode ### 2. **Aggregated Router Deployment** (`agg_router.yaml`) @@ -16,6 +17,7 @@ Enhanced aggregated deployment with KV cache routing capabilities. **Architecture:** - `Frontend`: OpenAI-compatible API server (with kv router mode enabled) +- `ModelExpress`: Shared model caching service across workers - `VLLMDecodeWorker`: Single worker handling both prefill and decode ### 3. **Disaggregated Deployment** (`disagg.yaml`) @@ -23,6 +25,7 @@ High-performance deployment with separated prefill and decode workers. **Architecture:** - `Frontend`: HTTP API server coordinating between workers +- `ModelExpress`: Shared model caching service across workers - `VLLMDecodeWorker`: Specialized decode-only worker - `VLLMPrefillWorker`: Specialized prefill-only worker (`--is-prefill-worker`) - Communication via NIXL transfer backend @@ -32,6 +35,7 @@ Advanced disaggregated deployment with KV cache routing capabilities. **Architecture:** - `Frontend`: HTTP API server with KV-aware routing +- `ModelExpress`: Shared model caching service across workers - `VLLMDecodeWorker`: Specialized decode-only worker - `VLLMPrefillWorker`: Specialized prefill-only worker (`--is-prefill-worker`) @@ -87,6 +91,18 @@ Before using these templates, ensure you have: 3. **Container registry access** for vLLM runtime images 4. **HuggingFace token secret** (referenced as `envFromSecret: hf-token-secret`) +### Persistent Volume Claim (PVC) + +All templates expect a pre-created PVC named `model-cache-pvc` for the shared model cache used by ModelExpress and vLLM workers. + +Apply the shared PVC once per namespace before deploying any graph: + +```bash +kubectl apply -f model_cache_pvc.yaml -n $NAMESPACE +``` + +Note: If your cluster requires a specific storage class, edit `model_cache_pvc.yaml` to set `storageClassName` accordingly. + ### Container Images We have public images available on [NGC Catalog](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/ai-dynamo/collections/ai-dynamo/artifacts). If you'd prefer to use your own registry, build and push your own image: @@ -144,6 +160,9 @@ Export the NAMESPACE you used in your Dynamo Cloud Installation. cd /examples/backends/vllm/deploy export DEPLOYMENT_FILE=agg.yaml +# Create the shared model cache PVC (run once per namespace) +kubectl apply -f model_cache_pvc.yaml -n $NAMESPACE + kubectl apply -f $DEPLOYMENT_FILE -n $NAMESPACE ``` diff --git a/examples/backends/vllm/deploy/agg.yaml b/examples/backends/vllm/deploy/agg.yaml index 544fb7912415..81bee1308e44 100644 --- a/examples/backends/vllm/deploy/agg.yaml +++ b/examples/backends/vllm/deploy/agg.yaml @@ -6,6 +6,16 @@ kind: DynamoGraphDeployment metadata: name: vllm-agg spec: + pvcs: + - name: model-cache-pvc + create: false + envs: + - name: MODEL_EXPRESS_CACHE_DIRECTORY + value: "/model/.model-express/cache" + - name: HF_HUB_CACHE + value: "/model/.model-express/cache" + - name: MODEL_EXPRESS_URL + value: "http://vllm-agg-modelexpress:8000" services: Frontend: dynamoNamespace: vllm-agg @@ -13,7 +23,59 @@ spec: replicas: 1 extraPodSpec: mainContainer: - image: gitlab-master.nvidia.com:5005/dl/ai-dynamo/dynamo/dynamo:vllm-1029 + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag + ModelExpress: + envFromSecret: hf-token-secret + dynamoNamespace: vllm-agg + componentType: frontend + readinessProbe: + tcpSocket: + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + replicas: 1 + resources: + requests: + cpu: "4" + memory: "16Gi" + limits: + cpu: "4" + memory: "16Gi" + extraPodSpec: + mainContainer: + image: nvcr.io/nvidia/ai-dynamo/modelexpress-server:my-tag + imagePullPolicy: IfNotPresent + env: + - name: MODEL_EXPRESS_SERVER_PORT + value: "8000" + - name: MODEL_EXPRESS_LOGGING_LEVEL + value: "info" + - name: MODEL_EXPRESS_DATABASE_PATH + value: "/model/models.db" + command: + - /bin/sh + - -c + args: + - | + echo "Setting up Model Express configuration..." + + mkdir -p $MODEL_EXPRESS_CACHE_DIRECTORY + cat > $MODEL_EXPRESS_CACHE_DIRECTORY/config.yaml << EOF + local_path: $MODEL_EXPRESS_CACHE_DIRECTORY + server_endpoint: http://localhost:8000 + timeout_secs: null + EOF + + ./modelexpress-server & + + SERVER_PID=$! + echo "Server started with PID: $SERVER_PID" + wait $SERVER_PID + volumeMounts: + - name: model-cache-pvc + mountPoint: /model VllmDecodeWorker: envFromSecret: hf-token-secret dynamoNamespace: vllm-agg @@ -24,7 +86,7 @@ spec: gpu: "1" extraPodSpec: mainContainer: - image: gitlab-master.nvidia.com:5005/dl/ai-dynamo/dynamo/dynamo:vllm-1029 + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag workingDir: /workspace/examples/backends/vllm command: - python3 @@ -33,3 +95,6 @@ spec: args: - --model - Qwen/Qwen3-0.6B + volumeMounts: + - name: model-cache-pvc + mountPoint: /model diff --git a/examples/backends/vllm/deploy/agg_kvbm.yaml b/examples/backends/vllm/deploy/agg_kvbm.yaml index 62e28386aa2a..24790aeda9ad 100644 --- a/examples/backends/vllm/deploy/agg_kvbm.yaml +++ b/examples/backends/vllm/deploy/agg_kvbm.yaml @@ -6,6 +6,16 @@ kind: DynamoGraphDeployment metadata: name: vllm-agg-kvbm spec: + pvcs: + - name: model-cache-pvc + create: false + envs: + - name: MODEL_EXPRESS_CACHE_DIRECTORY + value: "/model/.model-express/cache" + - name: HF_HUB_CACHE + value: "/model/.model-express/cache" + - name: MODEL_EXPRESS_URL + value: "http://vllm-agg-kvbm-modelexpress:8000" services: Frontend: dynamoNamespace: vllm-agg-kvbm @@ -14,6 +24,58 @@ spec: extraPodSpec: mainContainer: image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag + ModelExpress: + envFromSecret: hf-token-secret + dynamoNamespace: vllm-agg-kvbm + componentType: frontend + readinessProbe: + tcpSocket: + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + replicas: 1 + resources: + requests: + cpu: "4" + memory: "16Gi" + limits: + cpu: "4" + memory: "16Gi" + extraPodSpec: + mainContainer: + image: nvcr.io/nvidia/ai-dynamo/modelexpress-server:my-tag + imagePullPolicy: IfNotPresent + env: + - name: MODEL_EXPRESS_SERVER_PORT + value: "8000" + - name: MODEL_EXPRESS_LOGGING_LEVEL + value: "info" + - name: MODEL_EXPRESS_DATABASE_PATH + value: "/model/models.db" + command: + - /bin/sh + - -c + args: + - | + echo "Setting up Model Express configuration..." + + mkdir -p $MODEL_EXPRESS_CACHE_DIRECTORY + cat > $MODEL_EXPRESS_CACHE_DIRECTORY/config.yaml << EOF + local_path: $MODEL_EXPRESS_CACHE_DIRECTORY + server_endpoint: http://localhost:8000 + timeout_secs: null + EOF + + ./modelexpress-server & + + SERVER_PID=$! + echo "Server started with PID: $SERVER_PID" + wait $SERVER_PID + volumeMounts: + - name: model-cache-pvc + mountPoint: /model VllmDecodeWorker: envFromSecret: hf-token-secret dynamoNamespace: vllm-agg-kvbm @@ -29,6 +91,7 @@ spec: envs: - name: DYN_KVBM_CPU_CACHE_GB value: "100" + extraPodSpec: mainContainer: image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag @@ -48,3 +111,6 @@ spec: - --enforce-eager - --connector - kvbm + volumeMounts: + - name: model-cache-pvc + mountPoint: /model diff --git a/examples/backends/vllm/deploy/agg_router.yaml b/examples/backends/vllm/deploy/agg_router.yaml index 26b961a06198..650a5baf7c1a 100644 --- a/examples/backends/vllm/deploy/agg_router.yaml +++ b/examples/backends/vllm/deploy/agg_router.yaml @@ -6,6 +6,16 @@ kind: DynamoGraphDeployment metadata: name: vllm-agg-router spec: + pvcs: + - name: model-cache-pvc + create: false + envs: + - name: MODEL_EXPRESS_CACHE_DIRECTORY + value: "/model/.model-express/cache" + - name: HF_HUB_CACHE + value: "/model/.model-express/cache" + - name: MODEL_EXPRESS_URL + value: "http://vllm-agg-router-modelexpress:8000" services: Frontend: dynamoNamespace: vllm-agg-router @@ -17,6 +27,58 @@ spec: envs: - name: DYN_ROUTER_MODE value: kv + ModelExpress: + envFromSecret: hf-token-secret + dynamoNamespace: vllm-agg-router + componentType: frontend + readinessProbe: + tcpSocket: + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + replicas: 1 + resources: + requests: + cpu: "4" + memory: "16Gi" + limits: + cpu: "4" + memory: "16Gi" + extraPodSpec: + mainContainer: + image: nvcr.io/nvidia/ai-dynamo/modelexpress-server:my-tag + imagePullPolicy: IfNotPresent + env: + - name: MODEL_EXPRESS_SERVER_PORT + value: "8000" + - name: MODEL_EXPRESS_LOGGING_LEVEL + value: "info" + - name: MODEL_EXPRESS_DATABASE_PATH + value: "/model/models.db" + command: + - /bin/sh + - -c + args: + - | + echo "Setting up Model Express configuration..." + + mkdir -p $MODEL_EXPRESS_CACHE_DIRECTORY + cat > $MODEL_EXPRESS_CACHE_DIRECTORY/config.yaml << EOF + local_path: $MODEL_EXPRESS_CACHE_DIRECTORY + server_endpoint: http://localhost:8000 + timeout_secs: null + EOF + + ./modelexpress-server & + + SERVER_PID=$! + echo "Server started with PID: $SERVER_PID" + wait $SERVER_PID + volumeMounts: + - name: model-cache-pvc + mountPoint: /model VllmDecodeWorker: envFromSecret: hf-token-secret dynamoNamespace: vllm-agg-router @@ -36,3 +98,6 @@ spec: args: - --model - Qwen/Qwen3-0.6B + volumeMounts: + - name: model-cache-pvc + mountPoint: /model diff --git a/examples/backends/vllm/deploy/disagg-multinode.yaml b/examples/backends/vllm/deploy/disagg-multinode.yaml index bfb56e6daf05..aaf1719d2576 100644 --- a/examples/backends/vllm/deploy/disagg-multinode.yaml +++ b/examples/backends/vllm/deploy/disagg-multinode.yaml @@ -6,6 +6,16 @@ kind: DynamoGraphDeployment metadata: name: vllm-disagg spec: + pvcs: + - name: model-cache-pvc + create: false + envs: + - name: MODEL_EXPRESS_CACHE_DIRECTORY + value: "/model/.model-express/cache" + - name: HF_HUB_CACHE + value: "/model/.model-express/cache" + - name: MODEL_EXPRESS_URL + value: "http://vllm-disagg-modelexpress:8000" services: Frontend: dynamoNamespace: vllm-disagg @@ -22,6 +32,58 @@ spec: args: - --http-port - "8000" + ModelExpress: + envFromSecret: hf-token-secret + dynamoNamespace: vllm-disagg + componentType: frontend + readinessProbe: + tcpSocket: + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + replicas: 1 + resources: + requests: + cpu: "4" + memory: "16Gi" + limits: + cpu: "4" + memory: "16Gi" + extraPodSpec: + mainContainer: + image: nvcr.io/nvidia/ai-dynamo/modelexpress-server:my-tag + imagePullPolicy: IfNotPresent + env: + - name: MODEL_EXPRESS_SERVER_PORT + value: "8000" + - name: MODEL_EXPRESS_LOGGING_LEVEL + value: "info" + - name: MODEL_EXPRESS_DATABASE_PATH + value: "/model/models.db" + command: + - /bin/sh + - -c + args: + - | + echo "Setting up Model Express configuration..." + + mkdir -p $MODEL_EXPRESS_CACHE_DIRECTORY + cat > $MODEL_EXPRESS_CACHE_DIRECTORY/config.yaml << EOF + local_path: $MODEL_EXPRESS_CACHE_DIRECTORY + server_endpoint: http://localhost:8000 + timeout_secs: null + EOF + + ./modelexpress-server & + + SERVER_PID=$! + echo "Server started with PID: $SERVER_PID" + wait $SERVER_PID + volumeMounts: + - name: model-cache-pvc + mountPoint: /model decode: dynamoNamespace: vllm-disagg envFromSecret: hf-token-secret @@ -45,6 +107,9 @@ spec: - Qwen/Qwen3-0.6B - --tensor-parallel-size - "2" + volumeMounts: + - name: model-cache-pvc + mountPoint: /model prefill: dynamoNamespace: vllm-disagg envFromSecret: hf-token-secret @@ -69,3 +134,6 @@ spec: - --is-prefill-worker - --tensor-parallel-size - "2" + volumeMounts: + - name: model-cache-pvc + mountPoint: /model diff --git a/examples/backends/vllm/deploy/disagg.yaml b/examples/backends/vllm/deploy/disagg.yaml index 799d8741b9d7..243e691736ea 100644 --- a/examples/backends/vllm/deploy/disagg.yaml +++ b/examples/backends/vllm/deploy/disagg.yaml @@ -6,6 +6,16 @@ kind: DynamoGraphDeployment metadata: name: vllm-disagg spec: + pvcs: + - name: model-cache-pvc + create: false + envs: + - name: MODEL_EXPRESS_CACHE_DIRECTORY + value: "/model/.model-express/cache" + - name: HF_HUB_CACHE + value: "/model/.model-express/cache" + - name: MODEL_EXPRESS_URL + value: "http://vllm-disagg-modelexpress:8000" services: Frontend: dynamoNamespace: vllm-disagg @@ -14,6 +24,57 @@ spec: extraPodSpec: mainContainer: image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag + ModelExpress: + envFromSecret: hf-token-secret + dynamoNamespace: vllm-disagg + componentType: frontend + readinessProbe: + tcpSocket: + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + replicas: 1 + resources: + requests: + cpu: "4" + memory: "16Gi" + limits: + cpu: "4" + memory: "16Gi" + extraPodSpec: + mainContainer: + image: nvcr.io/nvidia/ai-dynamo/modelexpress-server:my-tag + env: + - name: MODEL_EXPRESS_SERVER_PORT + value: "8000" + - name: MODEL_EXPRESS_LOGGING_LEVEL + value: "info" + - name: MODEL_EXPRESS_DATABASE_PATH + value: "/model/models.db" + command: + - /bin/sh + - -c + args: + - | + echo "Setting up Model Express configuration..." + + mkdir -p $MODEL_EXPRESS_CACHE_DIRECTORY + cat > $MODEL_EXPRESS_CACHE_DIRECTORY/config.yaml << EOF + local_path: $MODEL_EXPRESS_CACHE_DIRECTORY + server_endpoint: http://localhost:8000 + timeout_secs: null + EOF + + ./modelexpress-server & + + SERVER_PID=$! + echo "Server started with PID: $SERVER_PID" + wait $SERVER_PID + volumeMounts: + - name: model-cache-pvc + mountPoint: /model VllmDecodeWorker: dynamoNamespace: vllm-disagg envFromSecret: hf-token-secret @@ -35,6 +96,9 @@ spec: - --model - Qwen/Qwen3-0.6B - --is-decode-worker + volumeMounts: + - name: model-cache-pvc + mountPoint: /model VllmPrefillWorker: dynamoNamespace: vllm-disagg envFromSecret: hf-token-secret @@ -56,3 +120,6 @@ spec: - --model - Qwen/Qwen3-0.6B - --is-prefill-worker + volumeMounts: + - name: model-cache-pvc + mountPoint: /model diff --git a/examples/backends/vllm/deploy/disagg_kvbm.yaml b/examples/backends/vllm/deploy/disagg_kvbm.yaml index f4315a13cdd4..27dc0c122322 100644 --- a/examples/backends/vllm/deploy/disagg_kvbm.yaml +++ b/examples/backends/vllm/deploy/disagg_kvbm.yaml @@ -6,6 +6,16 @@ kind: DynamoGraphDeployment metadata: name: vllm-disagg-kvbm spec: + pvcs: + - name: model-cache-pvc + create: false + envs: + - name: MODEL_EXPRESS_CACHE_DIRECTORY + value: "/model/.model-express/cache" + - name: HF_HUB_CACHE + value: "/model/.model-express/cache" + - name: MODEL_EXPRESS_URL + value: "http://vllm-disagg-kvbm-modelexpress:8000" services: Frontend: dynamoNamespace: vllm-disagg-kvbm @@ -14,6 +24,58 @@ spec: extraPodSpec: mainContainer: image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag + ModelExpress: + envFromSecret: hf-token-secret + dynamoNamespace: vllm-disagg-kvbm + componentType: frontend + readinessProbe: + tcpSocket: + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + replicas: 1 + resources: + requests: + cpu: "4" + memory: "16Gi" + limits: + cpu: "4" + memory: "16Gi" + extraPodSpec: + mainContainer: + image: nvcr.io/nvidia/ai-dynamo/modelexpress-server:my-tag + imagePullPolicy: IfNotPresent + env: + - name: MODEL_EXPRESS_SERVER_PORT + value: "8000" + - name: MODEL_EXPRESS_LOGGING_LEVEL + value: "info" + - name: MODEL_EXPRESS_DATABASE_PATH + value: "/model/models.db" + command: + - /bin/sh + - -c + args: + - | + echo "Setting up Model Express configuration..." + + mkdir -p $MODEL_EXPRESS_CACHE_DIRECTORY + cat > $MODEL_EXPRESS_CACHE_DIRECTORY/config.yaml << EOF + local_path: $MODEL_EXPRESS_CACHE_DIRECTORY + server_endpoint: http://localhost:8000 + timeout_secs: null + EOF + + ./modelexpress-server & + + SERVER_PID=$! + echo "Server started with PID: $SERVER_PID" + wait $SERVER_PID + volumeMounts: + - name: model-cache-pvc + mountPoint: /model VllmDecodeWorker: dynamoNamespace: vllm-disagg-kvbm envFromSecret: hf-token-secret @@ -39,6 +101,9 @@ spec: - --max-model-len - "32000" - --enforce-eager + volumeMounts: + - name: model-cache-pvc + mountPoint: /model VllmPrefillWorker: dynamoNamespace: vllm-disagg-kvbm envFromSecret: hf-token-secret @@ -75,3 +140,6 @@ spec: - --connector - kvbm - nixl + volumeMounts: + - name: model-cache-pvc + mountPoint: /model diff --git a/examples/backends/vllm/deploy/disagg_kvbm_2p2d.yaml b/examples/backends/vllm/deploy/disagg_kvbm_2p2d.yaml index 1aa5281d09cf..48266742a316 100644 --- a/examples/backends/vllm/deploy/disagg_kvbm_2p2d.yaml +++ b/examples/backends/vllm/deploy/disagg_kvbm_2p2d.yaml @@ -6,6 +6,16 @@ kind: DynamoGraphDeployment metadata: name: vllm-disagg-kvbm-2p2d spec: + pvcs: + - name: model-cache-pvc + create: false + envs: + - name: MODEL_EXPRESS_CACHE_DIRECTORY + value: "/model/.model-express/cache" + - name: HF_HUB_CACHE + value: "/model/.model-express/cache" + - name: MODEL_EXPRESS_URL + value: "http://vllm-disagg-kvbm-2p2d-modelexpress:8000" services: Frontend: dynamoNamespace: vllm-disagg-kvbm-2p2d @@ -14,6 +24,58 @@ spec: extraPodSpec: mainContainer: image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag + ModelExpress: + envFromSecret: hf-token-secret + dynamoNamespace: vllm-disagg-kvbm-2p2d + componentType: frontend + readinessProbe: + tcpSocket: + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + replicas: 1 + resources: + requests: + cpu: "4" + memory: "16Gi" + limits: + cpu: "4" + memory: "16Gi" + extraPodSpec: + mainContainer: + image: nvcr.io/nvidia/ai-dynamo/modelexpress-server:my-tag + imagePullPolicy: IfNotPresent + env: + - name: MODEL_EXPRESS_SERVER_PORT + value: "8000" + - name: MODEL_EXPRESS_LOGGING_LEVEL + value: "info" + - name: MODEL_EXPRESS_DATABASE_PATH + value: "/model/models.db" + command: + - /bin/sh + - -c + args: + - | + echo "Setting up Model Express configuration..." + + mkdir -p $MODEL_EXPRESS_CACHE_DIRECTORY + cat > $MODEL_EXPRESS_CACHE_DIRECTORY/config.yaml << EOF + local_path: $MODEL_EXPRESS_CACHE_DIRECTORY + server_endpoint: http://localhost:8000 + timeout_secs: null + EOF + + ./modelexpress-server & + + SERVER_PID=$! + echo "Server started with PID: $SERVER_PID" + wait $SERVER_PID + volumeMounts: + - name: model-cache-pvc + mountPoint: /model VllmDecodeWorker: dynamoNamespace: vllm-disagg-kvbm-2p2d envFromSecret: hf-token-secret @@ -39,6 +101,9 @@ spec: - --max-model-len - "32000" - --enforce-eager + volumeMounts: + - name: model-cache-pvc + mountPoint: /model VllmPrefillWorker: dynamoNamespace: vllm-disagg-kvbm-2p2d envFromSecret: hf-token-secret @@ -75,3 +140,6 @@ spec: - --connector - kvbm - nixl + volumeMounts: + - name: model-cache-pvc + mountPoint: /model diff --git a/examples/backends/vllm/deploy/disagg_kvbm_tp2.yaml b/examples/backends/vllm/deploy/disagg_kvbm_tp2.yaml index 439b17a91fc0..d14d90faa70f 100644 --- a/examples/backends/vllm/deploy/disagg_kvbm_tp2.yaml +++ b/examples/backends/vllm/deploy/disagg_kvbm_tp2.yaml @@ -6,6 +6,16 @@ kind: DynamoGraphDeployment metadata: name: vllm-disagg-kvbm-tp2 spec: + pvcs: + - name: model-cache-pvc + create: false + envs: + - name: MODEL_EXPRESS_CACHE_DIRECTORY + value: "/model/.model-express/cache" + - name: HF_HUB_CACHE + value: "/model/.model-express/cache" + - name: MODEL_EXPRESS_URL + value: "http://vllm-disagg-kvbm-tp2-modelexpress:8000" services: Frontend: dynamoNamespace: vllm-disagg-kvbm-tp2 @@ -14,6 +24,58 @@ spec: extraPodSpec: mainContainer: image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag + ModelExpress: + envFromSecret: hf-token-secret + dynamoNamespace: vllm-disagg-kvbm-tp2 + componentType: frontend + readinessProbe: + tcpSocket: + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + replicas: 1 + resources: + requests: + cpu: "4" + memory: "16Gi" + limits: + cpu: "4" + memory: "16Gi" + extraPodSpec: + mainContainer: + image: nvcr.io/nvidia/ai-dynamo/modelexpress-server:my-tag + imagePullPolicy: IfNotPresent + env: + - name: MODEL_EXPRESS_SERVER_PORT + value: "8000" + - name: MODEL_EXPRESS_LOGGING_LEVEL + value: "info" + - name: MODEL_EXPRESS_DATABASE_PATH + value: "/model/models.db" + command: + - /bin/sh + - -c + args: + - | + echo "Setting up Model Express configuration..." + + mkdir -p $MODEL_EXPRESS_CACHE_DIRECTORY + cat > $MODEL_EXPRESS_CACHE_DIRECTORY/config.yaml << EOF + local_path: $MODEL_EXPRESS_CACHE_DIRECTORY + server_endpoint: http://localhost:8000 + timeout_secs: null + EOF + + ./modelexpress-server & + + SERVER_PID=$! + echo "Server started with PID: $SERVER_PID" + wait $SERVER_PID + volumeMounts: + - name: model-cache-pvc + mountPoint: /model VllmDecodeWorker: dynamoNamespace: vllm-disagg-kvbm-tp2 envFromSecret: hf-token-secret @@ -43,6 +105,9 @@ spec: - --enforce-eager - --tensor-parallel-size - "2" + volumeMounts: + - name: model-cache-pvc + mountPoint: /model VllmPrefillWorker: dynamoNamespace: vllm-disagg-kvbm-tp2 envFromSecret: hf-token-secret @@ -81,3 +146,6 @@ spec: - nixl - --tensor-parallel-size - "2" + volumeMounts: + - name: model-cache-pvc + mountPoint: /model diff --git a/examples/backends/vllm/deploy/disagg_planner.yaml b/examples/backends/vllm/deploy/disagg_planner.yaml index 5e33a0d86634..5eaf6b137003 100644 --- a/examples/backends/vllm/deploy/disagg_planner.yaml +++ b/examples/backends/vllm/deploy/disagg_planner.yaml @@ -9,6 +9,15 @@ spec: pvcs: - name: dynamo-pvc create: false # Must be pre-created before deployment and SLA profiler must have been run + - name: model-cache-pvc + create: false + envs: + - name: MODEL_EXPRESS_CACHE_DIRECTORY + value: "/model/.model-express/cache" + - name: HF_HUB_CACHE + value: "/model/.model-express/cache" + - name: MODEL_EXPRESS_URL + value: "http://vllm-disagg-planner-modelexpress:8000" services: Frontend: dynamoNamespace: vllm-disagg-planner @@ -17,6 +26,58 @@ spec: extraPodSpec: mainContainer: image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag + ModelExpress: + envFromSecret: hf-token-secret + dynamoNamespace: vllm-disagg-planner + componentType: frontend + readinessProbe: + tcpSocket: + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + replicas: 1 + resources: + requests: + cpu: "4" + memory: "16Gi" + limits: + cpu: "4" + memory: "16Gi" + extraPodSpec: + mainContainer: + image: nvcr.io/nvidia/ai-dynamo/modelexpress-server:my-tag + imagePullPolicy: IfNotPresent + env: + - name: MODEL_EXPRESS_SERVER_PORT + value: "8000" + - name: MODEL_EXPRESS_LOGGING_LEVEL + value: "info" + - name: MODEL_EXPRESS_DATABASE_PATH + value: "/model/models.db" + command: + - /bin/sh + - -c + args: + - | + echo "Setting up Model Express configuration..." + + mkdir -p $MODEL_EXPRESS_CACHE_DIRECTORY + cat > $MODEL_EXPRESS_CACHE_DIRECTORY/config.yaml << EOF + local_path: $MODEL_EXPRESS_CACHE_DIRECTORY + server_endpoint: http://localhost:8000 + timeout_secs: null + EOF + + ./modelexpress-server & + + SERVER_PID=$! + echo "Server started with PID: $SERVER_PID" + wait $SERVER_PID + volumeMounts: + - name: model-cache-pvc + mountPoint: /model Planner: dynamoNamespace: vllm-disagg-planner componentType: planner @@ -57,6 +118,9 @@ spec: - dynamo.vllm - --model - Qwen/Qwen3-0.6B + volumeMounts: + - name: model-cache-pvc + mountPoint: /model VllmPrefillWorker: dynamoNamespace: vllm-disagg-planner envFromSecret: hf-token-secret @@ -78,3 +142,6 @@ spec: - --model - Qwen/Qwen3-0.6B - --is-prefill-worker + volumeMounts: + - name: model-cache-pvc + mountPoint: /model diff --git a/examples/backends/vllm/deploy/disagg_router.yaml b/examples/backends/vllm/deploy/disagg_router.yaml index 0a13bace4944..fe3a92f055e9 100644 --- a/examples/backends/vllm/deploy/disagg_router.yaml +++ b/examples/backends/vllm/deploy/disagg_router.yaml @@ -6,6 +6,16 @@ kind: DynamoGraphDeployment metadata: name: vllm-v1-disagg-router spec: + pvcs: + - name: model-cache-pvc + create: false + envs: + - name: MODEL_EXPRESS_CACHE_DIRECTORY + value: "/model/.model-express/cache" + - name: HF_HUB_CACHE + value: "/model/.model-express/cache" + - name: MODEL_EXPRESS_URL + value: "http://vllm-v1-disagg-router-modelexpress:8000" services: Frontend: dynamoNamespace: vllm-v1-disagg-router @@ -17,6 +27,58 @@ spec: envs: - name: DYN_ROUTER_MODE value: kv + ModelExpress: + envFromSecret: hf-token-secret + dynamoNamespace: vllm-v1-disagg-router + componentType: frontend + readinessProbe: + tcpSocket: + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + replicas: 1 + resources: + requests: + cpu: "4" + memory: "16Gi" + limits: + cpu: "4" + memory: "16Gi" + extraPodSpec: + mainContainer: + image: nvcr.io/nvidia/ai-dynamo/modelexpress-server:my-tag + imagePullPolicy: IfNotPresent + env: + - name: MODEL_EXPRESS_SERVER_PORT + value: "8000" + - name: MODEL_EXPRESS_LOGGING_LEVEL + value: "info" + - name: MODEL_EXPRESS_DATABASE_PATH + value: "/model/models.db" + command: + - /bin/sh + - -c + args: + - | + echo "Setting up Model Express configuration..." + + mkdir -p $MODEL_EXPRESS_CACHE_DIRECTORY + cat > $MODEL_EXPRESS_CACHE_DIRECTORY/config.yaml << EOF + local_path: $MODEL_EXPRESS_CACHE_DIRECTORY + server_endpoint: http://localhost:8000 + timeout_secs: null + EOF + + ./modelexpress-server & + + SERVER_PID=$! + echo "Server started with PID: $SERVER_PID" + wait $SERVER_PID + volumeMounts: + - name: model-cache-pvc + mountPoint: /model VllmDecodeWorker: dynamoNamespace: vllm-v1-disagg-router envFromSecret: hf-token-secret @@ -37,6 +99,9 @@ spec: - --model - Qwen/Qwen3-0.6B - --is-decode-worker + volumeMounts: + - name: model-cache-pvc + mountPoint: /model VllmPrefillWorker: dynamoNamespace: vllm-v1-disagg-router envFromSecret: hf-token-secret @@ -57,3 +122,6 @@ spec: - --model - Qwen/Qwen3-0.6B - --is-prefill-worker + volumeMounts: + - name: model-cache-pvc + mountPoint: /model diff --git a/examples/backends/vllm/deploy/model_cache_pvc.yaml b/examples/backends/vllm/deploy/model_cache_pvc.yaml new file mode 100644 index 000000000000..56ad5f005224 --- /dev/null +++ b/examples/backends/vllm/deploy/model_cache_pvc.yaml @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: model-cache-pvc +spec: + accessModes: + - ReadWriteMany + storageClassName: csi-mounted-fs-path-sc + resources: + requests: + storage: 256Gi + limits: + storage: 256Gi