feat: Update Dynamo k8s deployment example to use ModelExpress

examples/backends/sglang/deploy/README.md

@@ -9,20 +9,23 @@ Basic deployment pattern with frontend and a single decode worker.
 
 **Architecture:**
 - `Frontend`: OpenAI-compatible API server
+- `ModelExpress`: Shared model caching service across workers
 - `SGLangDecodeWorker`: Single worker handling both prefill and decode
 
 ### 2. **Aggregated Router Deployment** (`agg_router.yaml`)
 Enhanced aggregated deployment with KV cache routing capabilities.
 
 **Architecture:**
 - `Frontend`: OpenAI-compatible API server with router mode enabled (`--router-mode kv`)
+- `ModelExpress`: Shared model caching service across workers
 - `SGLangDecodeWorker`: Single worker handling both prefill and decode
 
-### 3. **Disaggregated Deployment** (`disagg.yaml`)**
+### 3. **Disaggregated Deployment** (`disagg.yaml`)
 High-performance deployment with separated prefill and decode workers.
 
 **Architecture:**
 - `Frontend`: HTTP API server coordinating between workers
+- `ModelExpress`: Shared model caching service across workers
 - `SGLangDecodeWorker`: Specialized decode-only worker (`--disaggregation-mode decode`)
 - `SGLangPrefillWorker`: Specialized prefill-only worker (`--disaggregation-mode prefill`)
 - Communication via NIXL transfer backend (`--disaggregation-transfer-backend nixl`)
@@ -79,6 +82,28 @@ Before using these templates, ensure you have:
 3. **Container registry access** for SGLang runtime images
 4. **HuggingFace token secret** (referenced as `envFromSecret: hf-token-secret`)
 
+### Persistent Volume Claim (PVC)
+
+All templates expect a pre-created PVC named `model-cache-pvc` for the shared model cache used by ModelExpress and SGLang workers.
+
+Apply the shared PVC once per namespace before deploying any graph:
+
+```bash
+kubectl apply -f model_cache_pvc.yaml -n $NAMESPACE
+```
+
+Note: If your cluster requires a specific storage class, edit `model_cache_pvc.yaml` to set `storageClassName` accordingly.
+
+### Container Images
+
+We have public images available on [NGC Catalog](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/ai-dynamo/collections/ai-dynamo/artifacts). If you'd prefer to use your own registry, build and push your own image:
+
+```bash
+./container/build.sh --framework sglang
+# Tag and push to your container registry
+# Update the image references in the YAML files
+```
+
 ## Usage
 
 ### 1. Choose Your Template
@@ -118,6 +143,11 @@ Then, deploy the model using the deployment file.
 
 ```bash
 export DEPLOYMENT_FILE=agg.yaml
+
+# Create the shared model cache PVC (run once per namespace)
+kubectl apply -f model_cache_pvc.yaml -n $NAMESPACE
+
+# Apply the SGLang deployment
 kubectl apply -f $DEPLOYMENT_FILE -n ${NAMESPACE}
 ```
 
@@ -135,7 +165,7 @@ kubectl apply -f $DEPLOYMENT_FILE.generated -n $NAMESPACE
 
 ## Model Configuration
 
-All templates use **DeepSeek-R1-Distill-Llama-8B** as the default model. But you can use any sglang argument and configuration. Key parameters:
+All templates use **Qwen/Qwen3-0.6B** as the default model. You can use any SGLang arguments and configuration. Key parameters include `--model-path`, `--served-model-name`, and disaggregation flags (see YAMLs for examples).
 
 ## Monitoring and Health
 

examples/backends/sglang/deploy/agg.yaml

@@ -6,6 +6,16 @@ kind: DynamoGraphDeployment
 metadata:
   name: sglang-agg
 spec:
+  pvcs:
+    - name: model-cache-pvc
+      create: false
+  envs:
+    - name: MODEL_EXPRESS_CACHE_DIRECTORY
+      value: "/model/.model-express/cache"
+    - name: HF_HUB_CACHE
+      value: "/model/.model-express/cache"
+    - name: MODEL_EXPRESS_URL
+      value: "http://sglang-agg-modelexpress:8000"
   services:
     Frontend:
       dynamoNamespace: sglang-agg
@@ -14,6 +24,58 @@ spec:
       extraPodSpec:
         mainContainer:
           image: my-registry/sglang-runtime:my-tag
+    ModelExpress:
+      envFromSecret: hf-token-secret
+      dynamoNamespace: sglang-agg
+      componentType: frontend
+      readinessProbe:
+        tcpSocket:
+          port: 8000
+        initialDelaySeconds: 10
+        periodSeconds: 5
+        timeoutSeconds: 3
+        failureThreshold: 3
+      replicas: 1
+      resources:
+        requests:
+          cpu: "4"
+          memory: "16Gi"
+        limits:
+          cpu: "4"
+          memory: "16Gi"
+      extraPodSpec:
+        mainContainer:
+          image: nvcr.io/nvidia/ai-dynamo/modelexpress-server:my-tag
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: MODEL_EXPRESS_SERVER_PORT
+              value: "8000"
+            - name: MODEL_EXPRESS_LOGGING_LEVEL
+              value: "info"
+            - name: MODEL_EXPRESS_DATABASE_PATH
+              value: "/model/models.db"
+          command:
+            - /bin/sh
+            - -c
+          args:
+            - |
+              echo "Setting up Model Express configuration..."
+
+              mkdir -p $MODEL_EXPRESS_CACHE_DIRECTORY
+              cat > $MODEL_EXPRESS_CACHE_DIRECTORY/config.yaml << EOF
+              local_path: $MODEL_EXPRESS_CACHE_DIRECTORY
+              server_endpoint: http://localhost:8000
+              timeout_secs: null
+              EOF
+
+              ./modelexpress-server &
+
+              SERVER_PID=$!
+              echo "Server started with PID: $SERVER_PID"
+              wait $SERVER_PID
+      volumeMounts:
+        - name: model-cache-pvc
+          mountPoint: /model
     decode:
       envFromSecret: hf-token-secret
       dynamoNamespace: sglang-agg
@@ -41,4 +103,7 @@ spec:
             - "1"
             - --trust-remote-code
             - --skip-tokenizer-init
+      volumeMounts:
+        - name: model-cache-pvc
+          mountPoint: /model
 

examples/backends/sglang/deploy/agg_logging.yaml

@@ -9,6 +9,15 @@ spec:
   envs:
   - name: DYN_LOGGING_JSONL
     value: "1"
+  - name: MODEL_EXPRESS_CACHE_DIRECTORY
+    value: "/model/.model-express/cache"
+  - name: HF_HUB_CACHE
+    value: "/model/.model-express/cache"
+  - name: MODEL_EXPRESS_URL
+    value: "http://sglang-agg-modelexpress:8000"
+  pvcs:
+    - name: model-cache-pvc
+      create: false
   services:
     Frontend:
       dynamoNamespace: sglang-agg
@@ -17,6 +26,58 @@ spec:
       extraPodSpec:
         mainContainer:
           image: my-registry/sglang-runtime:my-tag
+    ModelExpress:
+      envFromSecret: hf-token-secret
+      dynamoNamespace: sglang-agg
+      componentType: frontend
+      readinessProbe:
+        tcpSocket:
+          port: 8000
+        initialDelaySeconds: 10
+        periodSeconds: 5
+        timeoutSeconds: 3
+        failureThreshold: 3
+      replicas: 1
+      resources:
+        requests:
+          cpu: "4"
+          memory: "16Gi"
+        limits:
+          cpu: "4"
+          memory: "16Gi"
+      extraPodSpec:
+        mainContainer:
+          image: nvcr.io/nvidia/ai-dynamo/modelexpress-server:my-tag
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: MODEL_EXPRESS_SERVER_PORT
+              value: "8000"
+            - name: MODEL_EXPRESS_LOGGING_LEVEL
+              value: "info"
+            - name: MODEL_EXPRESS_DATABASE_PATH
+              value: "/model/models.db"
+          command:
+            - /bin/sh
+            - -c
+          args:
+            - |
+              echo "Setting up Model Express configuration..."
+
+              mkdir -p $MODEL_EXPRESS_CACHE_DIRECTORY
+              cat > $MODEL_EXPRESS_CACHE_DIRECTORY/config.yaml << EOF
+              local_path: $MODEL_EXPRESS_CACHE_DIRECTORY
+              server_endpoint: http://localhost:8000
+              timeout_secs: null
+              EOF
+
+              ./modelexpress-server &
+
+              SERVER_PID=$!
+              echo "Server started with PID: $SERVER_PID"
+              wait $SERVER_PID
+      volumeMounts:
+        - name: model-cache-pvc
+          mountPoint: /model
     decode:
       envFromSecret: hf-token-secret
       dynamoNamespace: sglang-agg
@@ -43,4 +104,7 @@ spec:
             - --tp
             - "1"
             - --trust-remote-code
-            - --skip-tokenizer-init
+            - --skip-tokenizer-init
+      volumeMounts:
+        - name: model-cache-pvc
+          mountPoint: /model

examples/backends/sglang/deploy/agg_router.yaml

@@ -6,6 +6,16 @@ kind: DynamoGraphDeployment
 metadata:
   name: sglang-agg-router
 spec:
+  pvcs:
+    - name: model-cache-pvc
+      create: false
+  envs:
+    - name: MODEL_EXPRESS_CACHE_DIRECTORY
+      value: "/model/.model-express/cache"
+    - name: HF_HUB_CACHE
+      value: "/model/.model-express/cache"
+    - name: MODEL_EXPRESS_URL
+      value: "http://sglang-agg-router-modelexpress:8000"
   services:
     Frontend:
       dynamoNamespace: sglang-agg-router
@@ -17,6 +27,58 @@ spec:
       envs:
         - name: DYN_ROUTER_MODE
           value: kv
+    ModelExpress:
+      envFromSecret: hf-token-secret
+      dynamoNamespace: sglang-agg-router
+      componentType: frontend
+      readinessProbe:
+        tcpSocket:
+          port: 8000
+        initialDelaySeconds: 10
+        periodSeconds: 5
+        timeoutSeconds: 3
+        failureThreshold: 3
+      replicas: 1
+      resources:
+        requests:
+          cpu: "4"
+          memory: "16Gi"
+        limits:
+          cpu: "4"
+          memory: "16Gi"
+      extraPodSpec:
+        mainContainer:
+          image: nvcr.io/nvidia/ai-dynamo/modelexpress-server:my-tag
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: MODEL_EXPRESS_SERVER_PORT
+              value: "8000"
+            - name: MODEL_EXPRESS_LOGGING_LEVEL
+              value: "info"
+            - name: MODEL_EXPRESS_DATABASE_PATH
+              value: "/model/models.db"
+          command:
+            - /bin/sh
+            - -c
+          args:
+            - |
+              echo "Setting up Model Express configuration..."
+
+              mkdir -p $MODEL_EXPRESS_CACHE_DIRECTORY
+              cat > $MODEL_EXPRESS_CACHE_DIRECTORY/config.yaml << EOF
+              local_path: $MODEL_EXPRESS_CACHE_DIRECTORY
+              server_endpoint: http://localhost:8000
+              timeout_secs: null
+              EOF
+
+              ./modelexpress-server &
+
+              SERVER_PID=$!
+              echo "Server started with PID: $SERVER_PID"
+              wait $SERVER_PID
+      volumeMounts:
+        - name: model-cache-pvc
+          mountPoint: /model
     decode:
       envFromSecret: hf-token-secret
       dynamoNamespace: sglang-agg-router
@@ -44,3 +106,6 @@ spec:
             - "1"
             - --trust-remote-code
             - --skip-tokenizer-init
+      volumeMounts:
+        - name: model-cache-pvc
+          mountPoint: /model