From f3b3afcf951a04f07eafafa8861617fd8f4a2722 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 21 Oct 2026 12:20:50 +0200 Subject: [PATCH 1/6] operator/tests: cover multinode LWS and DS upgrades Add production-controller goldens for all three engines' multinode Component and DisaggregatedSet paths. Protect populated graph takeover independently of goldens and require intentional edits in every leader/worker template. Keep released historical inputs and provenance frozen. Refs #16650. Signed-off-by: Dr. Stefan Schimanski --- .../hack/generate_upgrade_fixtures.py | 4 +- .../hack/test_upgrade_component_status.py | 32 + .../controller/clusterenv_helpers_test.go | 2 + .../internal/controller/clusterenv_test.go | 2 +- .../controller/testdata/upgrade/README.md | 34 +- .../aggregated/multi/component/input.yaml | 45 + .../aggregated/multi/component/output.yaml | 875 +++++++++++ .../disaggregated/multi/component/input.yaml | 69 + .../disaggregated/multi/component/output.yaml | 1304 +++++++++++++++ .../multi/disaggregatedset/input.yaml | 70 + .../multi/disaggregatedset/output.yaml | 1116 +++++++++++++ .../aggregated/multi/component/input.yaml | 45 + .../aggregated/multi/component/output.yaml | 922 +++++++++++ .../disaggregated/multi/component/input.yaml | 69 + .../disaggregated/multi/component/output.yaml | 1392 +++++++++++++++++ .../multi/disaggregatedset/input.yaml | 70 + .../multi/disaggregatedset/output.yaml | 1208 ++++++++++++++ .../aggregated/multi/component/input.yaml | 45 + .../aggregated/multi/component/output.yaml | 871 +++++++++++ .../disaggregated/multi/component/input.yaml | 69 + .../disaggregated/multi/component/output.yaml | 1296 +++++++++++++++ .../multi/disaggregatedset/input.yaml | 70 + .../multi/disaggregatedset/output.yaml | 1108 +++++++++++++ .../controller/upgrade_clustertest_test.go | 34 +- .../upgrade_component_clustertest_test.go | 10 +- .../controller/upgrade_component_test.go | 45 +- .../controller/upgrade_contract_test.go | 13 +- ...grade_disaggregatedset_clustertest_test.go | 105 ++ .../controller/upgrade_workload_edit_test.go | 108 ++ .../upgrade_workload_matrix_test.go | 114 ++ .../internal/testing/mocks/lws/lws.go | 21 +- .../internal/testing/mocks/lws/lws_test.go | 25 +- 32 files changed, 11126 insertions(+), 67 deletions(-) create mode 100644 deploy/operator/internal/controller/testdata/upgrade/latest/sglang/aggregated/multi/component/input.yaml create mode 100644 deploy/operator/internal/controller/testdata/upgrade/latest/sglang/aggregated/multi/component/output.yaml create mode 100644 deploy/operator/internal/controller/testdata/upgrade/latest/sglang/disaggregated/multi/component/input.yaml create mode 100644 deploy/operator/internal/controller/testdata/upgrade/latest/sglang/disaggregated/multi/component/output.yaml create mode 100644 deploy/operator/internal/controller/testdata/upgrade/latest/sglang/disaggregated/multi/disaggregatedset/input.yaml create mode 100644 deploy/operator/internal/controller/testdata/upgrade/latest/sglang/disaggregated/multi/disaggregatedset/output.yaml create mode 100644 deploy/operator/internal/controller/testdata/upgrade/latest/trtllm/aggregated/multi/component/input.yaml create mode 100644 deploy/operator/internal/controller/testdata/upgrade/latest/trtllm/aggregated/multi/component/output.yaml create mode 100644 deploy/operator/internal/controller/testdata/upgrade/latest/trtllm/disaggregated/multi/component/input.yaml create mode 100644 deploy/operator/internal/controller/testdata/upgrade/latest/trtllm/disaggregated/multi/component/output.yaml create mode 100644 deploy/operator/internal/controller/testdata/upgrade/latest/trtllm/disaggregated/multi/disaggregatedset/input.yaml create mode 100644 deploy/operator/internal/controller/testdata/upgrade/latest/trtllm/disaggregated/multi/disaggregatedset/output.yaml create mode 100644 deploy/operator/internal/controller/testdata/upgrade/latest/vllm/aggregated/multi/component/input.yaml create mode 100644 deploy/operator/internal/controller/testdata/upgrade/latest/vllm/aggregated/multi/component/output.yaml create mode 100644 deploy/operator/internal/controller/testdata/upgrade/latest/vllm/disaggregated/multi/component/input.yaml create mode 100644 deploy/operator/internal/controller/testdata/upgrade/latest/vllm/disaggregated/multi/component/output.yaml create mode 100644 deploy/operator/internal/controller/testdata/upgrade/latest/vllm/disaggregated/multi/disaggregatedset/input.yaml create mode 100644 deploy/operator/internal/controller/testdata/upgrade/latest/vllm/disaggregated/multi/disaggregatedset/output.yaml create mode 100644 deploy/operator/internal/controller/upgrade_disaggregatedset_clustertest_test.go create mode 100644 deploy/operator/internal/controller/upgrade_workload_edit_test.go create mode 100644 deploy/operator/internal/controller/upgrade_workload_matrix_test.go diff --git a/deploy/operator/hack/generate_upgrade_fixtures.py b/deploy/operator/hack/generate_upgrade_fixtures.py index f7c07b4e7880..6404494bfbd1 100755 --- a/deploy/operator/hack/generate_upgrade_fixtures.py +++ b/deploy/operator/hack/generate_upgrade_fixtures.py @@ -260,7 +260,7 @@ def golden_objects(objects, latest=False): annotations = metadata.get("annotations", {}) if "nvidia.com/last-applied-hash" in annotations: annotations["nvidia.com/last-applied-hash"] = "$ignore" - if obj["kind"] in ("Deployment", "LeaderWorkerSet"): + if obj["kind"] in ("Deployment", "LeaderWorkerSet") and "status" in obj: # Workload-controller availability is outside the operator lowering boundary. obj["status"] = {"$ignore": True} if obj["kind"] in ("DynamoComponentDeployment", "DynamoGraphDeployment"): @@ -328,6 +328,8 @@ def bind_latest_variables(value): for key, child in value.items(): if key == "nvidia.com/dynamo-operator-origin-version": value[key] = "$var:operator-version" + elif key == "disaggregatedset.x-k8s.io/revision": + value[key] = "$var:ds-revision" elif key in ( "nvidia.com/current-worker-hash-v2", "nvidia.com/dynamo-worker-hash", diff --git a/deploy/operator/hack/test_upgrade_component_status.py b/deploy/operator/hack/test_upgrade_component_status.py index 842cc785f571..e29df90df0e3 100644 --- a/deploy/operator/hack/test_upgrade_component_status.py +++ b/deploy/operator/hack/test_upgrade_component_status.py @@ -10,6 +10,38 @@ class UpgradeComponentStatusTest(unittest.TestCase): + def test_absent_workload_status_is_not_manufactured(self): + path = ( + fixtures.FIXTURES / "latest/sglang/aggregated/multi/component/output.yaml" + ) + objects = list(yaml.safe_load_all(path.read_text())) + workload = next(obj for obj in objects if obj["kind"] == "LeaderWorkerSet") + workload.pop("status", None) + + # No LWS controller runs at the immediate-workload boundary. + normalized = fixtures.golden_objects(objects, latest=True) + self.assertNotIn( + "status", + next(obj for obj in normalized if obj["kind"] == "LeaderWorkerSet"), + ) + + # An observed runtime status is ignored without changing the historical capture. + workload["status"] = {"readyReplicas": 0} + normalized = fixtures.golden_objects(objects, latest=True) + self.assertEqual( + next(obj for obj in normalized if obj["kind"] == "LeaderWorkerSet")[ + "status" + ], + {"$ignore": True}, + ) + historical = fixtures.golden_objects(objects) + self.assertEqual( + next(obj for obj in historical if obj["kind"] == "LeaderWorkerSet")[ + "status" + ], + {"readyReplicas": 0}, + ) + def test_runtime_status_does_not_change_component_golden(self): path = ( fixtures.FIXTURES / "latest/sglang/aggregated/single/component/output.yaml" diff --git a/deploy/operator/internal/controller/clusterenv_helpers_test.go b/deploy/operator/internal/controller/clusterenv_helpers_test.go index b50b03f72aad..d4329ffc4f94 100644 --- a/deploy/operator/internal/controller/clusterenv_helpers_test.go +++ b/deploy/operator/internal/controller/clusterenv_helpers_test.go @@ -29,6 +29,7 @@ import ( clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/utils/ptr" gaiev1 "sigs.k8s.io/gateway-api-inference-extension/api/v1" + disaggregatedsetv1 "sigs.k8s.io/lws/api/disaggregatedset/v1" lwsscheme "sigs.k8s.io/lws/client-go/clientset/versioned/scheme" vcbatchv1alpha1 "volcano.sh/apis/pkg/apis/batch/v1alpha1" volcanov1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1" @@ -44,6 +45,7 @@ func newClusterTestScheme() *runtime.Scheme { utilruntime.Must(monitoringv1.AddToScheme(scheme)) utilruntime.Must(istioclientsetscheme.AddToScheme(scheme)) utilruntime.Must(lwsscheme.AddToScheme(scheme)) + utilruntime.Must(disaggregatedsetv1.AddToScheme(scheme)) utilruntime.Must(gaiev1.Install(scheme)) utilruntime.Must(appsv1.AddToScheme(scheme)) utilruntime.Must(admissionregistrationv1.AddToScheme(scheme)) diff --git a/deploy/operator/internal/controller/clusterenv_test.go b/deploy/operator/internal/controller/clusterenv_test.go index dfef448c9e3a..65f9b8a3fde7 100644 --- a/deploy/operator/internal/controller/clusterenv_test.go +++ b/deploy/operator/internal/controller/clusterenv_test.go @@ -58,7 +58,7 @@ func runClusterTests(m *testing.M, prepare func(context.Context, client.Client) operatorConfig := &configv1alpha1.OperatorConfiguration{} configv1alpha1.SetDefaultsOperatorConfiguration(operatorConfig) operatorConfig.GPU.DiscoveryEnabled = ptr.To(false) - runtimeConfig := &commoncontroller.RuntimeConfig{Gate: features.Gates{Grove: true, LWS: true}} + runtimeConfig := &commoncontroller.RuntimeConfig{Gate: features.Gates{Grove: true, LWS: true, DisaggregatedSet: true}} additionalAdmission := lwsmock.Configurations() groveAdmission := grovemock.Configurations() additionalAdmission.Mutating = append(additionalAdmission.Mutating, groveAdmission.Mutating...) diff --git a/deploy/operator/internal/controller/testdata/upgrade/README.md b/deploy/operator/internal/controller/testdata/upgrade/README.md index b2df46defba0..44a7189312a8 100644 --- a/deploy/operator/internal/controller/testdata/upgrade/README.md +++ b/deploy/operator/internal/controller/testdata/upgrade/README.md @@ -33,6 +33,28 @@ PodSpec changes remain protected. The independent detector test rejects a new generation even if the output is identical. An intentional worker env edit must still reach a new workload generation after takeover. +## Multinode Component and DisaggregatedSet coverage + +The canonical suite includes aggregated and disaggregated multinode Component +scenarios for vLLM, SGLang, and TensorRT-LLM, plus one disaggregated multinode +DisaggregatedSet scenario per engine. Component scenarios stop at DCD-owned +LeaderWorkerSets; DisaggregatedSet scenarios stop at the DisaggregatedSet itself +and the frontend's auxiliary DCD/Deployment. The LWS/DS controllers and serving +Pods do not run. Dependency CRDs and production admission handlers come from the +pinned LWS module already used by the shared clustertest setup. + +Independent checks enforce the selected provider, component/role inventory and +multinode shape. Historical safety also protects the complete DisaggregatedSet +spec, identity and generation. Intentional worker edits must reach every edited +component and both leader/worker templates, including embedded DS role templates. +The populated graph replay controls cover Deployment, LWS and DS graphs without +claiming that these freshly captured test graphs came from an older release. +Formal released history still advances only through the pinned-release workflow. + +These cases complete 22 of the 27 canonical matrix entries. The five remaining +single-node Component cases and the missing-origin historical case remain in +#16650; the incomplete release-matrix gate stays enforced. + ## Storage and production lifecycle Run this suite only against a dedicated cluster with no installed controllers or @@ -142,11 +164,11 @@ git -C "$release_tree" restore --source="$harness" -- \ deploy/operator/internal/controller/clusterenv_helpers_test.go \ deploy/operator/internal/controller/clusterenv_test.go \ deploy/operator/internal/controller/clusterenv_main_test.go \ - deploy/operator/internal/controller/upgrade_clustertest_test.go \ - deploy/operator/internal/controller/upgrade_contract_test.go \ + ':(glob)deploy/operator/internal/controller/upgrade_*_test.go' \ deploy/operator/internal/controller/testdata/upgrade \ deploy/operator/internal/testing/clusterenv \ - deploy/operator/internal/testing/golden + deploy/operator/internal/testing/golden \ + deploy/operator/internal/testing/mocks make -C "$release_tree/deploy/operator" clustertest-upgrade-fixtures-regen \ UPGRADE_RELEASE_TAG=v1.6.0 ``` @@ -209,9 +231,9 @@ across releases; namespace collisions across all roles are rejected. Long namespace names include a digest to remain within Kubernetes' length limit. Latest origin annotations use `$var:operator-version`, bound exactly to the -operator chart's `appVersion` by the golden matcher. Worker hashes and the PCS -spec digest also depend on origin: their exact bindings are computed from the -reviewed golden DGD/PCS specifications with that version, never from candidate +operator chart's `appVersion` by the golden matcher. Worker hashes, the PCS +spec digest and the DS role revision also depend on origin: their exact bindings +are computed from the reviewed golden DGD/PCS/DS specifications with that version, never from candidate output. The independent CREATE origin assertion remains active. All other lowering fields remain exact, and historical origins and hashes remain concrete. Accepting a golden never bypasses rollout safety assertions. diff --git a/deploy/operator/internal/controller/testdata/upgrade/latest/sglang/aggregated/multi/component/input.yaml b/deploy/operator/internal/controller/testdata/upgrade/latest/sglang/aggregated/multi/component/input.yaml new file mode 100644 index 000000000000..54030b07bd66 --- /dev/null +++ b/deploy/operator/internal/controller/testdata/upgrade/latest/sglang/aggregated/multi/component/input.yaml @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + name: workers + namespace: upgrade-latest-sglang-aggregated-multi-component + annotations: + nvidia.com/enable-grove: 'false' + nvidia.com/dynamo-operator-origin-version: 0.0.0 +spec: + backendFramework: sglang + components: + - name: frontend + type: frontend + replicas: 1 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + command: + - python3 + - -m + - dynamo.frontend + - name: worker + type: worker + replicas: 2 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + command: + - python3 + - -m + - dynamo.sglang + args: + - --model-path + - Qwen/Qwen3-0.6B + resources: + limits: + nvidia.com/gpu: '1' + multinode: + nodeCount: 2 diff --git a/deploy/operator/internal/controller/testdata/upgrade/latest/sglang/aggregated/multi/component/output.yaml b/deploy/operator/internal/controller/testdata/upgrade/latest/sglang/aggregated/multi/component/output.yaml new file mode 100644 index 000000000000..163fe18125ea --- /dev/null +++ b/deploy/operator/internal/controller/testdata/upgrade/latest/sglang/aggregated/multi/component/output.yaml @@ -0,0 +1,875 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + annotations: + nvidia.com/current-worker-hash-v2: $var:worker-hash + nvidia.com/dynamo-operator-origin-version: $var:operator-version + nvidia.com/enable-grove: 'false' + nvidia.com/workload-provider: component + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + name: workers + namespace: upgrade-latest-sglang-aggregated-multi-component + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: sglang + components: + - name: frontend + podTemplate: + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + name: main + replicas: 1 + type: frontend + - multinode: + nodeCount: 2 + name: worker + podTemplate: + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + command: + - python3 + - -m + - dynamo.sglang + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + name: main + resources: + limits: + nvidia.com/gpu: '1' + replicas: 2 + type: worker +status: + components: + frontend: + $strict: false + componentKind: Deployment + componentNames: + - workers-frontend + runtimeNamespace: upgrade-latest-sglang-aggregated-multi-component-workers + worker: + $strict: false + componentKind: LeaderWorkerSet + componentNames: + - $var:worker-string:workers-worker-{{worker-hash}}-0 + runtimeNamespace: $var:worker-string:upgrade-latest-sglang-aggregated-multi-component-workers-{{worker-hash}} + servedModelName: Qwen/Qwen3-0.6B + conditions: + - lastTransitionTime: $ignore + message: '$var:worker-string:Resources not ready: workers-frontend: Component + deployment not ready - Available condition not true; workers-worker-{{worker-hash}}: + Component deployment not ready - Available condition not true' + observedGeneration: 1 + reason: some_resources_are_not_ready + status: 'False' + type: Ready + observedGeneration: 1 + state: pending +--- +apiVersion: v1 +data: + wait-for-leader.py: "import socket, time, json, ssl, urllib.request, os\n\nSA =\ + \ \"/var/run/secrets/kubernetes.io/serviceaccount\"\nhost = os.environ[\"LEADER_HOST\"\ + ]\nport = int(os.environ[\"LEADER_PORT\"])\n\ndef _k8s_ctx():\n return ssl.create_default_context(cafile=f\"\ + {SA}/ca.crt\")\n\ndef _k8s_headers():\n token = open(f\"{SA}/token\").read()\n\ + \ return {\"Authorization\": f\"Bearer {token}\"}\n\ndef _k8s_api():\n ns\ + \ = open(f\"{SA}/namespace\").read()\n return f\"https://kubernetes.default.svc/api/v1/namespaces/{ns}/pods\"\ + \n\ndef resolve_leader_ip():\n # gethostbyname is IPv4-only; fall back to IPv6\ + \ for IPv6-only clusters.\n try:\n return socket.gethostbyname(host)\n\ + \ except socket.gaierror:\n return socket.getaddrinfo(host, None, socket.AF_INET6,\ + \ socket.SOCK_STREAM)[0][4][0]\n\ndef leader_pod_is_healthy():\n try:\n \ + \ ip = resolve_leader_ip()\n except socket.gaierror:\n return False,\ + \ \"DNS resolution failed\", None, None\n try:\n req = urllib.request.Request(\n\ + \ f\"{_k8s_api()}?fieldSelector=status.podIP={ip}\",\n headers=_k8s_headers(),\n\ + \ )\n resp = json.loads(urllib.request.urlopen(req, context=_k8s_ctx(),\ + \ timeout=5).read())\n pods = resp.get(\"items\", [])\n if not pods:\n\ + \ return False, f\"no pod found with IP {ip}\", None, ip\n pod\ + \ = pods[0]\n name = pod[\"metadata\"].get(\"name\", \"unknown\")\n \ + \ uid = pod[\"metadata\"].get(\"uid\", \"unknown\")\n phase = pod.get(\"\ + status\", {}).get(\"phase\")\n deletion_ts = pod[\"metadata\"].get(\"deletionTimestamp\"\ + )\n info = f\"ip={ip} pod={name} uid={uid} phase={phase} deletionTimestamp={deletion_ts}\"\ + \n if deletion_ts:\n return False, f\"pod {name} is terminating\"\ + , info, ip\n if phase != \"Running\":\n return False, f\"pod\ + \ {name} phase is {phase}\", info, ip\n return True, \"\", info, ip\n \ + \ except Exception as e:\n # Fall back to TCP-only when the API is unavailable\ + \ (e.g. 403 no RBAC)\n return True, f\"K8s API unavailable ({e}), falling\ + \ back to TCP\", f\"ip={ip}\", ip\n\nprint(f\"Waiting for leader master port at\ + \ {host}:{port}...\", flush=True)\ntime.sleep(5)\nstart = time.monotonic()\nlast_status\ + \ = start\nlast_err = \"\"\nwhile True:\n healthy, reason, pod_info, leader_ip\ + \ = leader_pod_is_healthy()\n if healthy:\n try:\n s = socket.create_connection((leader_ip,\ + \ port), timeout=2)\n s.close()\n elapsed = time.monotonic()\ + \ - start\n print(f\"Leader master port ready (waited {elapsed:.1f}s)\ + \ [{pod_info}]\", flush=True)\n break\n except Exception as\ + \ e:\n last_err = f\"tcp: {type(e).__name__}: {e} [{pod_info}]\"\n\ + \ else:\n last_err = f\"{reason} [{pod_info}]\" if pod_info else reason\n\ + \ now = time.monotonic()\n if now - last_status >= 30:\n print(f\"\ + Still waiting for {host}:{port}... ({now - start:.0f}s elapsed, last: {last_err})\"\ + , flush=True)\n last_status = now\n time.sleep(5)\n" +kind: ConfigMap +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: a615bfc016f57bef619ea5c67d88da24755349bb07efff9da0b2c79f18d6d076 + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-graph-deployment-name: workers + name: workers-wait-leader-script + namespace: upgrade-latest-sglang-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-aggregated-multi-component-workers + name: workers-frontend + namespace: upgrade-latest-sglang-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: workers-frontend + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + clusterIP: $ignore + clusterIPs: + - $ignore + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: http + port: 8000 + protocol: TCP + targetPort: http + selector: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-component-type: frontend + nvidia.com/dynamo-namespace: upgrade-latest-sglang-aggregated-multi-component-workers + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-component: worker + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-aggregated-multi-component-workers + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-worker-{{worker-hash}} + namespace: upgrade-latest-sglang-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: $var:worker-string:workers-worker-{{worker-hash}} + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + clusterIP: $ignore + clusterIPs: + - $ignore + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: system + port: 9090 + protocol: TCP + targetPort: system + selector: + nvidia.com/dynamo-component: worker + nvidia.com/dynamo-component-type: worker + nvidia.com/dynamo-namespace: upgrade-latest-sglang-aggregated-multi-component-workers + role: leader + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery + namespace: upgrade-latest-sglang-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + generation: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-aggregated-multi-component-workers + name: workers-frontend + namespace: upgrade-latest-sglang-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: workers-frontend + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + nvidia.com/selector: workers-frontend + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-component-type: frontend + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-aggregated-multi-component-workers + nvidia.com/metrics-enabled: 'true' + nvidia.com/selector: workers-frontend + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + env: + - name: DYNAMO_PORT + value: '8000' + - name: DYN_COMPONENT + value: frontend + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_HTTP_PORT + value: '8000' + - name: DYN_NAMESPACE + value: upgrade-latest-sglang-aggregated-multi-component-workers + - name: DYN_NAMESPACE_PREFIX + value: upgrade-latest-sglang-aggregated-multi-component-workers + - name: DYN_NAMESPACE_PREFIX_STRICT + value: 'true' + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-sglang-aggregated-multi-component + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.uid + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /live + port: http + scheme: HTTP + initialDelaySeconds: 15 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: main + ports: + - containerPort: 8000 + name: http + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: http + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 3 + resources: {} + startupProbe: + failureThreshold: 30 + httpGet: + path: /live + port: http + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 1000 + serviceAccount: workers-k8s-service-discovery + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory +status: + $ignore: true +--- +apiVersion: leaderworkerset.x-k8s.io/v1 +kind: LeaderWorkerSet +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + generation: $ignore + labels: + nvidia.com/dynamo-component: worker + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-aggregated-multi-component-workers + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-worker-{{worker-hash}}-0 + namespace: upgrade-latest-sglang-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: $var:worker-string:workers-worker-{{worker-hash}} + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + leaderWorkerTemplate: + leaderTemplate: + metadata: + labels: + nvidia.com/dynamo-component: worker + nvidia.com/dynamo-component-type: worker + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-aggregated-multi-component-workers + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: leader + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --dist-init-addr + - $(DYNAMO_LEADER_ADDRESS):29500 + - --nnodes + - '2' + - --node-rank + - $(DYNAMO_RANK) + command: + - python3 + - -m + - dynamo.sglang + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: worker + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-sglang-aggregated-multi-component-workers + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-sglang-aggregated-multi-component + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + livenessProbe: + failureThreshold: 3 + httpGet: + path: /live + port: system + periodSeconds: 5 + timeoutSeconds: 4 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: system + periodSeconds: 10 + timeoutSeconds: 4 + resources: + limits: + nvidia.com/gpu: '1' + startupProbe: + failureThreshold: 720 + httpGet: + path: /live + port: system + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + restartPolicy: RecreateGroupOnPodRestart + size: 2 + workerTemplate: + metadata: + labels: + nvidia.com/dynamo-component: worker + nvidia.com/dynamo-component-type: worker + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-aggregated-multi-component-workers + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: worker + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --dist-init-addr + - $(DYNAMO_LEADER_ADDRESS):29500 + - --nnodes + - '2' + - --node-rank + - $(DYNAMO_RANK) + command: + - python3 + - -m + - dynamo.sglang + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: worker + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-sglang-aggregated-multi-component-workers + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-sglang-aggregated-multi-component + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + resources: + limits: + nvidia.com/gpu: '1' + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + networkConfig: + subdomainPolicy: Shared + replicas: 2 + rolloutStrategy: + rollingUpdateConfiguration: + maxSurge: 0 + maxUnavailable: 1 + partition: 0 + type: RollingUpdate + startupPolicy: LeaderCreated +--- +apiVersion: nvidia.com/v1beta1 +kind: DynamoComponentDeployment +metadata: + annotations: + nvidia.com/dcd-spec: '{"version":1,"spec":{},"rawJSON":[{"path":"serviceName","nil":true}]}' + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-aggregated-multi-component-workers + name: workers-frontend + namespace: upgrade-latest-sglang-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: sglang + name: frontend + podTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + name: main + replicas: 1 + type: frontend +status: + component: + $strict: false + componentKind: Deployment + componentNames: + - workers-frontend + runtimeNamespace: upgrade-latest-sglang-aggregated-multi-component-workers + conditions: + - lastTransitionTime: $ignore + message: $ignore + reason: $ignore + status: 'False' + type: Available + - lastTransitionTime: $ignore + message: DynamoComponent is not ready + reason: ComponentNotReady + status: 'False' + type: DynamoComponentReady + observedGeneration: 1 +--- +apiVersion: nvidia.com/v1beta1 +kind: DynamoComponentDeployment +metadata: + annotations: + nvidia.com/dcd-spec: '{"version":1,"spec":{"resources":{"limits":{"gpu":"1"}},"extraPodSpec":{"mainContainer":{"name":"","resources":{}}}},"rawJSON":[{"path":"serviceName","nil":true}]}' + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + labels: + nvidia.com/dynamo-component: worker + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-aggregated-multi-component-workers + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-worker-{{worker-hash}} + namespace: upgrade-latest-sglang-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: sglang + multinode: + nodeCount: 2 + name: worker + podTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + labels: + nvidia.com/dynamo-worker-hash: $var:worker-hash + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + command: + - python3 + - -m + - dynamo.sglang + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + name: main + resources: + limits: + nvidia.com/gpu: '1' + replicas: 2 + type: worker +status: + component: + $strict: false + componentKind: LeaderWorkerSet + componentNames: + - $var:worker-string:workers-worker-{{worker-hash}}-0 + runtimeNamespace: $var:worker-string:upgrade-latest-sglang-aggregated-multi-component-workers-{{worker-hash}} + servedModelName: Qwen/Qwen3-0.6B + conditions: + - lastTransitionTime: $ignore + message: $ignore + reason: $ignore + status: 'False' + type: Available + - lastTransitionTime: $ignore + message: DynamoComponent is not ready + reason: ComponentNotReady + status: 'False' + type: DynamoComponentReady + observedGeneration: 1 +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: 0d454cc8266ac38ed6453ee2ef3cf6d9dd237faebc63af22f67ca911e106796c + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery-role + namespace: upgrade-latest-sglang-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +rules: +- apiGroups: + - '' + resources: + - endpoints + - pods + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - nvidia.com + resources: + - dynamoworkermetadatas + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: a19fb46ad64f8dafe34a249d32d860a8f52f8fdfa049a428d6a4cb33254272d8 + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery-binding + namespace: upgrade-latest-sglang-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: workers-k8s-service-discovery-role +subjects: +- kind: ServiceAccount + name: workers-k8s-service-discovery + namespace: upgrade-latest-sglang-aggregated-multi-component diff --git a/deploy/operator/internal/controller/testdata/upgrade/latest/sglang/disaggregated/multi/component/input.yaml b/deploy/operator/internal/controller/testdata/upgrade/latest/sglang/disaggregated/multi/component/input.yaml new file mode 100644 index 000000000000..78132e36c437 --- /dev/null +++ b/deploy/operator/internal/controller/testdata/upgrade/latest/sglang/disaggregated/multi/component/input.yaml @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + name: workers + namespace: upgrade-latest-sglang-disaggregated-multi-component + annotations: + nvidia.com/enable-grove: 'false' + nvidia.com/dynamo-operator-origin-version: 0.0.0 +spec: + backendFramework: sglang + components: + - name: frontend + type: frontend + replicas: 1 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + command: + - python3 + - -m + - dynamo.frontend + - name: prefill + type: prefill + replicas: 2 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + command: + - python3 + - -m + - dynamo.sglang + args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - prefill + resources: + limits: + nvidia.com/gpu: '1' + multinode: + nodeCount: 2 + - name: decode + type: decode + replicas: 2 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + command: + - python3 + - -m + - dynamo.sglang + args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - decode + resources: + limits: + nvidia.com/gpu: '1' + multinode: + nodeCount: 2 diff --git a/deploy/operator/internal/controller/testdata/upgrade/latest/sglang/disaggregated/multi/component/output.yaml b/deploy/operator/internal/controller/testdata/upgrade/latest/sglang/disaggregated/multi/component/output.yaml new file mode 100644 index 000000000000..d13063daf09e --- /dev/null +++ b/deploy/operator/internal/controller/testdata/upgrade/latest/sglang/disaggregated/multi/component/output.yaml @@ -0,0 +1,1304 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + annotations: + nvidia.com/current-worker-hash-v2: $var:worker-hash + nvidia.com/dynamo-operator-origin-version: $var:operator-version + nvidia.com/enable-grove: 'false' + nvidia.com/workload-provider: component + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + name: workers + namespace: upgrade-latest-sglang-disaggregated-multi-component + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: sglang + components: + - name: frontend + podTemplate: + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + name: main + replicas: 1 + type: frontend + - multinode: + nodeCount: 2 + name: prefill + podTemplate: + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - prefill + command: + - python3 + - -m + - dynamo.sglang + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + name: main + resources: + limits: + nvidia.com/gpu: '1' + replicas: 2 + type: prefill + - multinode: + nodeCount: 2 + name: decode + podTemplate: + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - decode + command: + - python3 + - -m + - dynamo.sglang + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + name: main + resources: + limits: + nvidia.com/gpu: '1' + replicas: 2 + type: decode +status: + components: + decode: + $strict: false + componentKind: LeaderWorkerSet + componentNames: + - $var:worker-string:workers-decode-{{worker-hash}}-0 + runtimeNamespace: $var:worker-string:upgrade-latest-sglang-disaggregated-multi-component-workers-{{worker-hash}} + servedModelName: Qwen/Qwen3-0.6B + frontend: + $strict: false + componentKind: Deployment + componentNames: + - workers-frontend + runtimeNamespace: upgrade-latest-sglang-disaggregated-multi-component-workers + prefill: + $strict: false + componentKind: LeaderWorkerSet + componentNames: + - $var:worker-string:workers-prefill-{{worker-hash}}-0 + runtimeNamespace: $var:worker-string:upgrade-latest-sglang-disaggregated-multi-component-workers-{{worker-hash}} + servedModelName: Qwen/Qwen3-0.6B + conditions: + - lastTransitionTime: $ignore + message: '$var:worker-string:Resources not ready: workers-decode-{{worker-hash}}: + Component deployment not ready - Available condition not true; workers-frontend: + Component deployment not ready - Available condition not true; workers-prefill-{{worker-hash}}: + Component deployment not ready - Available condition not true' + observedGeneration: 1 + reason: some_resources_are_not_ready + status: 'False' + type: Ready + observedGeneration: 1 + state: pending +--- +apiVersion: v1 +data: + wait-for-leader.py: "import socket, time, json, ssl, urllib.request, os\n\nSA =\ + \ \"/var/run/secrets/kubernetes.io/serviceaccount\"\nhost = os.environ[\"LEADER_HOST\"\ + ]\nport = int(os.environ[\"LEADER_PORT\"])\n\ndef _k8s_ctx():\n return ssl.create_default_context(cafile=f\"\ + {SA}/ca.crt\")\n\ndef _k8s_headers():\n token = open(f\"{SA}/token\").read()\n\ + \ return {\"Authorization\": f\"Bearer {token}\"}\n\ndef _k8s_api():\n ns\ + \ = open(f\"{SA}/namespace\").read()\n return f\"https://kubernetes.default.svc/api/v1/namespaces/{ns}/pods\"\ + \n\ndef resolve_leader_ip():\n # gethostbyname is IPv4-only; fall back to IPv6\ + \ for IPv6-only clusters.\n try:\n return socket.gethostbyname(host)\n\ + \ except socket.gaierror:\n return socket.getaddrinfo(host, None, socket.AF_INET6,\ + \ socket.SOCK_STREAM)[0][4][0]\n\ndef leader_pod_is_healthy():\n try:\n \ + \ ip = resolve_leader_ip()\n except socket.gaierror:\n return False,\ + \ \"DNS resolution failed\", None, None\n try:\n req = urllib.request.Request(\n\ + \ f\"{_k8s_api()}?fieldSelector=status.podIP={ip}\",\n headers=_k8s_headers(),\n\ + \ )\n resp = json.loads(urllib.request.urlopen(req, context=_k8s_ctx(),\ + \ timeout=5).read())\n pods = resp.get(\"items\", [])\n if not pods:\n\ + \ return False, f\"no pod found with IP {ip}\", None, ip\n pod\ + \ = pods[0]\n name = pod[\"metadata\"].get(\"name\", \"unknown\")\n \ + \ uid = pod[\"metadata\"].get(\"uid\", \"unknown\")\n phase = pod.get(\"\ + status\", {}).get(\"phase\")\n deletion_ts = pod[\"metadata\"].get(\"deletionTimestamp\"\ + )\n info = f\"ip={ip} pod={name} uid={uid} phase={phase} deletionTimestamp={deletion_ts}\"\ + \n if deletion_ts:\n return False, f\"pod {name} is terminating\"\ + , info, ip\n if phase != \"Running\":\n return False, f\"pod\ + \ {name} phase is {phase}\", info, ip\n return True, \"\", info, ip\n \ + \ except Exception as e:\n # Fall back to TCP-only when the API is unavailable\ + \ (e.g. 403 no RBAC)\n return True, f\"K8s API unavailable ({e}), falling\ + \ back to TCP\", f\"ip={ip}\", ip\n\nprint(f\"Waiting for leader master port at\ + \ {host}:{port}...\", flush=True)\ntime.sleep(5)\nstart = time.monotonic()\nlast_status\ + \ = start\nlast_err = \"\"\nwhile True:\n healthy, reason, pod_info, leader_ip\ + \ = leader_pod_is_healthy()\n if healthy:\n try:\n s = socket.create_connection((leader_ip,\ + \ port), timeout=2)\n s.close()\n elapsed = time.monotonic()\ + \ - start\n print(f\"Leader master port ready (waited {elapsed:.1f}s)\ + \ [{pod_info}]\", flush=True)\n break\n except Exception as\ + \ e:\n last_err = f\"tcp: {type(e).__name__}: {e} [{pod_info}]\"\n\ + \ else:\n last_err = f\"{reason} [{pod_info}]\" if pod_info else reason\n\ + \ now = time.monotonic()\n if now - last_status >= 30:\n print(f\"\ + Still waiting for {host}:{port}... ({now - start:.0f}s elapsed, last: {last_err})\"\ + , flush=True)\n last_status = now\n time.sleep(5)\n" +kind: ConfigMap +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: a615bfc016f57bef619ea5c67d88da24755349bb07efff9da0b2c79f18d6d076 + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-graph-deployment-name: workers + name: workers-wait-leader-script + namespace: upgrade-latest-sglang-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-multi-component-workers + nvidia.com/dynamo-sub-component-type: decode + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-decode-{{worker-hash}} + namespace: upgrade-latest-sglang-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: $var:worker-string:workers-decode-{{worker-hash}} + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + clusterIP: $ignore + clusterIPs: + - $ignore + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: system + port: 9090 + protocol: TCP + targetPort: system + selector: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-component-type: decode + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-multi-component-workers + role: leader + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-multi-component-workers + name: workers-frontend + namespace: upgrade-latest-sglang-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: workers-frontend + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + clusterIP: $ignore + clusterIPs: + - $ignore + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: http + port: 8000 + protocol: TCP + targetPort: http + selector: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-component-type: frontend + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-multi-component-workers + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-multi-component-workers + nvidia.com/dynamo-sub-component-type: prefill + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-prefill-{{worker-hash}} + namespace: upgrade-latest-sglang-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: $var:worker-string:workers-prefill-{{worker-hash}} + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + clusterIP: $ignore + clusterIPs: + - $ignore + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: system + port: 9090 + protocol: TCP + targetPort: system + selector: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-component-type: prefill + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-multi-component-workers + role: leader + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery + namespace: upgrade-latest-sglang-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + generation: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-multi-component-workers + name: workers-frontend + namespace: upgrade-latest-sglang-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: workers-frontend + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + nvidia.com/selector: workers-frontend + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-component-type: frontend + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-multi-component-workers + nvidia.com/metrics-enabled: 'true' + nvidia.com/selector: workers-frontend + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + env: + - name: DYNAMO_PORT + value: '8000' + - name: DYN_COMPONENT + value: frontend + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_HTTP_PORT + value: '8000' + - name: DYN_NAMESPACE + value: upgrade-latest-sglang-disaggregated-multi-component-workers + - name: DYN_NAMESPACE_PREFIX + value: upgrade-latest-sglang-disaggregated-multi-component-workers + - name: DYN_NAMESPACE_PREFIX_STRICT + value: 'true' + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-sglang-disaggregated-multi-component + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.uid + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /live + port: http + scheme: HTTP + initialDelaySeconds: 15 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: main + ports: + - containerPort: 8000 + name: http + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: http + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 3 + resources: {} + startupProbe: + failureThreshold: 30 + httpGet: + path: /live + port: http + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 1000 + serviceAccount: workers-k8s-service-discovery + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory +status: + $ignore: true +--- +apiVersion: leaderworkerset.x-k8s.io/v1 +kind: LeaderWorkerSet +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + generation: $ignore + labels: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-multi-component-workers + nvidia.com/dynamo-sub-component-type: decode + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-decode-{{worker-hash}}-0 + namespace: upgrade-latest-sglang-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: $var:worker-string:workers-decode-{{worker-hash}} + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + leaderWorkerTemplate: + leaderTemplate: + metadata: + labels: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-component-type: decode + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-multi-component-workers + nvidia.com/dynamo-sub-component-type: decode + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: leader + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - decode + - --dist-init-addr + - $(DYNAMO_LEADER_ADDRESS):29500 + - --nnodes + - '2' + - --node-rank + - $(DYNAMO_RANK) + command: + - python3 + - -m + - dynamo.sglang + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: decode + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-sglang-disaggregated-multi-component-workers + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-sglang-disaggregated-multi-component + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + livenessProbe: + failureThreshold: 3 + httpGet: + path: /live + port: system + periodSeconds: 5 + timeoutSeconds: 4 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: system + periodSeconds: 10 + timeoutSeconds: 4 + resources: + limits: + nvidia.com/gpu: '1' + startupProbe: + failureThreshold: 720 + httpGet: + path: /live + port: system + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + restartPolicy: RecreateGroupOnPodRestart + size: 2 + workerTemplate: + metadata: + labels: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-component-type: decode + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-multi-component-workers + nvidia.com/dynamo-sub-component-type: decode + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: worker + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - decode + - --dist-init-addr + - $(DYNAMO_LEADER_ADDRESS):29500 + - --nnodes + - '2' + - --node-rank + - $(DYNAMO_RANK) + command: + - python3 + - -m + - dynamo.sglang + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: decode + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-sglang-disaggregated-multi-component-workers + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-sglang-disaggregated-multi-component + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + resources: + limits: + nvidia.com/gpu: '1' + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + networkConfig: + subdomainPolicy: Shared + replicas: 2 + rolloutStrategy: + rollingUpdateConfiguration: + maxSurge: 0 + maxUnavailable: 1 + partition: 0 + type: RollingUpdate + startupPolicy: LeaderCreated +--- +apiVersion: leaderworkerset.x-k8s.io/v1 +kind: LeaderWorkerSet +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + generation: $ignore + labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-multi-component-workers + nvidia.com/dynamo-sub-component-type: prefill + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-prefill-{{worker-hash}}-0 + namespace: upgrade-latest-sglang-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: $var:worker-string:workers-prefill-{{worker-hash}} + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + leaderWorkerTemplate: + leaderTemplate: + metadata: + labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-component-type: prefill + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-multi-component-workers + nvidia.com/dynamo-sub-component-type: prefill + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: leader + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - prefill + - --dist-init-addr + - $(DYNAMO_LEADER_ADDRESS):29500 + - --nnodes + - '2' + - --node-rank + - $(DYNAMO_RANK) + command: + - python3 + - -m + - dynamo.sglang + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: prefill + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-sglang-disaggregated-multi-component-workers + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-sglang-disaggregated-multi-component + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + livenessProbe: + failureThreshold: 3 + httpGet: + path: /live + port: system + periodSeconds: 5 + timeoutSeconds: 4 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: system + periodSeconds: 10 + timeoutSeconds: 4 + resources: + limits: + nvidia.com/gpu: '1' + startupProbe: + failureThreshold: 720 + httpGet: + path: /live + port: system + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + restartPolicy: RecreateGroupOnPodRestart + size: 2 + workerTemplate: + metadata: + labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-component-type: prefill + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-multi-component-workers + nvidia.com/dynamo-sub-component-type: prefill + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: worker + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - prefill + - --dist-init-addr + - $(DYNAMO_LEADER_ADDRESS):29500 + - --nnodes + - '2' + - --node-rank + - $(DYNAMO_RANK) + command: + - python3 + - -m + - dynamo.sglang + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: prefill + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-sglang-disaggregated-multi-component-workers + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-sglang-disaggregated-multi-component + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + resources: + limits: + nvidia.com/gpu: '1' + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + networkConfig: + subdomainPolicy: Shared + replicas: 2 + rolloutStrategy: + rollingUpdateConfiguration: + maxSurge: 0 + maxUnavailable: 1 + partition: 0 + type: RollingUpdate + startupPolicy: LeaderCreated +--- +apiVersion: nvidia.com/v1beta1 +kind: DynamoComponentDeployment +metadata: + annotations: + nvidia.com/dcd-spec: '{"version":1,"spec":{"resources":{"limits":{"gpu":"1"}},"extraPodSpec":{"mainContainer":{"name":"","resources":{}}}},"rawJSON":[{"path":"serviceName","nil":true}]}' + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + labels: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-multi-component-workers + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-decode-{{worker-hash}} + namespace: upgrade-latest-sglang-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: sglang + multinode: + nodeCount: 2 + name: decode + podTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + labels: + nvidia.com/dynamo-worker-hash: $var:worker-hash + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - decode + command: + - python3 + - -m + - dynamo.sglang + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + name: main + resources: + limits: + nvidia.com/gpu: '1' + replicas: 2 + type: decode +status: + component: + $strict: false + componentKind: LeaderWorkerSet + componentNames: + - $var:worker-string:workers-decode-{{worker-hash}}-0 + runtimeNamespace: $var:worker-string:upgrade-latest-sglang-disaggregated-multi-component-workers-{{worker-hash}} + servedModelName: Qwen/Qwen3-0.6B + conditions: + - lastTransitionTime: $ignore + message: $ignore + reason: $ignore + status: 'False' + type: Available + - lastTransitionTime: $ignore + message: DynamoComponent is not ready + reason: ComponentNotReady + status: 'False' + type: DynamoComponentReady + observedGeneration: 1 +--- +apiVersion: nvidia.com/v1beta1 +kind: DynamoComponentDeployment +metadata: + annotations: + nvidia.com/dcd-spec: '{"version":1,"spec":{},"rawJSON":[{"path":"serviceName","nil":true}]}' + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-multi-component-workers + name: workers-frontend + namespace: upgrade-latest-sglang-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: sglang + name: frontend + podTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + name: main + replicas: 1 + type: frontend +status: + component: + $strict: false + componentKind: Deployment + componentNames: + - workers-frontend + runtimeNamespace: upgrade-latest-sglang-disaggregated-multi-component-workers + conditions: + - lastTransitionTime: $ignore + message: $ignore + reason: $ignore + status: 'False' + type: Available + - lastTransitionTime: $ignore + message: DynamoComponent is not ready + reason: ComponentNotReady + status: 'False' + type: DynamoComponentReady + observedGeneration: 1 +--- +apiVersion: nvidia.com/v1beta1 +kind: DynamoComponentDeployment +metadata: + annotations: + nvidia.com/dcd-spec: '{"version":1,"spec":{"resources":{"limits":{"gpu":"1"}},"extraPodSpec":{"mainContainer":{"name":"","resources":{}}}},"rawJSON":[{"path":"serviceName","nil":true}]}' + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-multi-component-workers + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-prefill-{{worker-hash}} + namespace: upgrade-latest-sglang-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: sglang + multinode: + nodeCount: 2 + name: prefill + podTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + labels: + nvidia.com/dynamo-worker-hash: $var:worker-hash + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - prefill + command: + - python3 + - -m + - dynamo.sglang + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + name: main + resources: + limits: + nvidia.com/gpu: '1' + replicas: 2 + type: prefill +status: + component: + $strict: false + componentKind: LeaderWorkerSet + componentNames: + - $var:worker-string:workers-prefill-{{worker-hash}}-0 + runtimeNamespace: $var:worker-string:upgrade-latest-sglang-disaggregated-multi-component-workers-{{worker-hash}} + servedModelName: Qwen/Qwen3-0.6B + conditions: + - lastTransitionTime: $ignore + message: $ignore + reason: $ignore + status: 'False' + type: Available + - lastTransitionTime: $ignore + message: DynamoComponent is not ready + reason: ComponentNotReady + status: 'False' + type: DynamoComponentReady + observedGeneration: 1 +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: 0d454cc8266ac38ed6453ee2ef3cf6d9dd237faebc63af22f67ca911e106796c + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery-role + namespace: upgrade-latest-sglang-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +rules: +- apiGroups: + - '' + resources: + - endpoints + - pods + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - nvidia.com + resources: + - dynamoworkermetadatas + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: 9ef49a7d2d60d01b446ca0a469bef01c1112cc343ac94a0d2ea632d21230a79a + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery-binding + namespace: upgrade-latest-sglang-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: workers-k8s-service-discovery-role +subjects: +- kind: ServiceAccount + name: workers-k8s-service-discovery + namespace: upgrade-latest-sglang-disaggregated-multi-component diff --git a/deploy/operator/internal/controller/testdata/upgrade/latest/sglang/disaggregated/multi/disaggregatedset/input.yaml b/deploy/operator/internal/controller/testdata/upgrade/latest/sglang/disaggregated/multi/disaggregatedset/input.yaml new file mode 100644 index 000000000000..bd73ce7b5c0c --- /dev/null +++ b/deploy/operator/internal/controller/testdata/upgrade/latest/sglang/disaggregated/multi/disaggregatedset/input.yaml @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + name: workers + namespace: upgrade-latest-sglang-disaggregated-multi-disaggregatedset + annotations: + nvidia.com/enable-grove: 'false' + nvidia.com/dynamo-operator-origin-version: 0.0.0 + nvidia.com/enable-disaggregatedset: 'true' +spec: + backendFramework: sglang + components: + - name: frontend + type: frontend + replicas: 1 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + command: + - python3 + - -m + - dynamo.frontend + - name: prefill + type: prefill + replicas: 2 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + command: + - python3 + - -m + - dynamo.sglang + args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - prefill + resources: + limits: + nvidia.com/gpu: '1' + multinode: + nodeCount: 2 + - name: decode + type: decode + replicas: 2 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + command: + - python3 + - -m + - dynamo.sglang + args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - decode + resources: + limits: + nvidia.com/gpu: '1' + multinode: + nodeCount: 2 diff --git a/deploy/operator/internal/controller/testdata/upgrade/latest/sglang/disaggregated/multi/disaggregatedset/output.yaml b/deploy/operator/internal/controller/testdata/upgrade/latest/sglang/disaggregated/multi/disaggregatedset/output.yaml new file mode 100644 index 000000000000..9c9bee7f4bc4 --- /dev/null +++ b/deploy/operator/internal/controller/testdata/upgrade/latest/sglang/disaggregated/multi/disaggregatedset/output.yaml @@ -0,0 +1,1116 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + annotations: + nvidia.com/current-worker-hash-v2: $var:worker-hash + nvidia.com/dynamo-operator-origin-version: $var:operator-version + nvidia.com/enable-disaggregatedset: 'true' + nvidia.com/enable-grove: 'false' + nvidia.com/workload-provider: disaggregatedset + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + name: workers + namespace: upgrade-latest-sglang-disaggregated-multi-disaggregatedset + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: sglang + components: + - name: frontend + podTemplate: + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + name: main + replicas: 1 + type: frontend + - multinode: + nodeCount: 2 + name: prefill + podTemplate: + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - prefill + command: + - python3 + - -m + - dynamo.sglang + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + name: main + resources: + limits: + nvidia.com/gpu: '1' + replicas: 2 + type: prefill + - multinode: + nodeCount: 2 + name: decode + podTemplate: + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - decode + command: + - python3 + - -m + - dynamo.sglang + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + name: main + resources: + limits: + nvidia.com/gpu: '1' + replicas: 2 + type: decode +status: + components: + decode: + $strict: false + componentKind: LeaderWorkerSet + frontend: + $strict: false + componentKind: Deployment + componentNames: + - workers-frontend + runtimeNamespace: upgrade-latest-sglang-disaggregated-multi-disa-0a2f9f0657ca2f22 + prefill: + $strict: false + componentKind: LeaderWorkerSet + conditions: + - lastTransitionTime: $ignore + message: DisaggregatedSet pathway selected + observedGeneration: 1 + reason: Selected + status: 'True' + type: DisaggregatedSetEligible + - lastTransitionTime: $ignore + message: 'Resources not ready: workers: decode role "decode" slice 0 has 0 target + LeaderWorkerSets, expected 1; prefill role "prefill" slice 0 has 0 target LeaderWorkerSets, + expected 1; workers-frontend: Component deployment not ready - Available condition + not true' + observedGeneration: 1 + reason: some_resources_are_not_ready + status: 'False' + type: Ready + observedGeneration: 1 + state: pending +--- +apiVersion: v1 +data: + wait-for-leader.py: "import socket, time, json, ssl, urllib.request, os\n\nSA =\ + \ \"/var/run/secrets/kubernetes.io/serviceaccount\"\nhost = os.environ[\"LEADER_HOST\"\ + ]\nport = int(os.environ[\"LEADER_PORT\"])\n\ndef _k8s_ctx():\n return ssl.create_default_context(cafile=f\"\ + {SA}/ca.crt\")\n\ndef _k8s_headers():\n token = open(f\"{SA}/token\").read()\n\ + \ return {\"Authorization\": f\"Bearer {token}\"}\n\ndef _k8s_api():\n ns\ + \ = open(f\"{SA}/namespace\").read()\n return f\"https://kubernetes.default.svc/api/v1/namespaces/{ns}/pods\"\ + \n\ndef resolve_leader_ip():\n # gethostbyname is IPv4-only; fall back to IPv6\ + \ for IPv6-only clusters.\n try:\n return socket.gethostbyname(host)\n\ + \ except socket.gaierror:\n return socket.getaddrinfo(host, None, socket.AF_INET6,\ + \ socket.SOCK_STREAM)[0][4][0]\n\ndef leader_pod_is_healthy():\n try:\n \ + \ ip = resolve_leader_ip()\n except socket.gaierror:\n return False,\ + \ \"DNS resolution failed\", None, None\n try:\n req = urllib.request.Request(\n\ + \ f\"{_k8s_api()}?fieldSelector=status.podIP={ip}\",\n headers=_k8s_headers(),\n\ + \ )\n resp = json.loads(urllib.request.urlopen(req, context=_k8s_ctx(),\ + \ timeout=5).read())\n pods = resp.get(\"items\", [])\n if not pods:\n\ + \ return False, f\"no pod found with IP {ip}\", None, ip\n pod\ + \ = pods[0]\n name = pod[\"metadata\"].get(\"name\", \"unknown\")\n \ + \ uid = pod[\"metadata\"].get(\"uid\", \"unknown\")\n phase = pod.get(\"\ + status\", {}).get(\"phase\")\n deletion_ts = pod[\"metadata\"].get(\"deletionTimestamp\"\ + )\n info = f\"ip={ip} pod={name} uid={uid} phase={phase} deletionTimestamp={deletion_ts}\"\ + \n if deletion_ts:\n return False, f\"pod {name} is terminating\"\ + , info, ip\n if phase != \"Running\":\n return False, f\"pod\ + \ {name} phase is {phase}\", info, ip\n return True, \"\", info, ip\n \ + \ except Exception as e:\n # Fall back to TCP-only when the API is unavailable\ + \ (e.g. 403 no RBAC)\n return True, f\"K8s API unavailable ({e}), falling\ + \ back to TCP\", f\"ip={ip}\", ip\n\nprint(f\"Waiting for leader master port at\ + \ {host}:{port}...\", flush=True)\ntime.sleep(5)\nstart = time.monotonic()\nlast_status\ + \ = start\nlast_err = \"\"\nwhile True:\n healthy, reason, pod_info, leader_ip\ + \ = leader_pod_is_healthy()\n if healthy:\n try:\n s = socket.create_connection((leader_ip,\ + \ port), timeout=2)\n s.close()\n elapsed = time.monotonic()\ + \ - start\n print(f\"Leader master port ready (waited {elapsed:.1f}s)\ + \ [{pod_info}]\", flush=True)\n break\n except Exception as\ + \ e:\n last_err = f\"tcp: {type(e).__name__}: {e} [{pod_info}]\"\n\ + \ else:\n last_err = f\"{reason} [{pod_info}]\" if pod_info else reason\n\ + \ now = time.monotonic()\n if now - last_status >= 30:\n print(f\"\ + Still waiting for {host}:{port}... ({now - start:.0f}s elapsed, last: {last_err})\"\ + , flush=True)\n last_status = now\n time.sleep(5)\n" +kind: ConfigMap +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: a615bfc016f57bef619ea5c67d88da24755349bb07efff9da0b2c79f18d6d076 + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-graph-deployment-name: workers + name: workers-wait-leader-script + namespace: upgrade-latest-sglang-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + nvidia.com/dynamo-managed-service-metadata: '{"labels":["nvidia.com/dynamo-component","nvidia.com/dynamo-discovery-backend","nvidia.com/dynamo-discovery-enabled","nvidia.com/dynamo-graph-deployment-name","nvidia.com/dynamo-namespace","nvidia.com/dynamo-worker-hash"],"annotations":["nvidia.com/dynamo-operator-origin-version"]}' + nvidia.com/dynamo-operator-origin-version: $var:operator-version + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-multi-disa-0a2f9f0657ca2f22 + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-decode-{{worker-hash}} + namespace: upgrade-latest-sglang-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + clusterIP: $ignore + clusterIPs: + - $ignore + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: system + port: 9090 + protocol: TCP + targetPort: system + selector: + disaggregatedset.x-k8s.io/name: workers + disaggregatedset.x-k8s.io/revision: $var:ds-revision + disaggregatedset.x-k8s.io/role: decode + role: leader + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + nvidia.com/dynamo-managed-service-metadata: '{"labels":["nvidia.com/dynamo-component","nvidia.com/dynamo-discovery-backend","nvidia.com/dynamo-discovery-enabled","nvidia.com/dynamo-graph-deployment-name","nvidia.com/dynamo-namespace"],"annotations":["nvidia.com/dynamo-operator-origin-version"]}' + nvidia.com/dynamo-operator-origin-version: $var:operator-version + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-multi-disa-0a2f9f0657ca2f22 + name: workers-frontend + namespace: upgrade-latest-sglang-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + clusterIP: $ignore + clusterIPs: + - $ignore + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: http + port: 8000 + protocol: TCP + targetPort: http + selector: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-component-type: frontend + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-multi-disa-0a2f9f0657ca2f22 + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + nvidia.com/dynamo-managed-service-metadata: '{"labels":["nvidia.com/dynamo-component","nvidia.com/dynamo-discovery-backend","nvidia.com/dynamo-discovery-enabled","nvidia.com/dynamo-graph-deployment-name","nvidia.com/dynamo-namespace","nvidia.com/dynamo-worker-hash"],"annotations":["nvidia.com/dynamo-operator-origin-version"]}' + nvidia.com/dynamo-operator-origin-version: $var:operator-version + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-multi-disa-0a2f9f0657ca2f22 + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-prefill-{{worker-hash}} + namespace: upgrade-latest-sglang-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + clusterIP: $ignore + clusterIPs: + - $ignore + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: system + port: 9090 + protocol: TCP + targetPort: system + selector: + disaggregatedset.x-k8s.io/name: workers + disaggregatedset.x-k8s.io/revision: $var:ds-revision + disaggregatedset.x-k8s.io/role: prefill + role: leader + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery + namespace: upgrade-latest-sglang-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + generation: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-multi-disa-0a2f9f0657ca2f22 + name: workers-frontend + namespace: upgrade-latest-sglang-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: workers-frontend + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + nvidia.com/selector: workers-frontend + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-component-type: frontend + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-multi-disa-0a2f9f0657ca2f22 + nvidia.com/metrics-enabled: 'true' + nvidia.com/selector: workers-frontend + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + env: + - name: DYNAMO_PORT + value: '8000' + - name: DYN_COMPONENT + value: frontend + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_HTTP_PORT + value: '8000' + - name: DYN_NAMESPACE + value: upgrade-latest-sglang-disaggregated-multi-disa-0a2f9f0657ca2f22 + - name: DYN_NAMESPACE_PREFIX + value: upgrade-latest-sglang-disaggregated-multi-disa-0a2f9f0657ca2f22 + - name: DYN_NAMESPACE_PREFIX_STRICT + value: 'true' + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-sglang-disaggregated-multi-disaggregatedset + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.uid + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /live + port: http + scheme: HTTP + initialDelaySeconds: 15 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: main + ports: + - containerPort: 8000 + name: http + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: http + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 3 + resources: {} + startupProbe: + failureThreshold: 30 + httpGet: + path: /live + port: http + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 1000 + serviceAccount: workers-k8s-service-discovery + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory +status: + $ignore: true +--- +apiVersion: disaggregatedset.x-k8s.io/v1 +kind: DisaggregatedSet +metadata: + creationTimestamp: $ignore + generation: $ignore + labels: + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/selector: workers + name: workers + namespace: upgrade-latest-sglang-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + roles: + - name: prefill + spec: + leaderWorkerTemplate: + leaderTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-component-type: prefill + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-multi-disa-0a2f9f0657ca2f22 + nvidia.com/dynamo-sub-component-type: prefill + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: leader + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - prefill + - --dist-init-addr + - $(DYNAMO_LEADER_ADDRESS):29500 + - --nnodes + - '2' + - --node-rank + - $(DYNAMO_RANK) + command: + - python3 + - -m + - dynamo.sglang + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: prefill + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-sglang-disaggregated-multi-disa-0a2f9f0657ca2f22 + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-sglang-disaggregated-multi-disaggregatedset + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + livenessProbe: + failureThreshold: 3 + httpGet: + path: /live + port: system + periodSeconds: 5 + timeoutSeconds: 4 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: system + periodSeconds: 10 + timeoutSeconds: 4 + resources: + limits: + nvidia.com/gpu: '1' + startupProbe: + failureThreshold: 720 + httpGet: + path: /live + port: system + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + restartPolicy: RecreateGroupOnPodRestart + size: 2 + workerTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-component-type: prefill + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-multi-disa-0a2f9f0657ca2f22 + nvidia.com/dynamo-sub-component-type: prefill + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: worker + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - prefill + - --dist-init-addr + - $(DYNAMO_LEADER_ADDRESS):29500 + - --nnodes + - '2' + - --node-rank + - $(DYNAMO_RANK) + command: + - python3 + - -m + - dynamo.sglang + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: prefill + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-sglang-disaggregated-multi-disa-0a2f9f0657ca2f22 + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-sglang-disaggregated-multi-disaggregatedset + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + resources: + limits: + nvidia.com/gpu: '1' + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + replicas: 2 + rolloutStrategy: + type: RollingUpdate + startupPolicy: LeaderCreated + - name: decode + spec: + leaderWorkerTemplate: + leaderTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + labels: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-component-type: decode + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-multi-disa-0a2f9f0657ca2f22 + nvidia.com/dynamo-sub-component-type: decode + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: leader + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - decode + - --dist-init-addr + - $(DYNAMO_LEADER_ADDRESS):29500 + - --nnodes + - '2' + - --node-rank + - $(DYNAMO_RANK) + command: + - python3 + - -m + - dynamo.sglang + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: decode + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-sglang-disaggregated-multi-disa-0a2f9f0657ca2f22 + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-sglang-disaggregated-multi-disaggregatedset + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + livenessProbe: + failureThreshold: 3 + httpGet: + path: /live + port: system + periodSeconds: 5 + timeoutSeconds: 4 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: system + periodSeconds: 10 + timeoutSeconds: 4 + resources: + limits: + nvidia.com/gpu: '1' + startupProbe: + failureThreshold: 720 + httpGet: + path: /live + port: system + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + restartPolicy: RecreateGroupOnPodRestart + size: 2 + workerTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + labels: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-component-type: decode + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-multi-disa-0a2f9f0657ca2f22 + nvidia.com/dynamo-sub-component-type: decode + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: worker + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - decode + - --dist-init-addr + - $(DYNAMO_LEADER_ADDRESS):29500 + - --nnodes + - '2' + - --node-rank + - $(DYNAMO_RANK) + command: + - python3 + - -m + - dynamo.sglang + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: decode + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-sglang-disaggregated-multi-disa-0a2f9f0657ca2f22 + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-sglang-disaggregated-multi-disaggregatedset + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + resources: + limits: + nvidia.com/gpu: '1' + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + replicas: 2 + rolloutStrategy: + type: RollingUpdate + startupPolicy: LeaderCreated + slices: 1 +--- +apiVersion: nvidia.com/v1beta1 +kind: DynamoComponentDeployment +metadata: + annotations: + nvidia.com/dcd-spec: '{"version":1,"spec":{},"rawJSON":[{"path":"serviceName","nil":true}]}' + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-multi-disa-0a2f9f0657ca2f22 + name: workers-frontend + namespace: upgrade-latest-sglang-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: sglang + name: frontend + podTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + name: main + replicas: 1 + type: frontend +status: + component: + $strict: false + componentKind: Deployment + componentNames: + - workers-frontend + runtimeNamespace: upgrade-latest-sglang-disaggregated-multi-disa-0a2f9f0657ca2f22 + conditions: + - lastTransitionTime: $ignore + message: $ignore + reason: $ignore + status: 'False' + type: Available + - lastTransitionTime: $ignore + message: DynamoComponent is not ready + reason: ComponentNotReady + status: 'False' + type: DynamoComponentReady + observedGeneration: 1 +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: 0d454cc8266ac38ed6453ee2ef3cf6d9dd237faebc63af22f67ca911e106796c + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery-role + namespace: upgrade-latest-sglang-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +rules: +- apiGroups: + - '' + resources: + - endpoints + - pods + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - nvidia.com + resources: + - dynamoworkermetadatas + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: 6f014cc1987d161cbcc76353cb1d6be0d80d66da3f96e24e24bc98fcd6bc8818 + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery-binding + namespace: upgrade-latest-sglang-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: workers-k8s-service-discovery-role +subjects: +- kind: ServiceAccount + name: workers-k8s-service-discovery + namespace: upgrade-latest-sglang-disaggregated-multi-disaggregatedset diff --git a/deploy/operator/internal/controller/testdata/upgrade/latest/trtllm/aggregated/multi/component/input.yaml b/deploy/operator/internal/controller/testdata/upgrade/latest/trtllm/aggregated/multi/component/input.yaml new file mode 100644 index 000000000000..4ea7bd177dd3 --- /dev/null +++ b/deploy/operator/internal/controller/testdata/upgrade/latest/trtllm/aggregated/multi/component/input.yaml @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + name: workers + namespace: upgrade-latest-trtllm-aggregated-multi-component + annotations: + nvidia.com/enable-grove: 'false' + nvidia.com/dynamo-operator-origin-version: 0.0.0 +spec: + backendFramework: trtllm + components: + - name: frontend + type: frontend + replicas: 1 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + command: + - python3 + - -m + - dynamo.frontend + - name: worker + type: worker + replicas: 2 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/trtllm-runtime:1.6.0 + command: + - python3 + - -m + - dynamo.trtllm + args: + - --model-path + - Qwen/Qwen3-0.6B + resources: + limits: + nvidia.com/gpu: '1' + multinode: + nodeCount: 2 diff --git a/deploy/operator/internal/controller/testdata/upgrade/latest/trtllm/aggregated/multi/component/output.yaml b/deploy/operator/internal/controller/testdata/upgrade/latest/trtllm/aggregated/multi/component/output.yaml new file mode 100644 index 000000000000..6164fd7b7eaf --- /dev/null +++ b/deploy/operator/internal/controller/testdata/upgrade/latest/trtllm/aggregated/multi/component/output.yaml @@ -0,0 +1,922 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + annotations: + nvidia.com/current-worker-hash-v2: $var:worker-hash + nvidia.com/dynamo-operator-origin-version: $var:operator-version + nvidia.com/enable-grove: 'false' + nvidia.com/workload-provider: component + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + name: workers + namespace: upgrade-latest-trtllm-aggregated-multi-component + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: trtllm + components: + - name: frontend + podTemplate: + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + name: main + replicas: 1 + type: frontend + - multinode: + nodeCount: 2 + name: worker + podTemplate: + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + command: + - python3 + - -m + - dynamo.trtllm + image: nvcr.io/nvidia/ai-dynamo/trtllm-runtime:1.6.0 + name: main + resources: + limits: + nvidia.com/gpu: '1' + replicas: 2 + type: worker +status: + components: + frontend: + $strict: false + componentKind: Deployment + componentNames: + - workers-frontend + runtimeNamespace: upgrade-latest-trtllm-aggregated-multi-component-workers + worker: + $strict: false + componentKind: LeaderWorkerSet + componentNames: + - $var:worker-string:workers-worker-{{worker-hash}}-0 + runtimeNamespace: $var:worker-string:upgrade-latest-trtllm-aggregated-multi-component-workers-{{worker-hash}} + servedModelName: Qwen/Qwen3-0.6B + conditions: + - lastTransitionTime: $ignore + message: '$var:worker-string:Resources not ready: workers-frontend: Component + deployment not ready - Available condition not true; workers-worker-{{worker-hash}}: + Component deployment not ready - Available condition not true' + observedGeneration: 1 + reason: some_resources_are_not_ready + status: 'False' + type: Ready + observedGeneration: 1 + state: pending +--- +apiVersion: v1 +data: + wait-for-leader.py: "import socket, time, json, ssl, urllib.request, os\n\nSA =\ + \ \"/var/run/secrets/kubernetes.io/serviceaccount\"\nhost = os.environ[\"LEADER_HOST\"\ + ]\nport = int(os.environ[\"LEADER_PORT\"])\n\ndef _k8s_ctx():\n return ssl.create_default_context(cafile=f\"\ + {SA}/ca.crt\")\n\ndef _k8s_headers():\n token = open(f\"{SA}/token\").read()\n\ + \ return {\"Authorization\": f\"Bearer {token}\"}\n\ndef _k8s_api():\n ns\ + \ = open(f\"{SA}/namespace\").read()\n return f\"https://kubernetes.default.svc/api/v1/namespaces/{ns}/pods\"\ + \n\ndef resolve_leader_ip():\n # gethostbyname is IPv4-only; fall back to IPv6\ + \ for IPv6-only clusters.\n try:\n return socket.gethostbyname(host)\n\ + \ except socket.gaierror:\n return socket.getaddrinfo(host, None, socket.AF_INET6,\ + \ socket.SOCK_STREAM)[0][4][0]\n\ndef leader_pod_is_healthy():\n try:\n \ + \ ip = resolve_leader_ip()\n except socket.gaierror:\n return False,\ + \ \"DNS resolution failed\", None, None\n try:\n req = urllib.request.Request(\n\ + \ f\"{_k8s_api()}?fieldSelector=status.podIP={ip}\",\n headers=_k8s_headers(),\n\ + \ )\n resp = json.loads(urllib.request.urlopen(req, context=_k8s_ctx(),\ + \ timeout=5).read())\n pods = resp.get(\"items\", [])\n if not pods:\n\ + \ return False, f\"no pod found with IP {ip}\", None, ip\n pod\ + \ = pods[0]\n name = pod[\"metadata\"].get(\"name\", \"unknown\")\n \ + \ uid = pod[\"metadata\"].get(\"uid\", \"unknown\")\n phase = pod.get(\"\ + status\", {}).get(\"phase\")\n deletion_ts = pod[\"metadata\"].get(\"deletionTimestamp\"\ + )\n info = f\"ip={ip} pod={name} uid={uid} phase={phase} deletionTimestamp={deletion_ts}\"\ + \n if deletion_ts:\n return False, f\"pod {name} is terminating\"\ + , info, ip\n if phase != \"Running\":\n return False, f\"pod\ + \ {name} phase is {phase}\", info, ip\n return True, \"\", info, ip\n \ + \ except Exception as e:\n # Fall back to TCP-only when the API is unavailable\ + \ (e.g. 403 no RBAC)\n return True, f\"K8s API unavailable ({e}), falling\ + \ back to TCP\", f\"ip={ip}\", ip\n\nprint(f\"Waiting for leader master port at\ + \ {host}:{port}...\", flush=True)\ntime.sleep(5)\nstart = time.monotonic()\nlast_status\ + \ = start\nlast_err = \"\"\nwhile True:\n healthy, reason, pod_info, leader_ip\ + \ = leader_pod_is_healthy()\n if healthy:\n try:\n s = socket.create_connection((leader_ip,\ + \ port), timeout=2)\n s.close()\n elapsed = time.monotonic()\ + \ - start\n print(f\"Leader master port ready (waited {elapsed:.1f}s)\ + \ [{pod_info}]\", flush=True)\n break\n except Exception as\ + \ e:\n last_err = f\"tcp: {type(e).__name__}: {e} [{pod_info}]\"\n\ + \ else:\n last_err = f\"{reason} [{pod_info}]\" if pod_info else reason\n\ + \ now = time.monotonic()\n if now - last_status >= 30:\n print(f\"\ + Still waiting for {host}:{port}... ({now - start:.0f}s elapsed, last: {last_err})\"\ + , flush=True)\n last_status = now\n time.sleep(5)\n" +kind: ConfigMap +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: a615bfc016f57bef619ea5c67d88da24755349bb07efff9da0b2c79f18d6d076 + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-graph-deployment-name: workers + name: workers-wait-leader-script + namespace: upgrade-latest-trtllm-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-aggregated-multi-component-workers + name: workers-frontend + namespace: upgrade-latest-trtllm-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: workers-frontend + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + clusterIP: $ignore + clusterIPs: + - $ignore + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: http + port: 8000 + protocol: TCP + targetPort: http + selector: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-component-type: frontend + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-aggregated-multi-component-workers + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-component: worker + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-aggregated-multi-component-workers + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-worker-{{worker-hash}} + namespace: upgrade-latest-trtllm-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: $var:worker-string:workers-worker-{{worker-hash}} + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + clusterIP: $ignore + clusterIPs: + - $ignore + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: system + port: 9090 + protocol: TCP + targetPort: system + selector: + nvidia.com/dynamo-component: worker + nvidia.com/dynamo-component-type: worker + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-aggregated-multi-component-workers + role: leader + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery + namespace: upgrade-latest-trtllm-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + generation: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-aggregated-multi-component-workers + name: workers-frontend + namespace: upgrade-latest-trtllm-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: workers-frontend + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + nvidia.com/selector: workers-frontend + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-component-type: frontend + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-aggregated-multi-component-workers + nvidia.com/metrics-enabled: 'true' + nvidia.com/selector: workers-frontend + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + env: + - name: DYNAMO_PORT + value: '8000' + - name: DYN_COMPONENT + value: frontend + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_HTTP_PORT + value: '8000' + - name: DYN_NAMESPACE + value: upgrade-latest-trtllm-aggregated-multi-component-workers + - name: DYN_NAMESPACE_PREFIX + value: upgrade-latest-trtllm-aggregated-multi-component-workers + - name: DYN_NAMESPACE_PREFIX_STRICT + value: 'true' + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-trtllm-aggregated-multi-component + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.uid + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /live + port: http + scheme: HTTP + initialDelaySeconds: 15 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: main + ports: + - containerPort: 8000 + name: http + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: http + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 3 + resources: {} + startupProbe: + failureThreshold: 30 + httpGet: + path: /live + port: http + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 1000 + serviceAccount: workers-k8s-service-discovery + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory +status: + $ignore: true +--- +apiVersion: leaderworkerset.x-k8s.io/v1 +kind: LeaderWorkerSet +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + generation: $ignore + labels: + nvidia.com/dynamo-component: worker + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-aggregated-multi-component-workers + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-worker-{{worker-hash}}-0 + namespace: upgrade-latest-trtllm-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: $var:worker-string:workers-worker-{{worker-hash}} + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + leaderWorkerTemplate: + leaderTemplate: + metadata: + labels: + nvidia.com/dynamo-component: worker + nvidia.com/dynamo-component-type: worker + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-aggregated-multi-component-workers + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: leader + spec: + containers: + - args: + - "mkdir -p $HOME/.ssh && ls -la /ssh-pk/ && cp /ssh-pk/private.key $HOME/.ssh/id_rsa\ + \ && cp /ssh-pk/private.key.pub $HOME/.ssh/id_rsa.pub && cp /ssh-pk/private.key.pub\ + \ $HOME/.ssh/authorized_keys && chmod 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys\ + \ && chmod 644 $HOME/.ssh/id_rsa.pub && printf 'Host *\\nIdentityFile\ + \ '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort 2222\\n' > $HOME/.ssh/config\ + \ && TIMEOUT=300; START_TIME=$(date +%s); for worker in $(echo \"$LWS_LEADER_ADDRESS\"\ + \ | sed 's/\\./-1\\./'); do echo \"Waiting for DNS: $worker\"; until getent\ + \ hosts $worker >/dev/null 2>&1; do CURRENT_TIME=$(date +%s); if [ $((CURRENT_TIME\ + \ - START_TIME)) -gt $TIMEOUT ]; then echo \"ERROR: Timeout waiting for\ + \ DNS: $worker\"; exit 1; fi; echo \"DNS not ready for $worker, retrying...\"\ + ; sleep 2; done; echo \"\u2713 DNS resolved: $worker\"; done; echo \"\ + All workers DNS ready\" && mpirun $([ \"$(id -u)\" = \"0\" ] && echo --allow-run-as-root)\ + \ --oversubscribe -n 2 -H $(LWS_LEADER_ADDRESS),$(echo \"$LWS_LEADER_ADDRESS\"\ + \ | sed 's/\\./-1\\./') --mca pml ob1 --mca plm_rsh_args \"-p 2222 -o\ + \ StrictHostKeyChecking=no -i $HOME/.ssh/id_rsa\" -x CPATH -x CUDA_VISIBLE_DEVICES\ + \ -x DYNAMO_LEADER_ADDRESS -x DYNAMO_RANK -x DYN_COMPONENT -x DYN_DISCOVERY_BACKEND\ + \ -x DYN_FORWARDPASS_METRIC_PORT -x DYN_HEALTH_CHECK_ENABLED -x DYN_NAMESPACE\ + \ -x DYN_NAMESPACE_WORKER_SUFFIX -x DYN_PARENT_DGD_K8S_NAME -x DYN_PARENT_DGD_K8S_NAMESPACE\ + \ -x DYN_SYSTEM_ENABLED -x DYN_SYSTEM_PORT -x DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS\ + \ -x HF_DATASETS_CACHE -x HF_ENDPOINT -x HF_HOME -x HF_TOKEN -x HOME -x\ + \ HUGGING_FACE_HUB_TOKEN -x LD_LIBRARY_PATH -x MODEL_PATH -x NCCL_DEBUG\ + \ -x NCCL_IB_DISABLE -x NCCL_P2P_DISABLE -x NIXL_TELEMETRY_ENABLE -x NIXL_TELEMETRY_EXPORTER\ + \ -x NIXL_TELEMETRY_PROMETHEUS_PORT -x OMPI_MCA_orte_keep_fqdn_hostnames\ + \ -x PATH -x POD_NAME -x POD_NAMESPACE -x POD_UID -x PYTHONPATH -x TENSORRT_LLM_CACHE_DIR\ + \ -x TOKENIZERS_PARALLELISM -x TRANSFORMERS_CACHE -x TRITON_PTXAS_PATH\ + \ -x TRTLLM_USE_UCX_KVCACHE -x USER bash -c 'trtllm-llmapi-launch python3\ + \ -m dynamo.trtllm --model-path Qwen/Qwen3-0.6B'" + command: + - /bin/sh + - -c + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: worker + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-trtllm-aggregated-multi-component-workers + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-trtllm-aggregated-multi-component + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OMPI_MCA_orte_keep_fqdn_hostnames + value: '1' + image: nvcr.io/nvidia/ai-dynamo/trtllm-runtime:1.6.0 + livenessProbe: + failureThreshold: 3 + httpGet: + path: /live + port: system + periodSeconds: 5 + timeoutSeconds: 4 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: system + periodSeconds: 10 + timeoutSeconds: 4 + resources: + limits: + nvidia.com/gpu: '1' + startupProbe: + failureThreshold: 720 + httpGet: + path: /live + port: system + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /ssh-pk + name: upgrade-mpi-ssh + readOnly: true + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - name: upgrade-mpi-ssh + secret: + defaultMode: 420 + secretName: upgrade-mpi-ssh + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + restartPolicy: RecreateGroupOnPodRestart + size: 2 + workerTemplate: + metadata: + labels: + nvidia.com/dynamo-component: worker + nvidia.com/dynamo-component-type: worker + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-aggregated-multi-component-workers + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: worker + spec: + containers: + - args: + - mkdir -p $HOME/.ssh $HOME/.ssh/host_keys $HOME/.ssh/run && ls -la /ssh-pk/ + && cp /ssh-pk/private.key $HOME/.ssh/id_rsa && cp /ssh-pk/private.key.pub + $HOME/.ssh/id_rsa.pub && cp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys + && chmod 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys && chmod 644 + $HOME/.ssh/id_rsa.pub && printf 'Host *\nIdentityFile '$HOME'/.ssh/id_rsa\nStrictHostKeyChecking + no\nPort 2222\n' > $HOME/.ssh/config && ssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key + -N '' && ssh-keygen -t ecdsa -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key + -N '' && ssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key + -N '' && printf 'Port 2222\nHostKey '$HOME'/.ssh/host_keys/ssh_host_rsa_key\nHostKey + '$HOME'/.ssh/host_keys/ssh_host_ecdsa_key\nHostKey '$HOME'/.ssh/host_keys/ssh_host_ed25519_key\nPidFile + '$HOME'/.ssh/run/sshd.pid\nStrictModes no\nPermitRootLogin yes\nPasswordAuthentication + no\nPubkeyAuthentication yes\nAuthorizedKeysFile '$HOME'/.ssh/authorized_keys\n' + > $HOME/.ssh/sshd_config && /usr/sbin/sshd -D -f $HOME/.ssh/sshd_config + command: + - /bin/sh + - -c + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: worker + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-trtllm-aggregated-multi-component-workers + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-trtllm-aggregated-multi-component + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OMPI_MCA_orte_keep_fqdn_hostnames + value: '1' + image: nvcr.io/nvidia/ai-dynamo/trtllm-runtime:1.6.0 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 10 + initialDelaySeconds: 20 + periodSeconds: 20 + tcpSocket: + port: 2222 + timeoutSeconds: 5 + resources: + limits: + nvidia.com/gpu: '1' + volumeMounts: + - mountPath: /ssh-pk + name: upgrade-mpi-ssh + readOnly: true + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - name: upgrade-mpi-ssh + secret: + defaultMode: 420 + secretName: upgrade-mpi-ssh + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + networkConfig: + subdomainPolicy: Shared + replicas: 2 + rolloutStrategy: + rollingUpdateConfiguration: + maxSurge: 0 + maxUnavailable: 1 + partition: 0 + type: RollingUpdate + startupPolicy: LeaderCreated +--- +apiVersion: nvidia.com/v1beta1 +kind: DynamoComponentDeployment +metadata: + annotations: + nvidia.com/dcd-spec: '{"version":1,"spec":{},"rawJSON":[{"path":"serviceName","nil":true}]}' + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-aggregated-multi-component-workers + name: workers-frontend + namespace: upgrade-latest-trtllm-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: trtllm + name: frontend + podTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + name: main + replicas: 1 + type: frontend +status: + component: + $strict: false + componentKind: Deployment + componentNames: + - workers-frontend + runtimeNamespace: upgrade-latest-trtllm-aggregated-multi-component-workers + conditions: + - lastTransitionTime: $ignore + message: $ignore + reason: $ignore + status: 'False' + type: Available + - lastTransitionTime: $ignore + message: DynamoComponent is not ready + reason: ComponentNotReady + status: 'False' + type: DynamoComponentReady + observedGeneration: 1 +--- +apiVersion: nvidia.com/v1beta1 +kind: DynamoComponentDeployment +metadata: + annotations: + nvidia.com/dcd-spec: '{"version":1,"spec":{"resources":{"limits":{"gpu":"1"}},"extraPodSpec":{"mainContainer":{"name":"","resources":{}}}},"rawJSON":[{"path":"serviceName","nil":true}]}' + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + labels: + nvidia.com/dynamo-component: worker + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-aggregated-multi-component-workers + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-worker-{{worker-hash}} + namespace: upgrade-latest-trtllm-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: trtllm + multinode: + nodeCount: 2 + name: worker + podTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + labels: + nvidia.com/dynamo-worker-hash: $var:worker-hash + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + command: + - python3 + - -m + - dynamo.trtllm + image: nvcr.io/nvidia/ai-dynamo/trtllm-runtime:1.6.0 + name: main + resources: + limits: + nvidia.com/gpu: '1' + replicas: 2 + type: worker +status: + component: + $strict: false + componentKind: LeaderWorkerSet + componentNames: + - $var:worker-string:workers-worker-{{worker-hash}}-0 + runtimeNamespace: $var:worker-string:upgrade-latest-trtllm-aggregated-multi-component-workers-{{worker-hash}} + servedModelName: Qwen/Qwen3-0.6B + conditions: + - lastTransitionTime: $ignore + message: $ignore + reason: $ignore + status: 'False' + type: Available + - lastTransitionTime: $ignore + message: DynamoComponent is not ready + reason: ComponentNotReady + status: 'False' + type: DynamoComponentReady + observedGeneration: 1 +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: 0d454cc8266ac38ed6453ee2ef3cf6d9dd237faebc63af22f67ca911e106796c + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery-role + namespace: upgrade-latest-trtllm-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +rules: +- apiGroups: + - '' + resources: + - endpoints + - pods + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - nvidia.com + resources: + - dynamoworkermetadatas + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: 4efe37de7d9c476fb43b8cc88b3cff7ebb67288a410e32326d2ed29b17179c84 + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery-binding + namespace: upgrade-latest-trtllm-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: workers-k8s-service-discovery-role +subjects: +- kind: ServiceAccount + name: workers-k8s-service-discovery + namespace: upgrade-latest-trtllm-aggregated-multi-component diff --git a/deploy/operator/internal/controller/testdata/upgrade/latest/trtllm/disaggregated/multi/component/input.yaml b/deploy/operator/internal/controller/testdata/upgrade/latest/trtllm/disaggregated/multi/component/input.yaml new file mode 100644 index 000000000000..000d357e7a0c --- /dev/null +++ b/deploy/operator/internal/controller/testdata/upgrade/latest/trtllm/disaggregated/multi/component/input.yaml @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + name: workers + namespace: upgrade-latest-trtllm-disaggregated-multi-component + annotations: + nvidia.com/enable-grove: 'false' + nvidia.com/dynamo-operator-origin-version: 0.0.0 +spec: + backendFramework: trtllm + components: + - name: frontend + type: frontend + replicas: 1 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + command: + - python3 + - -m + - dynamo.frontend + - name: prefill + type: prefill + replicas: 2 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/trtllm-runtime:1.6.0 + command: + - python3 + - -m + - dynamo.trtllm + args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - prefill + resources: + limits: + nvidia.com/gpu: '1' + multinode: + nodeCount: 2 + - name: decode + type: decode + replicas: 2 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/trtllm-runtime:1.6.0 + command: + - python3 + - -m + - dynamo.trtllm + args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - decode + resources: + limits: + nvidia.com/gpu: '1' + multinode: + nodeCount: 2 diff --git a/deploy/operator/internal/controller/testdata/upgrade/latest/trtllm/disaggregated/multi/component/output.yaml b/deploy/operator/internal/controller/testdata/upgrade/latest/trtllm/disaggregated/multi/component/output.yaml new file mode 100644 index 000000000000..3a2bbfa672d2 --- /dev/null +++ b/deploy/operator/internal/controller/testdata/upgrade/latest/trtllm/disaggregated/multi/component/output.yaml @@ -0,0 +1,1392 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + annotations: + nvidia.com/current-worker-hash-v2: $var:worker-hash + nvidia.com/dynamo-operator-origin-version: $var:operator-version + nvidia.com/enable-grove: 'false' + nvidia.com/workload-provider: component + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + name: workers + namespace: upgrade-latest-trtllm-disaggregated-multi-component + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: trtllm + components: + - name: frontend + podTemplate: + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + name: main + replicas: 1 + type: frontend + - multinode: + nodeCount: 2 + name: prefill + podTemplate: + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - prefill + command: + - python3 + - -m + - dynamo.trtllm + image: nvcr.io/nvidia/ai-dynamo/trtllm-runtime:1.6.0 + name: main + resources: + limits: + nvidia.com/gpu: '1' + replicas: 2 + type: prefill + - multinode: + nodeCount: 2 + name: decode + podTemplate: + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - decode + command: + - python3 + - -m + - dynamo.trtllm + image: nvcr.io/nvidia/ai-dynamo/trtllm-runtime:1.6.0 + name: main + resources: + limits: + nvidia.com/gpu: '1' + replicas: 2 + type: decode +status: + components: + decode: + $strict: false + componentKind: LeaderWorkerSet + componentNames: + - $var:worker-string:workers-decode-{{worker-hash}}-0 + runtimeNamespace: $var:worker-string:upgrade-latest-trtllm-disaggregated-multi-component-workers-{{worker-hash}} + servedModelName: Qwen/Qwen3-0.6B + frontend: + $strict: false + componentKind: Deployment + componentNames: + - workers-frontend + runtimeNamespace: upgrade-latest-trtllm-disaggregated-multi-component-workers + prefill: + $strict: false + componentKind: LeaderWorkerSet + componentNames: + - $var:worker-string:workers-prefill-{{worker-hash}}-0 + runtimeNamespace: $var:worker-string:upgrade-latest-trtllm-disaggregated-multi-component-workers-{{worker-hash}} + servedModelName: Qwen/Qwen3-0.6B + conditions: + - lastTransitionTime: $ignore + message: '$var:worker-string:Resources not ready: workers-decode-{{worker-hash}}: + Component deployment not ready - Available condition not true; workers-frontend: + Component deployment not ready - Available condition not true; workers-prefill-{{worker-hash}}: + Component deployment not ready - Available condition not true' + observedGeneration: 1 + reason: some_resources_are_not_ready + status: 'False' + type: Ready + observedGeneration: 1 + state: pending +--- +apiVersion: v1 +data: + wait-for-leader.py: "import socket, time, json, ssl, urllib.request, os\n\nSA =\ + \ \"/var/run/secrets/kubernetes.io/serviceaccount\"\nhost = os.environ[\"LEADER_HOST\"\ + ]\nport = int(os.environ[\"LEADER_PORT\"])\n\ndef _k8s_ctx():\n return ssl.create_default_context(cafile=f\"\ + {SA}/ca.crt\")\n\ndef _k8s_headers():\n token = open(f\"{SA}/token\").read()\n\ + \ return {\"Authorization\": f\"Bearer {token}\"}\n\ndef _k8s_api():\n ns\ + \ = open(f\"{SA}/namespace\").read()\n return f\"https://kubernetes.default.svc/api/v1/namespaces/{ns}/pods\"\ + \n\ndef resolve_leader_ip():\n # gethostbyname is IPv4-only; fall back to IPv6\ + \ for IPv6-only clusters.\n try:\n return socket.gethostbyname(host)\n\ + \ except socket.gaierror:\n return socket.getaddrinfo(host, None, socket.AF_INET6,\ + \ socket.SOCK_STREAM)[0][4][0]\n\ndef leader_pod_is_healthy():\n try:\n \ + \ ip = resolve_leader_ip()\n except socket.gaierror:\n return False,\ + \ \"DNS resolution failed\", None, None\n try:\n req = urllib.request.Request(\n\ + \ f\"{_k8s_api()}?fieldSelector=status.podIP={ip}\",\n headers=_k8s_headers(),\n\ + \ )\n resp = json.loads(urllib.request.urlopen(req, context=_k8s_ctx(),\ + \ timeout=5).read())\n pods = resp.get(\"items\", [])\n if not pods:\n\ + \ return False, f\"no pod found with IP {ip}\", None, ip\n pod\ + \ = pods[0]\n name = pod[\"metadata\"].get(\"name\", \"unknown\")\n \ + \ uid = pod[\"metadata\"].get(\"uid\", \"unknown\")\n phase = pod.get(\"\ + status\", {}).get(\"phase\")\n deletion_ts = pod[\"metadata\"].get(\"deletionTimestamp\"\ + )\n info = f\"ip={ip} pod={name} uid={uid} phase={phase} deletionTimestamp={deletion_ts}\"\ + \n if deletion_ts:\n return False, f\"pod {name} is terminating\"\ + , info, ip\n if phase != \"Running\":\n return False, f\"pod\ + \ {name} phase is {phase}\", info, ip\n return True, \"\", info, ip\n \ + \ except Exception as e:\n # Fall back to TCP-only when the API is unavailable\ + \ (e.g. 403 no RBAC)\n return True, f\"K8s API unavailable ({e}), falling\ + \ back to TCP\", f\"ip={ip}\", ip\n\nprint(f\"Waiting for leader master port at\ + \ {host}:{port}...\", flush=True)\ntime.sleep(5)\nstart = time.monotonic()\nlast_status\ + \ = start\nlast_err = \"\"\nwhile True:\n healthy, reason, pod_info, leader_ip\ + \ = leader_pod_is_healthy()\n if healthy:\n try:\n s = socket.create_connection((leader_ip,\ + \ port), timeout=2)\n s.close()\n elapsed = time.monotonic()\ + \ - start\n print(f\"Leader master port ready (waited {elapsed:.1f}s)\ + \ [{pod_info}]\", flush=True)\n break\n except Exception as\ + \ e:\n last_err = f\"tcp: {type(e).__name__}: {e} [{pod_info}]\"\n\ + \ else:\n last_err = f\"{reason} [{pod_info}]\" if pod_info else reason\n\ + \ now = time.monotonic()\n if now - last_status >= 30:\n print(f\"\ + Still waiting for {host}:{port}... ({now - start:.0f}s elapsed, last: {last_err})\"\ + , flush=True)\n last_status = now\n time.sleep(5)\n" +kind: ConfigMap +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: a615bfc016f57bef619ea5c67d88da24755349bb07efff9da0b2c79f18d6d076 + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-graph-deployment-name: workers + name: workers-wait-leader-script + namespace: upgrade-latest-trtllm-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-disaggregated-multi-component-workers + nvidia.com/dynamo-sub-component-type: decode + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-decode-{{worker-hash}} + namespace: upgrade-latest-trtllm-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: $var:worker-string:workers-decode-{{worker-hash}} + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + clusterIP: $ignore + clusterIPs: + - $ignore + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: system + port: 9090 + protocol: TCP + targetPort: system + selector: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-component-type: decode + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-disaggregated-multi-component-workers + role: leader + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-disaggregated-multi-component-workers + name: workers-frontend + namespace: upgrade-latest-trtllm-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: workers-frontend + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + clusterIP: $ignore + clusterIPs: + - $ignore + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: http + port: 8000 + protocol: TCP + targetPort: http + selector: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-component-type: frontend + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-disaggregated-multi-component-workers + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-disaggregated-multi-component-workers + nvidia.com/dynamo-sub-component-type: prefill + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-prefill-{{worker-hash}} + namespace: upgrade-latest-trtllm-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: $var:worker-string:workers-prefill-{{worker-hash}} + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + clusterIP: $ignore + clusterIPs: + - $ignore + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: system + port: 9090 + protocol: TCP + targetPort: system + selector: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-component-type: prefill + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-disaggregated-multi-component-workers + role: leader + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery + namespace: upgrade-latest-trtllm-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + generation: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-disaggregated-multi-component-workers + name: workers-frontend + namespace: upgrade-latest-trtllm-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: workers-frontend + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + nvidia.com/selector: workers-frontend + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-component-type: frontend + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-disaggregated-multi-component-workers + nvidia.com/metrics-enabled: 'true' + nvidia.com/selector: workers-frontend + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + env: + - name: DYNAMO_PORT + value: '8000' + - name: DYN_COMPONENT + value: frontend + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_HTTP_PORT + value: '8000' + - name: DYN_NAMESPACE + value: upgrade-latest-trtllm-disaggregated-multi-component-workers + - name: DYN_NAMESPACE_PREFIX + value: upgrade-latest-trtllm-disaggregated-multi-component-workers + - name: DYN_NAMESPACE_PREFIX_STRICT + value: 'true' + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-trtllm-disaggregated-multi-component + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.uid + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /live + port: http + scheme: HTTP + initialDelaySeconds: 15 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: main + ports: + - containerPort: 8000 + name: http + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: http + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 3 + resources: {} + startupProbe: + failureThreshold: 30 + httpGet: + path: /live + port: http + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 1000 + serviceAccount: workers-k8s-service-discovery + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory +status: + $ignore: true +--- +apiVersion: leaderworkerset.x-k8s.io/v1 +kind: LeaderWorkerSet +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + generation: $ignore + labels: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-disaggregated-multi-component-workers + nvidia.com/dynamo-sub-component-type: decode + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-decode-{{worker-hash}}-0 + namespace: upgrade-latest-trtllm-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: $var:worker-string:workers-decode-{{worker-hash}} + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + leaderWorkerTemplate: + leaderTemplate: + metadata: + labels: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-component-type: decode + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-disaggregated-multi-component-workers + nvidia.com/dynamo-sub-component-type: decode + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: leader + spec: + containers: + - args: + - "mkdir -p $HOME/.ssh && ls -la /ssh-pk/ && cp /ssh-pk/private.key $HOME/.ssh/id_rsa\ + \ && cp /ssh-pk/private.key.pub $HOME/.ssh/id_rsa.pub && cp /ssh-pk/private.key.pub\ + \ $HOME/.ssh/authorized_keys && chmod 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys\ + \ && chmod 644 $HOME/.ssh/id_rsa.pub && printf 'Host *\\nIdentityFile\ + \ '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort 2222\\n' > $HOME/.ssh/config\ + \ && TIMEOUT=300; START_TIME=$(date +%s); for worker in $(echo \"$LWS_LEADER_ADDRESS\"\ + \ | sed 's/\\./-1\\./'); do echo \"Waiting for DNS: $worker\"; until getent\ + \ hosts $worker >/dev/null 2>&1; do CURRENT_TIME=$(date +%s); if [ $((CURRENT_TIME\ + \ - START_TIME)) -gt $TIMEOUT ]; then echo \"ERROR: Timeout waiting for\ + \ DNS: $worker\"; exit 1; fi; echo \"DNS not ready for $worker, retrying...\"\ + ; sleep 2; done; echo \"\u2713 DNS resolved: $worker\"; done; echo \"\ + All workers DNS ready\" && mpirun $([ \"$(id -u)\" = \"0\" ] && echo --allow-run-as-root)\ + \ --oversubscribe -n 2 -H $(LWS_LEADER_ADDRESS),$(echo \"$LWS_LEADER_ADDRESS\"\ + \ | sed 's/\\./-1\\./') --mca pml ob1 --mca plm_rsh_args \"-p 2222 -o\ + \ StrictHostKeyChecking=no -i $HOME/.ssh/id_rsa\" -x CPATH -x CUDA_VISIBLE_DEVICES\ + \ -x DYNAMO_LEADER_ADDRESS -x DYNAMO_RANK -x DYN_COMPONENT -x DYN_DISCOVERY_BACKEND\ + \ -x DYN_FORWARDPASS_METRIC_PORT -x DYN_HEALTH_CHECK_ENABLED -x DYN_NAMESPACE\ + \ -x DYN_NAMESPACE_WORKER_SUFFIX -x DYN_PARENT_DGD_K8S_NAME -x DYN_PARENT_DGD_K8S_NAMESPACE\ + \ -x DYN_SYSTEM_ENABLED -x DYN_SYSTEM_PORT -x DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS\ + \ -x HF_DATASETS_CACHE -x HF_ENDPOINT -x HF_HOME -x HF_TOKEN -x HOME -x\ + \ HUGGING_FACE_HUB_TOKEN -x LD_LIBRARY_PATH -x MODEL_PATH -x NCCL_DEBUG\ + \ -x NCCL_IB_DISABLE -x NCCL_P2P_DISABLE -x NIXL_TELEMETRY_ENABLE -x NIXL_TELEMETRY_EXPORTER\ + \ -x NIXL_TELEMETRY_PROMETHEUS_PORT -x OMPI_MCA_orte_keep_fqdn_hostnames\ + \ -x PATH -x POD_NAME -x POD_NAMESPACE -x POD_UID -x PYTHONPATH -x TENSORRT_LLM_CACHE_DIR\ + \ -x TOKENIZERS_PARALLELISM -x TRANSFORMERS_CACHE -x TRITON_PTXAS_PATH\ + \ -x TRTLLM_USE_UCX_KVCACHE -x USER bash -c 'trtllm-llmapi-launch python3\ + \ -m dynamo.trtllm --model-path Qwen/Qwen3-0.6B --disaggregation-mode\ + \ decode'" + command: + - /bin/sh + - -c + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: decode + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-trtllm-disaggregated-multi-component-workers + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-trtllm-disaggregated-multi-component + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OMPI_MCA_orte_keep_fqdn_hostnames + value: '1' + image: nvcr.io/nvidia/ai-dynamo/trtllm-runtime:1.6.0 + livenessProbe: + failureThreshold: 3 + httpGet: + path: /live + port: system + periodSeconds: 5 + timeoutSeconds: 4 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: system + periodSeconds: 10 + timeoutSeconds: 4 + resources: + limits: + nvidia.com/gpu: '1' + startupProbe: + failureThreshold: 720 + httpGet: + path: /live + port: system + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /ssh-pk + name: upgrade-mpi-ssh + readOnly: true + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - name: upgrade-mpi-ssh + secret: + defaultMode: 420 + secretName: upgrade-mpi-ssh + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + restartPolicy: RecreateGroupOnPodRestart + size: 2 + workerTemplate: + metadata: + labels: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-component-type: decode + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-disaggregated-multi-component-workers + nvidia.com/dynamo-sub-component-type: decode + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: worker + spec: + containers: + - args: + - mkdir -p $HOME/.ssh $HOME/.ssh/host_keys $HOME/.ssh/run && ls -la /ssh-pk/ + && cp /ssh-pk/private.key $HOME/.ssh/id_rsa && cp /ssh-pk/private.key.pub + $HOME/.ssh/id_rsa.pub && cp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys + && chmod 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys && chmod 644 + $HOME/.ssh/id_rsa.pub && printf 'Host *\nIdentityFile '$HOME'/.ssh/id_rsa\nStrictHostKeyChecking + no\nPort 2222\n' > $HOME/.ssh/config && ssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key + -N '' && ssh-keygen -t ecdsa -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key + -N '' && ssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key + -N '' && printf 'Port 2222\nHostKey '$HOME'/.ssh/host_keys/ssh_host_rsa_key\nHostKey + '$HOME'/.ssh/host_keys/ssh_host_ecdsa_key\nHostKey '$HOME'/.ssh/host_keys/ssh_host_ed25519_key\nPidFile + '$HOME'/.ssh/run/sshd.pid\nStrictModes no\nPermitRootLogin yes\nPasswordAuthentication + no\nPubkeyAuthentication yes\nAuthorizedKeysFile '$HOME'/.ssh/authorized_keys\n' + > $HOME/.ssh/sshd_config && /usr/sbin/sshd -D -f $HOME/.ssh/sshd_config + command: + - /bin/sh + - -c + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: decode + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-trtllm-disaggregated-multi-component-workers + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-trtllm-disaggregated-multi-component + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OMPI_MCA_orte_keep_fqdn_hostnames + value: '1' + image: nvcr.io/nvidia/ai-dynamo/trtllm-runtime:1.6.0 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 10 + initialDelaySeconds: 20 + periodSeconds: 20 + tcpSocket: + port: 2222 + timeoutSeconds: 5 + resources: + limits: + nvidia.com/gpu: '1' + volumeMounts: + - mountPath: /ssh-pk + name: upgrade-mpi-ssh + readOnly: true + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - name: upgrade-mpi-ssh + secret: + defaultMode: 420 + secretName: upgrade-mpi-ssh + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + networkConfig: + subdomainPolicy: Shared + replicas: 2 + rolloutStrategy: + rollingUpdateConfiguration: + maxSurge: 0 + maxUnavailable: 1 + partition: 0 + type: RollingUpdate + startupPolicy: LeaderCreated +--- +apiVersion: leaderworkerset.x-k8s.io/v1 +kind: LeaderWorkerSet +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + generation: $ignore + labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-disaggregated-multi-component-workers + nvidia.com/dynamo-sub-component-type: prefill + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-prefill-{{worker-hash}}-0 + namespace: upgrade-latest-trtllm-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: $var:worker-string:workers-prefill-{{worker-hash}} + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + leaderWorkerTemplate: + leaderTemplate: + metadata: + labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-component-type: prefill + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-disaggregated-multi-component-workers + nvidia.com/dynamo-sub-component-type: prefill + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: leader + spec: + containers: + - args: + - "mkdir -p $HOME/.ssh && ls -la /ssh-pk/ && cp /ssh-pk/private.key $HOME/.ssh/id_rsa\ + \ && cp /ssh-pk/private.key.pub $HOME/.ssh/id_rsa.pub && cp /ssh-pk/private.key.pub\ + \ $HOME/.ssh/authorized_keys && chmod 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys\ + \ && chmod 644 $HOME/.ssh/id_rsa.pub && printf 'Host *\\nIdentityFile\ + \ '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort 2222\\n' > $HOME/.ssh/config\ + \ && TIMEOUT=300; START_TIME=$(date +%s); for worker in $(echo \"$LWS_LEADER_ADDRESS\"\ + \ | sed 's/\\./-1\\./'); do echo \"Waiting for DNS: $worker\"; until getent\ + \ hosts $worker >/dev/null 2>&1; do CURRENT_TIME=$(date +%s); if [ $((CURRENT_TIME\ + \ - START_TIME)) -gt $TIMEOUT ]; then echo \"ERROR: Timeout waiting for\ + \ DNS: $worker\"; exit 1; fi; echo \"DNS not ready for $worker, retrying...\"\ + ; sleep 2; done; echo \"\u2713 DNS resolved: $worker\"; done; echo \"\ + All workers DNS ready\" && mpirun $([ \"$(id -u)\" = \"0\" ] && echo --allow-run-as-root)\ + \ --oversubscribe -n 2 -H $(LWS_LEADER_ADDRESS),$(echo \"$LWS_LEADER_ADDRESS\"\ + \ | sed 's/\\./-1\\./') --mca pml ob1 --mca plm_rsh_args \"-p 2222 -o\ + \ StrictHostKeyChecking=no -i $HOME/.ssh/id_rsa\" -x CPATH -x CUDA_VISIBLE_DEVICES\ + \ -x DYNAMO_LEADER_ADDRESS -x DYNAMO_RANK -x DYN_COMPONENT -x DYN_DISCOVERY_BACKEND\ + \ -x DYN_FORWARDPASS_METRIC_PORT -x DYN_HEALTH_CHECK_ENABLED -x DYN_NAMESPACE\ + \ -x DYN_NAMESPACE_WORKER_SUFFIX -x DYN_PARENT_DGD_K8S_NAME -x DYN_PARENT_DGD_K8S_NAMESPACE\ + \ -x DYN_SYSTEM_ENABLED -x DYN_SYSTEM_PORT -x DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS\ + \ -x HF_DATASETS_CACHE -x HF_ENDPOINT -x HF_HOME -x HF_TOKEN -x HOME -x\ + \ HUGGING_FACE_HUB_TOKEN -x LD_LIBRARY_PATH -x MODEL_PATH -x NCCL_DEBUG\ + \ -x NCCL_IB_DISABLE -x NCCL_P2P_DISABLE -x NIXL_TELEMETRY_ENABLE -x NIXL_TELEMETRY_EXPORTER\ + \ -x NIXL_TELEMETRY_PROMETHEUS_PORT -x OMPI_MCA_orte_keep_fqdn_hostnames\ + \ -x PATH -x POD_NAME -x POD_NAMESPACE -x POD_UID -x PYTHONPATH -x TENSORRT_LLM_CACHE_DIR\ + \ -x TOKENIZERS_PARALLELISM -x TRANSFORMERS_CACHE -x TRITON_PTXAS_PATH\ + \ -x TRTLLM_USE_UCX_KVCACHE -x USER bash -c 'trtllm-llmapi-launch python3\ + \ -m dynamo.trtllm --model-path Qwen/Qwen3-0.6B --disaggregation-mode\ + \ prefill'" + command: + - /bin/sh + - -c + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: prefill + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-trtllm-disaggregated-multi-component-workers + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-trtllm-disaggregated-multi-component + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OMPI_MCA_orte_keep_fqdn_hostnames + value: '1' + image: nvcr.io/nvidia/ai-dynamo/trtllm-runtime:1.6.0 + livenessProbe: + failureThreshold: 3 + httpGet: + path: /live + port: system + periodSeconds: 5 + timeoutSeconds: 4 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: system + periodSeconds: 10 + timeoutSeconds: 4 + resources: + limits: + nvidia.com/gpu: '1' + startupProbe: + failureThreshold: 720 + httpGet: + path: /live + port: system + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /ssh-pk + name: upgrade-mpi-ssh + readOnly: true + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - name: upgrade-mpi-ssh + secret: + defaultMode: 420 + secretName: upgrade-mpi-ssh + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + restartPolicy: RecreateGroupOnPodRestart + size: 2 + workerTemplate: + metadata: + labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-component-type: prefill + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-disaggregated-multi-component-workers + nvidia.com/dynamo-sub-component-type: prefill + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: worker + spec: + containers: + - args: + - mkdir -p $HOME/.ssh $HOME/.ssh/host_keys $HOME/.ssh/run && ls -la /ssh-pk/ + && cp /ssh-pk/private.key $HOME/.ssh/id_rsa && cp /ssh-pk/private.key.pub + $HOME/.ssh/id_rsa.pub && cp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys + && chmod 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys && chmod 644 + $HOME/.ssh/id_rsa.pub && printf 'Host *\nIdentityFile '$HOME'/.ssh/id_rsa\nStrictHostKeyChecking + no\nPort 2222\n' > $HOME/.ssh/config && ssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key + -N '' && ssh-keygen -t ecdsa -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key + -N '' && ssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key + -N '' && printf 'Port 2222\nHostKey '$HOME'/.ssh/host_keys/ssh_host_rsa_key\nHostKey + '$HOME'/.ssh/host_keys/ssh_host_ecdsa_key\nHostKey '$HOME'/.ssh/host_keys/ssh_host_ed25519_key\nPidFile + '$HOME'/.ssh/run/sshd.pid\nStrictModes no\nPermitRootLogin yes\nPasswordAuthentication + no\nPubkeyAuthentication yes\nAuthorizedKeysFile '$HOME'/.ssh/authorized_keys\n' + > $HOME/.ssh/sshd_config && /usr/sbin/sshd -D -f $HOME/.ssh/sshd_config + command: + - /bin/sh + - -c + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: prefill + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-trtllm-disaggregated-multi-component-workers + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-trtllm-disaggregated-multi-component + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OMPI_MCA_orte_keep_fqdn_hostnames + value: '1' + image: nvcr.io/nvidia/ai-dynamo/trtllm-runtime:1.6.0 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 10 + initialDelaySeconds: 20 + periodSeconds: 20 + tcpSocket: + port: 2222 + timeoutSeconds: 5 + resources: + limits: + nvidia.com/gpu: '1' + volumeMounts: + - mountPath: /ssh-pk + name: upgrade-mpi-ssh + readOnly: true + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - name: upgrade-mpi-ssh + secret: + defaultMode: 420 + secretName: upgrade-mpi-ssh + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + networkConfig: + subdomainPolicy: Shared + replicas: 2 + rolloutStrategy: + rollingUpdateConfiguration: + maxSurge: 0 + maxUnavailable: 1 + partition: 0 + type: RollingUpdate + startupPolicy: LeaderCreated +--- +apiVersion: nvidia.com/v1beta1 +kind: DynamoComponentDeployment +metadata: + annotations: + nvidia.com/dcd-spec: '{"version":1,"spec":{"resources":{"limits":{"gpu":"1"}},"extraPodSpec":{"mainContainer":{"name":"","resources":{}}}},"rawJSON":[{"path":"serviceName","nil":true}]}' + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + labels: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-disaggregated-multi-component-workers + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-decode-{{worker-hash}} + namespace: upgrade-latest-trtllm-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: trtllm + multinode: + nodeCount: 2 + name: decode + podTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + labels: + nvidia.com/dynamo-worker-hash: $var:worker-hash + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - decode + command: + - python3 + - -m + - dynamo.trtllm + image: nvcr.io/nvidia/ai-dynamo/trtllm-runtime:1.6.0 + name: main + resources: + limits: + nvidia.com/gpu: '1' + replicas: 2 + type: decode +status: + component: + $strict: false + componentKind: LeaderWorkerSet + componentNames: + - $var:worker-string:workers-decode-{{worker-hash}}-0 + runtimeNamespace: $var:worker-string:upgrade-latest-trtllm-disaggregated-multi-component-workers-{{worker-hash}} + servedModelName: Qwen/Qwen3-0.6B + conditions: + - lastTransitionTime: $ignore + message: $ignore + reason: $ignore + status: 'False' + type: Available + - lastTransitionTime: $ignore + message: DynamoComponent is not ready + reason: ComponentNotReady + status: 'False' + type: DynamoComponentReady + observedGeneration: 1 +--- +apiVersion: nvidia.com/v1beta1 +kind: DynamoComponentDeployment +metadata: + annotations: + nvidia.com/dcd-spec: '{"version":1,"spec":{},"rawJSON":[{"path":"serviceName","nil":true}]}' + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-disaggregated-multi-component-workers + name: workers-frontend + namespace: upgrade-latest-trtllm-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: trtllm + name: frontend + podTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + name: main + replicas: 1 + type: frontend +status: + component: + $strict: false + componentKind: Deployment + componentNames: + - workers-frontend + runtimeNamespace: upgrade-latest-trtllm-disaggregated-multi-component-workers + conditions: + - lastTransitionTime: $ignore + message: $ignore + reason: $ignore + status: 'False' + type: Available + - lastTransitionTime: $ignore + message: DynamoComponent is not ready + reason: ComponentNotReady + status: 'False' + type: DynamoComponentReady + observedGeneration: 1 +--- +apiVersion: nvidia.com/v1beta1 +kind: DynamoComponentDeployment +metadata: + annotations: + nvidia.com/dcd-spec: '{"version":1,"spec":{"resources":{"limits":{"gpu":"1"}},"extraPodSpec":{"mainContainer":{"name":"","resources":{}}}},"rawJSON":[{"path":"serviceName","nil":true}]}' + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-disaggregated-multi-component-workers + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-prefill-{{worker-hash}} + namespace: upgrade-latest-trtllm-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: trtllm + multinode: + nodeCount: 2 + name: prefill + podTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + labels: + nvidia.com/dynamo-worker-hash: $var:worker-hash + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - prefill + command: + - python3 + - -m + - dynamo.trtllm + image: nvcr.io/nvidia/ai-dynamo/trtllm-runtime:1.6.0 + name: main + resources: + limits: + nvidia.com/gpu: '1' + replicas: 2 + type: prefill +status: + component: + $strict: false + componentKind: LeaderWorkerSet + componentNames: + - $var:worker-string:workers-prefill-{{worker-hash}}-0 + runtimeNamespace: $var:worker-string:upgrade-latest-trtllm-disaggregated-multi-component-workers-{{worker-hash}} + servedModelName: Qwen/Qwen3-0.6B + conditions: + - lastTransitionTime: $ignore + message: $ignore + reason: $ignore + status: 'False' + type: Available + - lastTransitionTime: $ignore + message: DynamoComponent is not ready + reason: ComponentNotReady + status: 'False' + type: DynamoComponentReady + observedGeneration: 1 +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: 0d454cc8266ac38ed6453ee2ef3cf6d9dd237faebc63af22f67ca911e106796c + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery-role + namespace: upgrade-latest-trtllm-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +rules: +- apiGroups: + - '' + resources: + - endpoints + - pods + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - nvidia.com + resources: + - dynamoworkermetadatas + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: 6a08b189e7ede8a3324ae46c53b1cf0cea2de3619be51d407088cfc68520a4b3 + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery-binding + namespace: upgrade-latest-trtllm-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: workers-k8s-service-discovery-role +subjects: +- kind: ServiceAccount + name: workers-k8s-service-discovery + namespace: upgrade-latest-trtllm-disaggregated-multi-component diff --git a/deploy/operator/internal/controller/testdata/upgrade/latest/trtllm/disaggregated/multi/disaggregatedset/input.yaml b/deploy/operator/internal/controller/testdata/upgrade/latest/trtllm/disaggregated/multi/disaggregatedset/input.yaml new file mode 100644 index 000000000000..e0443fd368ac --- /dev/null +++ b/deploy/operator/internal/controller/testdata/upgrade/latest/trtllm/disaggregated/multi/disaggregatedset/input.yaml @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + name: workers + namespace: upgrade-latest-trtllm-disaggregated-multi-disaggregatedset + annotations: + nvidia.com/enable-grove: 'false' + nvidia.com/dynamo-operator-origin-version: 0.0.0 + nvidia.com/enable-disaggregatedset: 'true' +spec: + backendFramework: trtllm + components: + - name: frontend + type: frontend + replicas: 1 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + command: + - python3 + - -m + - dynamo.frontend + - name: prefill + type: prefill + replicas: 2 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/trtllm-runtime:1.6.0 + command: + - python3 + - -m + - dynamo.trtllm + args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - prefill + resources: + limits: + nvidia.com/gpu: '1' + multinode: + nodeCount: 2 + - name: decode + type: decode + replicas: 2 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/trtllm-runtime:1.6.0 + command: + - python3 + - -m + - dynamo.trtllm + args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - decode + resources: + limits: + nvidia.com/gpu: '1' + multinode: + nodeCount: 2 diff --git a/deploy/operator/internal/controller/testdata/upgrade/latest/trtllm/disaggregated/multi/disaggregatedset/output.yaml b/deploy/operator/internal/controller/testdata/upgrade/latest/trtllm/disaggregated/multi/disaggregatedset/output.yaml new file mode 100644 index 000000000000..131b7cfb7bea --- /dev/null +++ b/deploy/operator/internal/controller/testdata/upgrade/latest/trtllm/disaggregated/multi/disaggregatedset/output.yaml @@ -0,0 +1,1208 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + annotations: + nvidia.com/current-worker-hash-v2: $var:worker-hash + nvidia.com/dynamo-operator-origin-version: $var:operator-version + nvidia.com/enable-disaggregatedset: 'true' + nvidia.com/enable-grove: 'false' + nvidia.com/workload-provider: disaggregatedset + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + name: workers + namespace: upgrade-latest-trtllm-disaggregated-multi-disaggregatedset + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: trtllm + components: + - name: frontend + podTemplate: + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + name: main + replicas: 1 + type: frontend + - multinode: + nodeCount: 2 + name: prefill + podTemplate: + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - prefill + command: + - python3 + - -m + - dynamo.trtllm + image: nvcr.io/nvidia/ai-dynamo/trtllm-runtime:1.6.0 + name: main + resources: + limits: + nvidia.com/gpu: '1' + replicas: 2 + type: prefill + - multinode: + nodeCount: 2 + name: decode + podTemplate: + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - decode + command: + - python3 + - -m + - dynamo.trtllm + image: nvcr.io/nvidia/ai-dynamo/trtllm-runtime:1.6.0 + name: main + resources: + limits: + nvidia.com/gpu: '1' + replicas: 2 + type: decode +status: + components: + decode: + $strict: false + componentKind: LeaderWorkerSet + frontend: + $strict: false + componentKind: Deployment + componentNames: + - workers-frontend + runtimeNamespace: upgrade-latest-trtllm-disaggregated-multi-disa-3a1137bc0fc6a3b4 + prefill: + $strict: false + componentKind: LeaderWorkerSet + conditions: + - lastTransitionTime: $ignore + message: DisaggregatedSet pathway selected + observedGeneration: 1 + reason: Selected + status: 'True' + type: DisaggregatedSetEligible + - lastTransitionTime: $ignore + message: 'Resources not ready: workers: decode role "decode" slice 0 has 0 target + LeaderWorkerSets, expected 1; prefill role "prefill" slice 0 has 0 target LeaderWorkerSets, + expected 1; workers-frontend: Component deployment not ready - Available condition + not true' + observedGeneration: 1 + reason: some_resources_are_not_ready + status: 'False' + type: Ready + observedGeneration: 1 + state: pending +--- +apiVersion: v1 +data: + wait-for-leader.py: "import socket, time, json, ssl, urllib.request, os\n\nSA =\ + \ \"/var/run/secrets/kubernetes.io/serviceaccount\"\nhost = os.environ[\"LEADER_HOST\"\ + ]\nport = int(os.environ[\"LEADER_PORT\"])\n\ndef _k8s_ctx():\n return ssl.create_default_context(cafile=f\"\ + {SA}/ca.crt\")\n\ndef _k8s_headers():\n token = open(f\"{SA}/token\").read()\n\ + \ return {\"Authorization\": f\"Bearer {token}\"}\n\ndef _k8s_api():\n ns\ + \ = open(f\"{SA}/namespace\").read()\n return f\"https://kubernetes.default.svc/api/v1/namespaces/{ns}/pods\"\ + \n\ndef resolve_leader_ip():\n # gethostbyname is IPv4-only; fall back to IPv6\ + \ for IPv6-only clusters.\n try:\n return socket.gethostbyname(host)\n\ + \ except socket.gaierror:\n return socket.getaddrinfo(host, None, socket.AF_INET6,\ + \ socket.SOCK_STREAM)[0][4][0]\n\ndef leader_pod_is_healthy():\n try:\n \ + \ ip = resolve_leader_ip()\n except socket.gaierror:\n return False,\ + \ \"DNS resolution failed\", None, None\n try:\n req = urllib.request.Request(\n\ + \ f\"{_k8s_api()}?fieldSelector=status.podIP={ip}\",\n headers=_k8s_headers(),\n\ + \ )\n resp = json.loads(urllib.request.urlopen(req, context=_k8s_ctx(),\ + \ timeout=5).read())\n pods = resp.get(\"items\", [])\n if not pods:\n\ + \ return False, f\"no pod found with IP {ip}\", None, ip\n pod\ + \ = pods[0]\n name = pod[\"metadata\"].get(\"name\", \"unknown\")\n \ + \ uid = pod[\"metadata\"].get(\"uid\", \"unknown\")\n phase = pod.get(\"\ + status\", {}).get(\"phase\")\n deletion_ts = pod[\"metadata\"].get(\"deletionTimestamp\"\ + )\n info = f\"ip={ip} pod={name} uid={uid} phase={phase} deletionTimestamp={deletion_ts}\"\ + \n if deletion_ts:\n return False, f\"pod {name} is terminating\"\ + , info, ip\n if phase != \"Running\":\n return False, f\"pod\ + \ {name} phase is {phase}\", info, ip\n return True, \"\", info, ip\n \ + \ except Exception as e:\n # Fall back to TCP-only when the API is unavailable\ + \ (e.g. 403 no RBAC)\n return True, f\"K8s API unavailable ({e}), falling\ + \ back to TCP\", f\"ip={ip}\", ip\n\nprint(f\"Waiting for leader master port at\ + \ {host}:{port}...\", flush=True)\ntime.sleep(5)\nstart = time.monotonic()\nlast_status\ + \ = start\nlast_err = \"\"\nwhile True:\n healthy, reason, pod_info, leader_ip\ + \ = leader_pod_is_healthy()\n if healthy:\n try:\n s = socket.create_connection((leader_ip,\ + \ port), timeout=2)\n s.close()\n elapsed = time.monotonic()\ + \ - start\n print(f\"Leader master port ready (waited {elapsed:.1f}s)\ + \ [{pod_info}]\", flush=True)\n break\n except Exception as\ + \ e:\n last_err = f\"tcp: {type(e).__name__}: {e} [{pod_info}]\"\n\ + \ else:\n last_err = f\"{reason} [{pod_info}]\" if pod_info else reason\n\ + \ now = time.monotonic()\n if now - last_status >= 30:\n print(f\"\ + Still waiting for {host}:{port}... ({now - start:.0f}s elapsed, last: {last_err})\"\ + , flush=True)\n last_status = now\n time.sleep(5)\n" +kind: ConfigMap +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: a615bfc016f57bef619ea5c67d88da24755349bb07efff9da0b2c79f18d6d076 + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-graph-deployment-name: workers + name: workers-wait-leader-script + namespace: upgrade-latest-trtllm-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + nvidia.com/dynamo-managed-service-metadata: '{"labels":["nvidia.com/dynamo-component","nvidia.com/dynamo-discovery-backend","nvidia.com/dynamo-discovery-enabled","nvidia.com/dynamo-graph-deployment-name","nvidia.com/dynamo-namespace","nvidia.com/dynamo-worker-hash"],"annotations":["nvidia.com/dynamo-operator-origin-version"]}' + nvidia.com/dynamo-operator-origin-version: $var:operator-version + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-disaggregated-multi-disa-3a1137bc0fc6a3b4 + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-decode-{{worker-hash}} + namespace: upgrade-latest-trtllm-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + clusterIP: $ignore + clusterIPs: + - $ignore + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: system + port: 9090 + protocol: TCP + targetPort: system + selector: + disaggregatedset.x-k8s.io/name: workers + disaggregatedset.x-k8s.io/revision: $var:ds-revision + disaggregatedset.x-k8s.io/role: decode + role: leader + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + nvidia.com/dynamo-managed-service-metadata: '{"labels":["nvidia.com/dynamo-component","nvidia.com/dynamo-discovery-backend","nvidia.com/dynamo-discovery-enabled","nvidia.com/dynamo-graph-deployment-name","nvidia.com/dynamo-namespace"],"annotations":["nvidia.com/dynamo-operator-origin-version"]}' + nvidia.com/dynamo-operator-origin-version: $var:operator-version + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-disaggregated-multi-disa-3a1137bc0fc6a3b4 + name: workers-frontend + namespace: upgrade-latest-trtllm-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + clusterIP: $ignore + clusterIPs: + - $ignore + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: http + port: 8000 + protocol: TCP + targetPort: http + selector: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-component-type: frontend + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-disaggregated-multi-disa-3a1137bc0fc6a3b4 + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + nvidia.com/dynamo-managed-service-metadata: '{"labels":["nvidia.com/dynamo-component","nvidia.com/dynamo-discovery-backend","nvidia.com/dynamo-discovery-enabled","nvidia.com/dynamo-graph-deployment-name","nvidia.com/dynamo-namespace","nvidia.com/dynamo-worker-hash"],"annotations":["nvidia.com/dynamo-operator-origin-version"]}' + nvidia.com/dynamo-operator-origin-version: $var:operator-version + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-disaggregated-multi-disa-3a1137bc0fc6a3b4 + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-prefill-{{worker-hash}} + namespace: upgrade-latest-trtllm-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + clusterIP: $ignore + clusterIPs: + - $ignore + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: system + port: 9090 + protocol: TCP + targetPort: system + selector: + disaggregatedset.x-k8s.io/name: workers + disaggregatedset.x-k8s.io/revision: $var:ds-revision + disaggregatedset.x-k8s.io/role: prefill + role: leader + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery + namespace: upgrade-latest-trtllm-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + generation: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-disaggregated-multi-disa-3a1137bc0fc6a3b4 + name: workers-frontend + namespace: upgrade-latest-trtllm-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: workers-frontend + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + nvidia.com/selector: workers-frontend + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-component-type: frontend + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-disaggregated-multi-disa-3a1137bc0fc6a3b4 + nvidia.com/metrics-enabled: 'true' + nvidia.com/selector: workers-frontend + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + env: + - name: DYNAMO_PORT + value: '8000' + - name: DYN_COMPONENT + value: frontend + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_HTTP_PORT + value: '8000' + - name: DYN_NAMESPACE + value: upgrade-latest-trtllm-disaggregated-multi-disa-3a1137bc0fc6a3b4 + - name: DYN_NAMESPACE_PREFIX + value: upgrade-latest-trtllm-disaggregated-multi-disa-3a1137bc0fc6a3b4 + - name: DYN_NAMESPACE_PREFIX_STRICT + value: 'true' + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-trtllm-disaggregated-multi-disaggregatedset + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.uid + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /live + port: http + scheme: HTTP + initialDelaySeconds: 15 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: main + ports: + - containerPort: 8000 + name: http + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: http + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 3 + resources: {} + startupProbe: + failureThreshold: 30 + httpGet: + path: /live + port: http + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 1000 + serviceAccount: workers-k8s-service-discovery + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory +status: + $ignore: true +--- +apiVersion: disaggregatedset.x-k8s.io/v1 +kind: DisaggregatedSet +metadata: + creationTimestamp: $ignore + generation: $ignore + labels: + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/selector: workers + name: workers + namespace: upgrade-latest-trtllm-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + roles: + - name: prefill + spec: + leaderWorkerTemplate: + leaderTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-component-type: prefill + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-disaggregated-multi-disa-3a1137bc0fc6a3b4 + nvidia.com/dynamo-sub-component-type: prefill + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: leader + spec: + containers: + - args: + - "mkdir -p $HOME/.ssh && ls -la /ssh-pk/ && cp /ssh-pk/private.key\ + \ $HOME/.ssh/id_rsa && cp /ssh-pk/private.key.pub $HOME/.ssh/id_rsa.pub\ + \ && cp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys && chmod\ + \ 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys && chmod 644 $HOME/.ssh/id_rsa.pub\ + \ && printf 'Host *\\nIdentityFile '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking\ + \ no\\nPort 2222\\n' > $HOME/.ssh/config && TIMEOUT=300; START_TIME=$(date\ + \ +%s); for worker in $(echo \"$LWS_LEADER_ADDRESS\" | sed 's/\\./-1\\\ + ./'); do echo \"Waiting for DNS: $worker\"; until getent hosts $worker\ + \ >/dev/null 2>&1; do CURRENT_TIME=$(date +%s); if [ $((CURRENT_TIME\ + \ - START_TIME)) -gt $TIMEOUT ]; then echo \"ERROR: Timeout waiting\ + \ for DNS: $worker\"; exit 1; fi; echo \"DNS not ready for $worker,\ + \ retrying...\"; sleep 2; done; echo \"\u2713 DNS resolved: $worker\"\ + ; done; echo \"All workers DNS ready\" && mpirun $([ \"$(id -u)\"\ + \ = \"0\" ] && echo --allow-run-as-root) --oversubscribe -n 2 -H $(LWS_LEADER_ADDRESS),$(echo\ + \ \"$LWS_LEADER_ADDRESS\" | sed 's/\\./-1\\./') --mca pml ob1 --mca\ + \ plm_rsh_args \"-p 2222 -o StrictHostKeyChecking=no -i $HOME/.ssh/id_rsa\"\ + \ -x CPATH -x CUDA_VISIBLE_DEVICES -x DYNAMO_LEADER_ADDRESS -x DYNAMO_RANK\ + \ -x DYN_COMPONENT -x DYN_DISCOVERY_BACKEND -x DYN_FORWARDPASS_METRIC_PORT\ + \ -x DYN_HEALTH_CHECK_ENABLED -x DYN_NAMESPACE -x DYN_NAMESPACE_WORKER_SUFFIX\ + \ -x DYN_PARENT_DGD_K8S_NAME -x DYN_PARENT_DGD_K8S_NAMESPACE -x DYN_SYSTEM_ENABLED\ + \ -x DYN_SYSTEM_PORT -x DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS -x HF_DATASETS_CACHE\ + \ -x HF_ENDPOINT -x HF_HOME -x HF_TOKEN -x HOME -x HUGGING_FACE_HUB_TOKEN\ + \ -x LD_LIBRARY_PATH -x MODEL_PATH -x NCCL_DEBUG -x NCCL_IB_DISABLE\ + \ -x NCCL_P2P_DISABLE -x NIXL_TELEMETRY_ENABLE -x NIXL_TELEMETRY_EXPORTER\ + \ -x NIXL_TELEMETRY_PROMETHEUS_PORT -x OMPI_MCA_orte_keep_fqdn_hostnames\ + \ -x PATH -x POD_NAME -x POD_NAMESPACE -x POD_UID -x PYTHONPATH -x\ + \ TENSORRT_LLM_CACHE_DIR -x TOKENIZERS_PARALLELISM -x TRANSFORMERS_CACHE\ + \ -x TRITON_PTXAS_PATH -x TRTLLM_USE_UCX_KVCACHE -x USER bash -c 'trtllm-llmapi-launch\ + \ python3 -m dynamo.trtllm --model-path Qwen/Qwen3-0.6B --disaggregation-mode\ + \ prefill'" + command: + - /bin/sh + - -c + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: prefill + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-trtllm-disaggregated-multi-disa-3a1137bc0fc6a3b4 + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-trtllm-disaggregated-multi-disaggregatedset + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OMPI_MCA_orte_keep_fqdn_hostnames + value: '1' + image: nvcr.io/nvidia/ai-dynamo/trtllm-runtime:1.6.0 + livenessProbe: + failureThreshold: 3 + httpGet: + path: /live + port: system + periodSeconds: 5 + timeoutSeconds: 4 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: system + periodSeconds: 10 + timeoutSeconds: 4 + resources: + limits: + nvidia.com/gpu: '1' + startupProbe: + failureThreshold: 720 + httpGet: + path: /live + port: system + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /ssh-pk + name: upgrade-mpi-ssh + readOnly: true + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - name: upgrade-mpi-ssh + secret: + defaultMode: 420 + secretName: upgrade-mpi-ssh + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + restartPolicy: RecreateGroupOnPodRestart + size: 2 + workerTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-component-type: prefill + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-disaggregated-multi-disa-3a1137bc0fc6a3b4 + nvidia.com/dynamo-sub-component-type: prefill + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: worker + spec: + containers: + - args: + - mkdir -p $HOME/.ssh $HOME/.ssh/host_keys $HOME/.ssh/run && ls -la + /ssh-pk/ && cp /ssh-pk/private.key $HOME/.ssh/id_rsa && cp /ssh-pk/private.key.pub + $HOME/.ssh/id_rsa.pub && cp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys + && chmod 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys && chmod + 644 $HOME/.ssh/id_rsa.pub && printf 'Host *\nIdentityFile '$HOME'/.ssh/id_rsa\nStrictHostKeyChecking + no\nPort 2222\n' > $HOME/.ssh/config && ssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key + -N '' && ssh-keygen -t ecdsa -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key + -N '' && ssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key + -N '' && printf 'Port 2222\nHostKey '$HOME'/.ssh/host_keys/ssh_host_rsa_key\nHostKey + '$HOME'/.ssh/host_keys/ssh_host_ecdsa_key\nHostKey '$HOME'/.ssh/host_keys/ssh_host_ed25519_key\nPidFile + '$HOME'/.ssh/run/sshd.pid\nStrictModes no\nPermitRootLogin yes\nPasswordAuthentication + no\nPubkeyAuthentication yes\nAuthorizedKeysFile '$HOME'/.ssh/authorized_keys\n' + > $HOME/.ssh/sshd_config && /usr/sbin/sshd -D -f $HOME/.ssh/sshd_config + command: + - /bin/sh + - -c + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: prefill + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-trtllm-disaggregated-multi-disa-3a1137bc0fc6a3b4 + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-trtllm-disaggregated-multi-disaggregatedset + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OMPI_MCA_orte_keep_fqdn_hostnames + value: '1' + image: nvcr.io/nvidia/ai-dynamo/trtllm-runtime:1.6.0 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 10 + initialDelaySeconds: 20 + periodSeconds: 20 + tcpSocket: + port: 2222 + timeoutSeconds: 5 + resources: + limits: + nvidia.com/gpu: '1' + volumeMounts: + - mountPath: /ssh-pk + name: upgrade-mpi-ssh + readOnly: true + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - name: upgrade-mpi-ssh + secret: + defaultMode: 420 + secretName: upgrade-mpi-ssh + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + replicas: 2 + rolloutStrategy: + type: RollingUpdate + startupPolicy: LeaderCreated + - name: decode + spec: + leaderWorkerTemplate: + leaderTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + labels: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-component-type: decode + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-disaggregated-multi-disa-3a1137bc0fc6a3b4 + nvidia.com/dynamo-sub-component-type: decode + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: leader + spec: + containers: + - args: + - "mkdir -p $HOME/.ssh && ls -la /ssh-pk/ && cp /ssh-pk/private.key\ + \ $HOME/.ssh/id_rsa && cp /ssh-pk/private.key.pub $HOME/.ssh/id_rsa.pub\ + \ && cp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys && chmod\ + \ 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys && chmod 644 $HOME/.ssh/id_rsa.pub\ + \ && printf 'Host *\\nIdentityFile '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking\ + \ no\\nPort 2222\\n' > $HOME/.ssh/config && TIMEOUT=300; START_TIME=$(date\ + \ +%s); for worker in $(echo \"$LWS_LEADER_ADDRESS\" | sed 's/\\./-1\\\ + ./'); do echo \"Waiting for DNS: $worker\"; until getent hosts $worker\ + \ >/dev/null 2>&1; do CURRENT_TIME=$(date +%s); if [ $((CURRENT_TIME\ + \ - START_TIME)) -gt $TIMEOUT ]; then echo \"ERROR: Timeout waiting\ + \ for DNS: $worker\"; exit 1; fi; echo \"DNS not ready for $worker,\ + \ retrying...\"; sleep 2; done; echo \"\u2713 DNS resolved: $worker\"\ + ; done; echo \"All workers DNS ready\" && mpirun $([ \"$(id -u)\"\ + \ = \"0\" ] && echo --allow-run-as-root) --oversubscribe -n 2 -H $(LWS_LEADER_ADDRESS),$(echo\ + \ \"$LWS_LEADER_ADDRESS\" | sed 's/\\./-1\\./') --mca pml ob1 --mca\ + \ plm_rsh_args \"-p 2222 -o StrictHostKeyChecking=no -i $HOME/.ssh/id_rsa\"\ + \ -x CPATH -x CUDA_VISIBLE_DEVICES -x DYNAMO_LEADER_ADDRESS -x DYNAMO_RANK\ + \ -x DYN_COMPONENT -x DYN_DISCOVERY_BACKEND -x DYN_FORWARDPASS_METRIC_PORT\ + \ -x DYN_HEALTH_CHECK_ENABLED -x DYN_NAMESPACE -x DYN_NAMESPACE_WORKER_SUFFIX\ + \ -x DYN_PARENT_DGD_K8S_NAME -x DYN_PARENT_DGD_K8S_NAMESPACE -x DYN_SYSTEM_ENABLED\ + \ -x DYN_SYSTEM_PORT -x DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS -x HF_DATASETS_CACHE\ + \ -x HF_ENDPOINT -x HF_HOME -x HF_TOKEN -x HOME -x HUGGING_FACE_HUB_TOKEN\ + \ -x LD_LIBRARY_PATH -x MODEL_PATH -x NCCL_DEBUG -x NCCL_IB_DISABLE\ + \ -x NCCL_P2P_DISABLE -x NIXL_TELEMETRY_ENABLE -x NIXL_TELEMETRY_EXPORTER\ + \ -x NIXL_TELEMETRY_PROMETHEUS_PORT -x OMPI_MCA_orte_keep_fqdn_hostnames\ + \ -x PATH -x POD_NAME -x POD_NAMESPACE -x POD_UID -x PYTHONPATH -x\ + \ TENSORRT_LLM_CACHE_DIR -x TOKENIZERS_PARALLELISM -x TRANSFORMERS_CACHE\ + \ -x TRITON_PTXAS_PATH -x TRTLLM_USE_UCX_KVCACHE -x USER bash -c 'trtllm-llmapi-launch\ + \ python3 -m dynamo.trtllm --model-path Qwen/Qwen3-0.6B --disaggregation-mode\ + \ decode'" + command: + - /bin/sh + - -c + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: decode + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-trtllm-disaggregated-multi-disa-3a1137bc0fc6a3b4 + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-trtllm-disaggregated-multi-disaggregatedset + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OMPI_MCA_orte_keep_fqdn_hostnames + value: '1' + image: nvcr.io/nvidia/ai-dynamo/trtllm-runtime:1.6.0 + livenessProbe: + failureThreshold: 3 + httpGet: + path: /live + port: system + periodSeconds: 5 + timeoutSeconds: 4 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: system + periodSeconds: 10 + timeoutSeconds: 4 + resources: + limits: + nvidia.com/gpu: '1' + startupProbe: + failureThreshold: 720 + httpGet: + path: /live + port: system + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /ssh-pk + name: upgrade-mpi-ssh + readOnly: true + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - name: upgrade-mpi-ssh + secret: + defaultMode: 420 + secretName: upgrade-mpi-ssh + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + restartPolicy: RecreateGroupOnPodRestart + size: 2 + workerTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + labels: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-component-type: decode + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-disaggregated-multi-disa-3a1137bc0fc6a3b4 + nvidia.com/dynamo-sub-component-type: decode + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: worker + spec: + containers: + - args: + - mkdir -p $HOME/.ssh $HOME/.ssh/host_keys $HOME/.ssh/run && ls -la + /ssh-pk/ && cp /ssh-pk/private.key $HOME/.ssh/id_rsa && cp /ssh-pk/private.key.pub + $HOME/.ssh/id_rsa.pub && cp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys + && chmod 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys && chmod + 644 $HOME/.ssh/id_rsa.pub && printf 'Host *\nIdentityFile '$HOME'/.ssh/id_rsa\nStrictHostKeyChecking + no\nPort 2222\n' > $HOME/.ssh/config && ssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key + -N '' && ssh-keygen -t ecdsa -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key + -N '' && ssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key + -N '' && printf 'Port 2222\nHostKey '$HOME'/.ssh/host_keys/ssh_host_rsa_key\nHostKey + '$HOME'/.ssh/host_keys/ssh_host_ecdsa_key\nHostKey '$HOME'/.ssh/host_keys/ssh_host_ed25519_key\nPidFile + '$HOME'/.ssh/run/sshd.pid\nStrictModes no\nPermitRootLogin yes\nPasswordAuthentication + no\nPubkeyAuthentication yes\nAuthorizedKeysFile '$HOME'/.ssh/authorized_keys\n' + > $HOME/.ssh/sshd_config && /usr/sbin/sshd -D -f $HOME/.ssh/sshd_config + command: + - /bin/sh + - -c + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: decode + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-trtllm-disaggregated-multi-disa-3a1137bc0fc6a3b4 + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-trtllm-disaggregated-multi-disaggregatedset + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OMPI_MCA_orte_keep_fqdn_hostnames + value: '1' + image: nvcr.io/nvidia/ai-dynamo/trtllm-runtime:1.6.0 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 10 + initialDelaySeconds: 20 + periodSeconds: 20 + tcpSocket: + port: 2222 + timeoutSeconds: 5 + resources: + limits: + nvidia.com/gpu: '1' + volumeMounts: + - mountPath: /ssh-pk + name: upgrade-mpi-ssh + readOnly: true + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - name: upgrade-mpi-ssh + secret: + defaultMode: 420 + secretName: upgrade-mpi-ssh + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + replicas: 2 + rolloutStrategy: + type: RollingUpdate + startupPolicy: LeaderCreated + slices: 1 +--- +apiVersion: nvidia.com/v1beta1 +kind: DynamoComponentDeployment +metadata: + annotations: + nvidia.com/dcd-spec: '{"version":1,"spec":{},"rawJSON":[{"path":"serviceName","nil":true}]}' + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-trtllm-disaggregated-multi-disa-3a1137bc0fc6a3b4 + name: workers-frontend + namespace: upgrade-latest-trtllm-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: trtllm + name: frontend + podTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + name: main + replicas: 1 + type: frontend +status: + component: + $strict: false + componentKind: Deployment + componentNames: + - workers-frontend + runtimeNamespace: upgrade-latest-trtllm-disaggregated-multi-disa-3a1137bc0fc6a3b4 + conditions: + - lastTransitionTime: $ignore + message: $ignore + reason: $ignore + status: 'False' + type: Available + - lastTransitionTime: $ignore + message: DynamoComponent is not ready + reason: ComponentNotReady + status: 'False' + type: DynamoComponentReady + observedGeneration: 1 +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: 0d454cc8266ac38ed6453ee2ef3cf6d9dd237faebc63af22f67ca911e106796c + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery-role + namespace: upgrade-latest-trtllm-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +rules: +- apiGroups: + - '' + resources: + - endpoints + - pods + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - nvidia.com + resources: + - dynamoworkermetadatas + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: 7577111207ed4bced0c134736fd2a00286dedd1e9627a57741affe1a6fd69a42 + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery-binding + namespace: upgrade-latest-trtllm-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: workers-k8s-service-discovery-role +subjects: +- kind: ServiceAccount + name: workers-k8s-service-discovery + namespace: upgrade-latest-trtllm-disaggregated-multi-disaggregatedset diff --git a/deploy/operator/internal/controller/testdata/upgrade/latest/vllm/aggregated/multi/component/input.yaml b/deploy/operator/internal/controller/testdata/upgrade/latest/vllm/aggregated/multi/component/input.yaml new file mode 100644 index 000000000000..7f973939a37e --- /dev/null +++ b/deploy/operator/internal/controller/testdata/upgrade/latest/vllm/aggregated/multi/component/input.yaml @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + name: workers + namespace: upgrade-latest-vllm-aggregated-multi-component + annotations: + nvidia.com/enable-grove: 'false' + nvidia.com/dynamo-operator-origin-version: 0.0.0 +spec: + backendFramework: vllm + components: + - name: frontend + type: frontend + replicas: 1 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + command: + - python3 + - -m + - dynamo.frontend + - name: worker + type: worker + replicas: 2 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.6.0 + command: + - python3 + - -m + - dynamo.vllm + args: + - --model-path + - Qwen/Qwen3-0.6B + resources: + limits: + nvidia.com/gpu: '1' + multinode: + nodeCount: 2 diff --git a/deploy/operator/internal/controller/testdata/upgrade/latest/vllm/aggregated/multi/component/output.yaml b/deploy/operator/internal/controller/testdata/upgrade/latest/vllm/aggregated/multi/component/output.yaml new file mode 100644 index 000000000000..9aafa77a0bbb --- /dev/null +++ b/deploy/operator/internal/controller/testdata/upgrade/latest/vllm/aggregated/multi/component/output.yaml @@ -0,0 +1,871 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + annotations: + nvidia.com/current-worker-hash-v2: $var:worker-hash + nvidia.com/dynamo-operator-origin-version: $var:operator-version + nvidia.com/enable-grove: 'false' + nvidia.com/workload-provider: component + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + name: workers + namespace: upgrade-latest-vllm-aggregated-multi-component + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: vllm + components: + - name: frontend + podTemplate: + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + name: main + replicas: 1 + type: frontend + - multinode: + nodeCount: 2 + name: worker + podTemplate: + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + command: + - python3 + - -m + - dynamo.vllm + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.6.0 + name: main + resources: + limits: + nvidia.com/gpu: '1' + replicas: 2 + type: worker +status: + components: + frontend: + $strict: false + componentKind: Deployment + componentNames: + - workers-frontend + runtimeNamespace: upgrade-latest-vllm-aggregated-multi-component-workers + worker: + $strict: false + componentKind: LeaderWorkerSet + componentNames: + - $var:worker-string:workers-worker-{{worker-hash}}-0 + runtimeNamespace: $var:worker-string:upgrade-latest-vllm-aggregated-multi-component-workers-{{worker-hash}} + servedModelName: Qwen/Qwen3-0.6B + conditions: + - lastTransitionTime: $ignore + message: '$var:worker-string:Resources not ready: workers-frontend: Component + deployment not ready - Available condition not true; workers-worker-{{worker-hash}}: + Component deployment not ready - Available condition not true' + observedGeneration: 1 + reason: some_resources_are_not_ready + status: 'False' + type: Ready + observedGeneration: 1 + state: pending +--- +apiVersion: v1 +data: + wait-for-leader.py: "import socket, time, json, ssl, urllib.request, os\n\nSA =\ + \ \"/var/run/secrets/kubernetes.io/serviceaccount\"\nhost = os.environ[\"LEADER_HOST\"\ + ]\nport = int(os.environ[\"LEADER_PORT\"])\n\ndef _k8s_ctx():\n return ssl.create_default_context(cafile=f\"\ + {SA}/ca.crt\")\n\ndef _k8s_headers():\n token = open(f\"{SA}/token\").read()\n\ + \ return {\"Authorization\": f\"Bearer {token}\"}\n\ndef _k8s_api():\n ns\ + \ = open(f\"{SA}/namespace\").read()\n return f\"https://kubernetes.default.svc/api/v1/namespaces/{ns}/pods\"\ + \n\ndef resolve_leader_ip():\n # gethostbyname is IPv4-only; fall back to IPv6\ + \ for IPv6-only clusters.\n try:\n return socket.gethostbyname(host)\n\ + \ except socket.gaierror:\n return socket.getaddrinfo(host, None, socket.AF_INET6,\ + \ socket.SOCK_STREAM)[0][4][0]\n\ndef leader_pod_is_healthy():\n try:\n \ + \ ip = resolve_leader_ip()\n except socket.gaierror:\n return False,\ + \ \"DNS resolution failed\", None, None\n try:\n req = urllib.request.Request(\n\ + \ f\"{_k8s_api()}?fieldSelector=status.podIP={ip}\",\n headers=_k8s_headers(),\n\ + \ )\n resp = json.loads(urllib.request.urlopen(req, context=_k8s_ctx(),\ + \ timeout=5).read())\n pods = resp.get(\"items\", [])\n if not pods:\n\ + \ return False, f\"no pod found with IP {ip}\", None, ip\n pod\ + \ = pods[0]\n name = pod[\"metadata\"].get(\"name\", \"unknown\")\n \ + \ uid = pod[\"metadata\"].get(\"uid\", \"unknown\")\n phase = pod.get(\"\ + status\", {}).get(\"phase\")\n deletion_ts = pod[\"metadata\"].get(\"deletionTimestamp\"\ + )\n info = f\"ip={ip} pod={name} uid={uid} phase={phase} deletionTimestamp={deletion_ts}\"\ + \n if deletion_ts:\n return False, f\"pod {name} is terminating\"\ + , info, ip\n if phase != \"Running\":\n return False, f\"pod\ + \ {name} phase is {phase}\", info, ip\n return True, \"\", info, ip\n \ + \ except Exception as e:\n # Fall back to TCP-only when the API is unavailable\ + \ (e.g. 403 no RBAC)\n return True, f\"K8s API unavailable ({e}), falling\ + \ back to TCP\", f\"ip={ip}\", ip\n\nprint(f\"Waiting for leader master port at\ + \ {host}:{port}...\", flush=True)\ntime.sleep(5)\nstart = time.monotonic()\nlast_status\ + \ = start\nlast_err = \"\"\nwhile True:\n healthy, reason, pod_info, leader_ip\ + \ = leader_pod_is_healthy()\n if healthy:\n try:\n s = socket.create_connection((leader_ip,\ + \ port), timeout=2)\n s.close()\n elapsed = time.monotonic()\ + \ - start\n print(f\"Leader master port ready (waited {elapsed:.1f}s)\ + \ [{pod_info}]\", flush=True)\n break\n except Exception as\ + \ e:\n last_err = f\"tcp: {type(e).__name__}: {e} [{pod_info}]\"\n\ + \ else:\n last_err = f\"{reason} [{pod_info}]\" if pod_info else reason\n\ + \ now = time.monotonic()\n if now - last_status >= 30:\n print(f\"\ + Still waiting for {host}:{port}... ({now - start:.0f}s elapsed, last: {last_err})\"\ + , flush=True)\n last_status = now\n time.sleep(5)\n" +kind: ConfigMap +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: a615bfc016f57bef619ea5c67d88da24755349bb07efff9da0b2c79f18d6d076 + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-graph-deployment-name: workers + name: workers-wait-leader-script + namespace: upgrade-latest-vllm-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-aggregated-multi-component-workers + name: workers-frontend + namespace: upgrade-latest-vllm-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: workers-frontend + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + clusterIP: $ignore + clusterIPs: + - $ignore + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: http + port: 8000 + protocol: TCP + targetPort: http + selector: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-component-type: frontend + nvidia.com/dynamo-namespace: upgrade-latest-vllm-aggregated-multi-component-workers + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-component: worker + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-aggregated-multi-component-workers + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-worker-{{worker-hash}} + namespace: upgrade-latest-vllm-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: $var:worker-string:workers-worker-{{worker-hash}} + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + clusterIP: $ignore + clusterIPs: + - $ignore + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: system + port: 9090 + protocol: TCP + targetPort: system + selector: + nvidia.com/dynamo-component: worker + nvidia.com/dynamo-component-type: worker + nvidia.com/dynamo-namespace: upgrade-latest-vllm-aggregated-multi-component-workers + role: leader + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery + namespace: upgrade-latest-vllm-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + generation: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-aggregated-multi-component-workers + name: workers-frontend + namespace: upgrade-latest-vllm-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: workers-frontend + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + nvidia.com/selector: workers-frontend + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-component-type: frontend + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-aggregated-multi-component-workers + nvidia.com/metrics-enabled: 'true' + nvidia.com/selector: workers-frontend + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + env: + - name: DYNAMO_PORT + value: '8000' + - name: DYN_COMPONENT + value: frontend + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_HTTP_PORT + value: '8000' + - name: DYN_NAMESPACE + value: upgrade-latest-vllm-aggregated-multi-component-workers + - name: DYN_NAMESPACE_PREFIX + value: upgrade-latest-vllm-aggregated-multi-component-workers + - name: DYN_NAMESPACE_PREFIX_STRICT + value: 'true' + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-vllm-aggregated-multi-component + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.uid + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /live + port: http + scheme: HTTP + initialDelaySeconds: 15 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: main + ports: + - containerPort: 8000 + name: http + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: http + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 3 + resources: {} + startupProbe: + failureThreshold: 30 + httpGet: + path: /live + port: http + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 1000 + serviceAccount: workers-k8s-service-discovery + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory +status: + $ignore: true +--- +apiVersion: leaderworkerset.x-k8s.io/v1 +kind: LeaderWorkerSet +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + generation: $ignore + labels: + nvidia.com/dynamo-component: worker + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-aggregated-multi-component-workers + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-worker-{{worker-hash}}-0 + namespace: upgrade-latest-vllm-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: $var:worker-string:workers-worker-{{worker-hash}} + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + leaderWorkerTemplate: + leaderTemplate: + metadata: + labels: + nvidia.com/dynamo-component: worker + nvidia.com/dynamo-component-type: worker + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-aggregated-multi-component-workers + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: leader + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + command: + - python3 + - -m + - dynamo.vllm + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: worker + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-vllm-aggregated-multi-component-workers + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-vllm-aggregated-multi-component + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + fieldPath: status.podIP + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.6.0 + livenessProbe: + failureThreshold: 3 + httpGet: + path: /live + port: system + periodSeconds: 5 + timeoutSeconds: 4 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: system + periodSeconds: 10 + timeoutSeconds: 4 + resources: + limits: + nvidia.com/gpu: '1' + startupProbe: + failureThreshold: 720 + httpGet: + path: /live + port: system + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + restartPolicy: RecreateGroupOnPodRestart + size: 2 + workerTemplate: + metadata: + labels: + nvidia.com/dynamo-component: worker + nvidia.com/dynamo-component-type: worker + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-aggregated-multi-component-workers + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: worker + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + command: + - python3 + - -m + - dynamo.vllm + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: worker + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-vllm-aggregated-multi-component-workers + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-vllm-aggregated-multi-component + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + fieldPath: status.podIP + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.6.0 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + resources: + limits: + nvidia.com/gpu: '1' + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + networkConfig: + subdomainPolicy: Shared + replicas: 2 + rolloutStrategy: + rollingUpdateConfiguration: + maxSurge: 0 + maxUnavailable: 1 + partition: 0 + type: RollingUpdate + startupPolicy: LeaderCreated +--- +apiVersion: nvidia.com/v1beta1 +kind: DynamoComponentDeployment +metadata: + annotations: + nvidia.com/dcd-spec: '{"version":1,"spec":{},"rawJSON":[{"path":"serviceName","nil":true}]}' + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-aggregated-multi-component-workers + name: workers-frontend + namespace: upgrade-latest-vllm-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: vllm + name: frontend + podTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + name: main + replicas: 1 + type: frontend +status: + component: + $strict: false + componentKind: Deployment + componentNames: + - workers-frontend + runtimeNamespace: upgrade-latest-vllm-aggregated-multi-component-workers + conditions: + - lastTransitionTime: $ignore + message: $ignore + reason: $ignore + status: 'False' + type: Available + - lastTransitionTime: $ignore + message: DynamoComponent is not ready + reason: ComponentNotReady + status: 'False' + type: DynamoComponentReady + observedGeneration: 1 +--- +apiVersion: nvidia.com/v1beta1 +kind: DynamoComponentDeployment +metadata: + annotations: + nvidia.com/dcd-spec: '{"version":1,"spec":{"resources":{"limits":{"gpu":"1"}},"extraPodSpec":{"mainContainer":{"name":"","resources":{}}}},"rawJSON":[{"path":"serviceName","nil":true}]}' + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + labels: + nvidia.com/dynamo-component: worker + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-aggregated-multi-component-workers + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-worker-{{worker-hash}} + namespace: upgrade-latest-vllm-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: vllm + multinode: + nodeCount: 2 + name: worker + podTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + labels: + nvidia.com/dynamo-worker-hash: $var:worker-hash + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + command: + - python3 + - -m + - dynamo.vllm + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.6.0 + name: main + resources: + limits: + nvidia.com/gpu: '1' + replicas: 2 + type: worker +status: + component: + $strict: false + componentKind: LeaderWorkerSet + componentNames: + - $var:worker-string:workers-worker-{{worker-hash}}-0 + runtimeNamespace: $var:worker-string:upgrade-latest-vllm-aggregated-multi-component-workers-{{worker-hash}} + servedModelName: Qwen/Qwen3-0.6B + conditions: + - lastTransitionTime: $ignore + message: $ignore + reason: $ignore + status: 'False' + type: Available + - lastTransitionTime: $ignore + message: DynamoComponent is not ready + reason: ComponentNotReady + status: 'False' + type: DynamoComponentReady + observedGeneration: 1 +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: 0d454cc8266ac38ed6453ee2ef3cf6d9dd237faebc63af22f67ca911e106796c + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery-role + namespace: upgrade-latest-vllm-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +rules: +- apiGroups: + - '' + resources: + - endpoints + - pods + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - nvidia.com + resources: + - dynamoworkermetadatas + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: 62b05ded2c1922ad3b5815ab50091a91a91990cfd3f659e13b274ceb5cdecdeb + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery-binding + namespace: upgrade-latest-vllm-aggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: workers-k8s-service-discovery-role +subjects: +- kind: ServiceAccount + name: workers-k8s-service-discovery + namespace: upgrade-latest-vllm-aggregated-multi-component diff --git a/deploy/operator/internal/controller/testdata/upgrade/latest/vllm/disaggregated/multi/component/input.yaml b/deploy/operator/internal/controller/testdata/upgrade/latest/vllm/disaggregated/multi/component/input.yaml new file mode 100644 index 000000000000..e3c14af857c5 --- /dev/null +++ b/deploy/operator/internal/controller/testdata/upgrade/latest/vllm/disaggregated/multi/component/input.yaml @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + name: workers + namespace: upgrade-latest-vllm-disaggregated-multi-component + annotations: + nvidia.com/enable-grove: 'false' + nvidia.com/dynamo-operator-origin-version: 0.0.0 +spec: + backendFramework: vllm + components: + - name: frontend + type: frontend + replicas: 1 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + command: + - python3 + - -m + - dynamo.frontend + - name: prefill + type: prefill + replicas: 2 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.6.0 + command: + - python3 + - -m + - dynamo.vllm + args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - prefill + resources: + limits: + nvidia.com/gpu: '1' + multinode: + nodeCount: 2 + - name: decode + type: decode + replicas: 2 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.6.0 + command: + - python3 + - -m + - dynamo.vllm + args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - decode + resources: + limits: + nvidia.com/gpu: '1' + multinode: + nodeCount: 2 diff --git a/deploy/operator/internal/controller/testdata/upgrade/latest/vllm/disaggregated/multi/component/output.yaml b/deploy/operator/internal/controller/testdata/upgrade/latest/vllm/disaggregated/multi/component/output.yaml new file mode 100644 index 000000000000..c47f4595b902 --- /dev/null +++ b/deploy/operator/internal/controller/testdata/upgrade/latest/vllm/disaggregated/multi/component/output.yaml @@ -0,0 +1,1296 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + annotations: + nvidia.com/current-worker-hash-v2: $var:worker-hash + nvidia.com/dynamo-operator-origin-version: $var:operator-version + nvidia.com/enable-grove: 'false' + nvidia.com/workload-provider: component + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + name: workers + namespace: upgrade-latest-vllm-disaggregated-multi-component + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: vllm + components: + - name: frontend + podTemplate: + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + name: main + replicas: 1 + type: frontend + - multinode: + nodeCount: 2 + name: prefill + podTemplate: + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - prefill + command: + - python3 + - -m + - dynamo.vllm + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.6.0 + name: main + resources: + limits: + nvidia.com/gpu: '1' + replicas: 2 + type: prefill + - multinode: + nodeCount: 2 + name: decode + podTemplate: + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - decode + command: + - python3 + - -m + - dynamo.vllm + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.6.0 + name: main + resources: + limits: + nvidia.com/gpu: '1' + replicas: 2 + type: decode +status: + components: + decode: + $strict: false + componentKind: LeaderWorkerSet + componentNames: + - $var:worker-string:workers-decode-{{worker-hash}}-0 + runtimeNamespace: $var:worker-string:upgrade-latest-vllm-disaggregated-multi-component-workers-{{worker-hash}} + servedModelName: Qwen/Qwen3-0.6B + frontend: + $strict: false + componentKind: Deployment + componentNames: + - workers-frontend + runtimeNamespace: upgrade-latest-vllm-disaggregated-multi-component-workers + prefill: + $strict: false + componentKind: LeaderWorkerSet + componentNames: + - $var:worker-string:workers-prefill-{{worker-hash}}-0 + runtimeNamespace: $var:worker-string:upgrade-latest-vllm-disaggregated-multi-component-workers-{{worker-hash}} + servedModelName: Qwen/Qwen3-0.6B + conditions: + - lastTransitionTime: $ignore + message: '$var:worker-string:Resources not ready: workers-decode-{{worker-hash}}: + Component deployment not ready - Available condition not true; workers-frontend: + Component deployment not ready - Available condition not true; workers-prefill-{{worker-hash}}: + Component deployment not ready - Available condition not true' + observedGeneration: 1 + reason: some_resources_are_not_ready + status: 'False' + type: Ready + observedGeneration: 1 + state: pending +--- +apiVersion: v1 +data: + wait-for-leader.py: "import socket, time, json, ssl, urllib.request, os\n\nSA =\ + \ \"/var/run/secrets/kubernetes.io/serviceaccount\"\nhost = os.environ[\"LEADER_HOST\"\ + ]\nport = int(os.environ[\"LEADER_PORT\"])\n\ndef _k8s_ctx():\n return ssl.create_default_context(cafile=f\"\ + {SA}/ca.crt\")\n\ndef _k8s_headers():\n token = open(f\"{SA}/token\").read()\n\ + \ return {\"Authorization\": f\"Bearer {token}\"}\n\ndef _k8s_api():\n ns\ + \ = open(f\"{SA}/namespace\").read()\n return f\"https://kubernetes.default.svc/api/v1/namespaces/{ns}/pods\"\ + \n\ndef resolve_leader_ip():\n # gethostbyname is IPv4-only; fall back to IPv6\ + \ for IPv6-only clusters.\n try:\n return socket.gethostbyname(host)\n\ + \ except socket.gaierror:\n return socket.getaddrinfo(host, None, socket.AF_INET6,\ + \ socket.SOCK_STREAM)[0][4][0]\n\ndef leader_pod_is_healthy():\n try:\n \ + \ ip = resolve_leader_ip()\n except socket.gaierror:\n return False,\ + \ \"DNS resolution failed\", None, None\n try:\n req = urllib.request.Request(\n\ + \ f\"{_k8s_api()}?fieldSelector=status.podIP={ip}\",\n headers=_k8s_headers(),\n\ + \ )\n resp = json.loads(urllib.request.urlopen(req, context=_k8s_ctx(),\ + \ timeout=5).read())\n pods = resp.get(\"items\", [])\n if not pods:\n\ + \ return False, f\"no pod found with IP {ip}\", None, ip\n pod\ + \ = pods[0]\n name = pod[\"metadata\"].get(\"name\", \"unknown\")\n \ + \ uid = pod[\"metadata\"].get(\"uid\", \"unknown\")\n phase = pod.get(\"\ + status\", {}).get(\"phase\")\n deletion_ts = pod[\"metadata\"].get(\"deletionTimestamp\"\ + )\n info = f\"ip={ip} pod={name} uid={uid} phase={phase} deletionTimestamp={deletion_ts}\"\ + \n if deletion_ts:\n return False, f\"pod {name} is terminating\"\ + , info, ip\n if phase != \"Running\":\n return False, f\"pod\ + \ {name} phase is {phase}\", info, ip\n return True, \"\", info, ip\n \ + \ except Exception as e:\n # Fall back to TCP-only when the API is unavailable\ + \ (e.g. 403 no RBAC)\n return True, f\"K8s API unavailable ({e}), falling\ + \ back to TCP\", f\"ip={ip}\", ip\n\nprint(f\"Waiting for leader master port at\ + \ {host}:{port}...\", flush=True)\ntime.sleep(5)\nstart = time.monotonic()\nlast_status\ + \ = start\nlast_err = \"\"\nwhile True:\n healthy, reason, pod_info, leader_ip\ + \ = leader_pod_is_healthy()\n if healthy:\n try:\n s = socket.create_connection((leader_ip,\ + \ port), timeout=2)\n s.close()\n elapsed = time.monotonic()\ + \ - start\n print(f\"Leader master port ready (waited {elapsed:.1f}s)\ + \ [{pod_info}]\", flush=True)\n break\n except Exception as\ + \ e:\n last_err = f\"tcp: {type(e).__name__}: {e} [{pod_info}]\"\n\ + \ else:\n last_err = f\"{reason} [{pod_info}]\" if pod_info else reason\n\ + \ now = time.monotonic()\n if now - last_status >= 30:\n print(f\"\ + Still waiting for {host}:{port}... ({now - start:.0f}s elapsed, last: {last_err})\"\ + , flush=True)\n last_status = now\n time.sleep(5)\n" +kind: ConfigMap +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: a615bfc016f57bef619ea5c67d88da24755349bb07efff9da0b2c79f18d6d076 + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-graph-deployment-name: workers + name: workers-wait-leader-script + namespace: upgrade-latest-vllm-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-disaggregated-multi-component-workers + nvidia.com/dynamo-sub-component-type: decode + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-decode-{{worker-hash}} + namespace: upgrade-latest-vllm-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: $var:worker-string:workers-decode-{{worker-hash}} + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + clusterIP: $ignore + clusterIPs: + - $ignore + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: system + port: 9090 + protocol: TCP + targetPort: system + selector: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-component-type: decode + nvidia.com/dynamo-namespace: upgrade-latest-vllm-disaggregated-multi-component-workers + role: leader + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-disaggregated-multi-component-workers + name: workers-frontend + namespace: upgrade-latest-vllm-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: workers-frontend + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + clusterIP: $ignore + clusterIPs: + - $ignore + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: http + port: 8000 + protocol: TCP + targetPort: http + selector: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-component-type: frontend + nvidia.com/dynamo-namespace: upgrade-latest-vllm-disaggregated-multi-component-workers + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-disaggregated-multi-component-workers + nvidia.com/dynamo-sub-component-type: prefill + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-prefill-{{worker-hash}} + namespace: upgrade-latest-vllm-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: $var:worker-string:workers-prefill-{{worker-hash}} + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + clusterIP: $ignore + clusterIPs: + - $ignore + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: system + port: 9090 + protocol: TCP + targetPort: system + selector: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-component-type: prefill + nvidia.com/dynamo-namespace: upgrade-latest-vllm-disaggregated-multi-component-workers + role: leader + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery + namespace: upgrade-latest-vllm-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + generation: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-disaggregated-multi-component-workers + name: workers-frontend + namespace: upgrade-latest-vllm-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: workers-frontend + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + nvidia.com/selector: workers-frontend + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-component-type: frontend + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-disaggregated-multi-component-workers + nvidia.com/metrics-enabled: 'true' + nvidia.com/selector: workers-frontend + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + env: + - name: DYNAMO_PORT + value: '8000' + - name: DYN_COMPONENT + value: frontend + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_HTTP_PORT + value: '8000' + - name: DYN_NAMESPACE + value: upgrade-latest-vllm-disaggregated-multi-component-workers + - name: DYN_NAMESPACE_PREFIX + value: upgrade-latest-vllm-disaggregated-multi-component-workers + - name: DYN_NAMESPACE_PREFIX_STRICT + value: 'true' + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-vllm-disaggregated-multi-component + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.uid + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /live + port: http + scheme: HTTP + initialDelaySeconds: 15 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: main + ports: + - containerPort: 8000 + name: http + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: http + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 3 + resources: {} + startupProbe: + failureThreshold: 30 + httpGet: + path: /live + port: http + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 1000 + serviceAccount: workers-k8s-service-discovery + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory +status: + $ignore: true +--- +apiVersion: leaderworkerset.x-k8s.io/v1 +kind: LeaderWorkerSet +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + generation: $ignore + labels: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-disaggregated-multi-component-workers + nvidia.com/dynamo-sub-component-type: decode + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-decode-{{worker-hash}}-0 + namespace: upgrade-latest-vllm-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: $var:worker-string:workers-decode-{{worker-hash}} + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + leaderWorkerTemplate: + leaderTemplate: + metadata: + labels: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-component-type: decode + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-disaggregated-multi-component-workers + nvidia.com/dynamo-sub-component-type: decode + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: leader + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - decode + command: + - python3 + - -m + - dynamo.vllm + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: decode + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-vllm-disaggregated-multi-component-workers + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-vllm-disaggregated-multi-component + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + fieldPath: status.podIP + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.6.0 + livenessProbe: + failureThreshold: 3 + httpGet: + path: /live + port: system + periodSeconds: 5 + timeoutSeconds: 4 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: system + periodSeconds: 10 + timeoutSeconds: 4 + resources: + limits: + nvidia.com/gpu: '1' + startupProbe: + failureThreshold: 720 + httpGet: + path: /live + port: system + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + restartPolicy: RecreateGroupOnPodRestart + size: 2 + workerTemplate: + metadata: + labels: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-component-type: decode + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-disaggregated-multi-component-workers + nvidia.com/dynamo-sub-component-type: decode + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: worker + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - decode + command: + - python3 + - -m + - dynamo.vllm + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: decode + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-vllm-disaggregated-multi-component-workers + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-vllm-disaggregated-multi-component + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + fieldPath: status.podIP + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.6.0 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + resources: + limits: + nvidia.com/gpu: '1' + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + networkConfig: + subdomainPolicy: Shared + replicas: 2 + rolloutStrategy: + rollingUpdateConfiguration: + maxSurge: 0 + maxUnavailable: 1 + partition: 0 + type: RollingUpdate + startupPolicy: LeaderCreated +--- +apiVersion: leaderworkerset.x-k8s.io/v1 +kind: LeaderWorkerSet +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + generation: $ignore + labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-disaggregated-multi-component-workers + nvidia.com/dynamo-sub-component-type: prefill + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-prefill-{{worker-hash}}-0 + namespace: upgrade-latest-vllm-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: $var:worker-string:workers-prefill-{{worker-hash}} + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + leaderWorkerTemplate: + leaderTemplate: + metadata: + labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-component-type: prefill + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-disaggregated-multi-component-workers + nvidia.com/dynamo-sub-component-type: prefill + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: leader + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - prefill + command: + - python3 + - -m + - dynamo.vllm + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: prefill + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-vllm-disaggregated-multi-component-workers + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-vllm-disaggregated-multi-component + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + fieldPath: status.podIP + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.6.0 + livenessProbe: + failureThreshold: 3 + httpGet: + path: /live + port: system + periodSeconds: 5 + timeoutSeconds: 4 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: system + periodSeconds: 10 + timeoutSeconds: 4 + resources: + limits: + nvidia.com/gpu: '1' + startupProbe: + failureThreshold: 720 + httpGet: + path: /live + port: system + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + restartPolicy: RecreateGroupOnPodRestart + size: 2 + workerTemplate: + metadata: + labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-component-type: prefill + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-disaggregated-multi-component-workers + nvidia.com/dynamo-sub-component-type: prefill + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: worker + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - prefill + command: + - python3 + - -m + - dynamo.vllm + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: prefill + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-vllm-disaggregated-multi-component-workers + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-vllm-disaggregated-multi-component + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + fieldPath: status.podIP + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.6.0 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + resources: + limits: + nvidia.com/gpu: '1' + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + networkConfig: + subdomainPolicy: Shared + replicas: 2 + rolloutStrategy: + rollingUpdateConfiguration: + maxSurge: 0 + maxUnavailable: 1 + partition: 0 + type: RollingUpdate + startupPolicy: LeaderCreated +--- +apiVersion: nvidia.com/v1beta1 +kind: DynamoComponentDeployment +metadata: + annotations: + nvidia.com/dcd-spec: '{"version":1,"spec":{"resources":{"limits":{"gpu":"1"}},"extraPodSpec":{"mainContainer":{"name":"","resources":{}}}},"rawJSON":[{"path":"serviceName","nil":true}]}' + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + labels: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-disaggregated-multi-component-workers + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-decode-{{worker-hash}} + namespace: upgrade-latest-vllm-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: vllm + multinode: + nodeCount: 2 + name: decode + podTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + labels: + nvidia.com/dynamo-worker-hash: $var:worker-hash + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - decode + command: + - python3 + - -m + - dynamo.vllm + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.6.0 + name: main + resources: + limits: + nvidia.com/gpu: '1' + replicas: 2 + type: decode +status: + component: + $strict: false + componentKind: LeaderWorkerSet + componentNames: + - $var:worker-string:workers-decode-{{worker-hash}}-0 + runtimeNamespace: $var:worker-string:upgrade-latest-vllm-disaggregated-multi-component-workers-{{worker-hash}} + servedModelName: Qwen/Qwen3-0.6B + conditions: + - lastTransitionTime: $ignore + message: $ignore + reason: $ignore + status: 'False' + type: Available + - lastTransitionTime: $ignore + message: DynamoComponent is not ready + reason: ComponentNotReady + status: 'False' + type: DynamoComponentReady + observedGeneration: 1 +--- +apiVersion: nvidia.com/v1beta1 +kind: DynamoComponentDeployment +metadata: + annotations: + nvidia.com/dcd-spec: '{"version":1,"spec":{},"rawJSON":[{"path":"serviceName","nil":true}]}' + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-disaggregated-multi-component-workers + name: workers-frontend + namespace: upgrade-latest-vllm-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: vllm + name: frontend + podTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + name: main + replicas: 1 + type: frontend +status: + component: + $strict: false + componentKind: Deployment + componentNames: + - workers-frontend + runtimeNamespace: upgrade-latest-vllm-disaggregated-multi-component-workers + conditions: + - lastTransitionTime: $ignore + message: $ignore + reason: $ignore + status: 'False' + type: Available + - lastTransitionTime: $ignore + message: DynamoComponent is not ready + reason: ComponentNotReady + status: 'False' + type: DynamoComponentReady + observedGeneration: 1 +--- +apiVersion: nvidia.com/v1beta1 +kind: DynamoComponentDeployment +metadata: + annotations: + nvidia.com/dcd-spec: '{"version":1,"spec":{"resources":{"limits":{"gpu":"1"}},"extraPodSpec":{"mainContainer":{"name":"","resources":{}}}},"rawJSON":[{"path":"serviceName","nil":true}]}' + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-disaggregated-multi-component-workers + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-prefill-{{worker-hash}} + namespace: upgrade-latest-vllm-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: vllm + multinode: + nodeCount: 2 + name: prefill + podTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + labels: + nvidia.com/dynamo-worker-hash: $var:worker-hash + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - prefill + command: + - python3 + - -m + - dynamo.vllm + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.6.0 + name: main + resources: + limits: + nvidia.com/gpu: '1' + replicas: 2 + type: prefill +status: + component: + $strict: false + componentKind: LeaderWorkerSet + componentNames: + - $var:worker-string:workers-prefill-{{worker-hash}}-0 + runtimeNamespace: $var:worker-string:upgrade-latest-vllm-disaggregated-multi-component-workers-{{worker-hash}} + servedModelName: Qwen/Qwen3-0.6B + conditions: + - lastTransitionTime: $ignore + message: $ignore + reason: $ignore + status: 'False' + type: Available + - lastTransitionTime: $ignore + message: DynamoComponent is not ready + reason: ComponentNotReady + status: 'False' + type: DynamoComponentReady + observedGeneration: 1 +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: 0d454cc8266ac38ed6453ee2ef3cf6d9dd237faebc63af22f67ca911e106796c + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery-role + namespace: upgrade-latest-vllm-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +rules: +- apiGroups: + - '' + resources: + - endpoints + - pods + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - nvidia.com + resources: + - dynamoworkermetadatas + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: a46fb1f8419f755fd12a60a20fb9dfde23e7b811eff3a73e674062a582bfd5eb + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery-binding + namespace: upgrade-latest-vllm-disaggregated-multi-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: workers-k8s-service-discovery-role +subjects: +- kind: ServiceAccount + name: workers-k8s-service-discovery + namespace: upgrade-latest-vllm-disaggregated-multi-component diff --git a/deploy/operator/internal/controller/testdata/upgrade/latest/vllm/disaggregated/multi/disaggregatedset/input.yaml b/deploy/operator/internal/controller/testdata/upgrade/latest/vllm/disaggregated/multi/disaggregatedset/input.yaml new file mode 100644 index 000000000000..9109397a2107 --- /dev/null +++ b/deploy/operator/internal/controller/testdata/upgrade/latest/vllm/disaggregated/multi/disaggregatedset/input.yaml @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + name: workers + namespace: upgrade-latest-vllm-disaggregated-multi-disaggregatedset + annotations: + nvidia.com/enable-grove: 'false' + nvidia.com/dynamo-operator-origin-version: 0.0.0 + nvidia.com/enable-disaggregatedset: 'true' +spec: + backendFramework: vllm + components: + - name: frontend + type: frontend + replicas: 1 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + command: + - python3 + - -m + - dynamo.frontend + - name: prefill + type: prefill + replicas: 2 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.6.0 + command: + - python3 + - -m + - dynamo.vllm + args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - prefill + resources: + limits: + nvidia.com/gpu: '1' + multinode: + nodeCount: 2 + - name: decode + type: decode + replicas: 2 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.6.0 + command: + - python3 + - -m + - dynamo.vllm + args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - decode + resources: + limits: + nvidia.com/gpu: '1' + multinode: + nodeCount: 2 diff --git a/deploy/operator/internal/controller/testdata/upgrade/latest/vllm/disaggregated/multi/disaggregatedset/output.yaml b/deploy/operator/internal/controller/testdata/upgrade/latest/vllm/disaggregated/multi/disaggregatedset/output.yaml new file mode 100644 index 000000000000..76f50da0d478 --- /dev/null +++ b/deploy/operator/internal/controller/testdata/upgrade/latest/vllm/disaggregated/multi/disaggregatedset/output.yaml @@ -0,0 +1,1108 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + annotations: + nvidia.com/current-worker-hash-v2: $var:worker-hash + nvidia.com/dynamo-operator-origin-version: $var:operator-version + nvidia.com/enable-disaggregatedset: 'true' + nvidia.com/enable-grove: 'false' + nvidia.com/workload-provider: disaggregatedset + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + name: workers + namespace: upgrade-latest-vllm-disaggregated-multi-disaggregatedset + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: vllm + components: + - name: frontend + podTemplate: + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + name: main + replicas: 1 + type: frontend + - multinode: + nodeCount: 2 + name: prefill + podTemplate: + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - prefill + command: + - python3 + - -m + - dynamo.vllm + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.6.0 + name: main + resources: + limits: + nvidia.com/gpu: '1' + replicas: 2 + type: prefill + - multinode: + nodeCount: 2 + name: decode + podTemplate: + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - decode + command: + - python3 + - -m + - dynamo.vllm + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.6.0 + name: main + resources: + limits: + nvidia.com/gpu: '1' + replicas: 2 + type: decode +status: + components: + decode: + $strict: false + componentKind: LeaderWorkerSet + frontend: + $strict: false + componentKind: Deployment + componentNames: + - workers-frontend + runtimeNamespace: upgrade-latest-vllm-disaggregated-multi-disagg-965397d6c4380e14 + prefill: + $strict: false + componentKind: LeaderWorkerSet + conditions: + - lastTransitionTime: $ignore + message: DisaggregatedSet pathway selected + observedGeneration: 1 + reason: Selected + status: 'True' + type: DisaggregatedSetEligible + - lastTransitionTime: $ignore + message: 'Resources not ready: workers: decode role "decode" slice 0 has 0 target + LeaderWorkerSets, expected 1; prefill role "prefill" slice 0 has 0 target LeaderWorkerSets, + expected 1; workers-frontend: Component deployment not ready - Available condition + not true' + observedGeneration: 1 + reason: some_resources_are_not_ready + status: 'False' + type: Ready + observedGeneration: 1 + state: pending +--- +apiVersion: v1 +data: + wait-for-leader.py: "import socket, time, json, ssl, urllib.request, os\n\nSA =\ + \ \"/var/run/secrets/kubernetes.io/serviceaccount\"\nhost = os.environ[\"LEADER_HOST\"\ + ]\nport = int(os.environ[\"LEADER_PORT\"])\n\ndef _k8s_ctx():\n return ssl.create_default_context(cafile=f\"\ + {SA}/ca.crt\")\n\ndef _k8s_headers():\n token = open(f\"{SA}/token\").read()\n\ + \ return {\"Authorization\": f\"Bearer {token}\"}\n\ndef _k8s_api():\n ns\ + \ = open(f\"{SA}/namespace\").read()\n return f\"https://kubernetes.default.svc/api/v1/namespaces/{ns}/pods\"\ + \n\ndef resolve_leader_ip():\n # gethostbyname is IPv4-only; fall back to IPv6\ + \ for IPv6-only clusters.\n try:\n return socket.gethostbyname(host)\n\ + \ except socket.gaierror:\n return socket.getaddrinfo(host, None, socket.AF_INET6,\ + \ socket.SOCK_STREAM)[0][4][0]\n\ndef leader_pod_is_healthy():\n try:\n \ + \ ip = resolve_leader_ip()\n except socket.gaierror:\n return False,\ + \ \"DNS resolution failed\", None, None\n try:\n req = urllib.request.Request(\n\ + \ f\"{_k8s_api()}?fieldSelector=status.podIP={ip}\",\n headers=_k8s_headers(),\n\ + \ )\n resp = json.loads(urllib.request.urlopen(req, context=_k8s_ctx(),\ + \ timeout=5).read())\n pods = resp.get(\"items\", [])\n if not pods:\n\ + \ return False, f\"no pod found with IP {ip}\", None, ip\n pod\ + \ = pods[0]\n name = pod[\"metadata\"].get(\"name\", \"unknown\")\n \ + \ uid = pod[\"metadata\"].get(\"uid\", \"unknown\")\n phase = pod.get(\"\ + status\", {}).get(\"phase\")\n deletion_ts = pod[\"metadata\"].get(\"deletionTimestamp\"\ + )\n info = f\"ip={ip} pod={name} uid={uid} phase={phase} deletionTimestamp={deletion_ts}\"\ + \n if deletion_ts:\n return False, f\"pod {name} is terminating\"\ + , info, ip\n if phase != \"Running\":\n return False, f\"pod\ + \ {name} phase is {phase}\", info, ip\n return True, \"\", info, ip\n \ + \ except Exception as e:\n # Fall back to TCP-only when the API is unavailable\ + \ (e.g. 403 no RBAC)\n return True, f\"K8s API unavailable ({e}), falling\ + \ back to TCP\", f\"ip={ip}\", ip\n\nprint(f\"Waiting for leader master port at\ + \ {host}:{port}...\", flush=True)\ntime.sleep(5)\nstart = time.monotonic()\nlast_status\ + \ = start\nlast_err = \"\"\nwhile True:\n healthy, reason, pod_info, leader_ip\ + \ = leader_pod_is_healthy()\n if healthy:\n try:\n s = socket.create_connection((leader_ip,\ + \ port), timeout=2)\n s.close()\n elapsed = time.monotonic()\ + \ - start\n print(f\"Leader master port ready (waited {elapsed:.1f}s)\ + \ [{pod_info}]\", flush=True)\n break\n except Exception as\ + \ e:\n last_err = f\"tcp: {type(e).__name__}: {e} [{pod_info}]\"\n\ + \ else:\n last_err = f\"{reason} [{pod_info}]\" if pod_info else reason\n\ + \ now = time.monotonic()\n if now - last_status >= 30:\n print(f\"\ + Still waiting for {host}:{port}... ({now - start:.0f}s elapsed, last: {last_err})\"\ + , flush=True)\n last_status = now\n time.sleep(5)\n" +kind: ConfigMap +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: a615bfc016f57bef619ea5c67d88da24755349bb07efff9da0b2c79f18d6d076 + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-graph-deployment-name: workers + name: workers-wait-leader-script + namespace: upgrade-latest-vllm-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + nvidia.com/dynamo-managed-service-metadata: '{"labels":["nvidia.com/dynamo-component","nvidia.com/dynamo-discovery-backend","nvidia.com/dynamo-discovery-enabled","nvidia.com/dynamo-graph-deployment-name","nvidia.com/dynamo-namespace","nvidia.com/dynamo-worker-hash"],"annotations":["nvidia.com/dynamo-operator-origin-version"]}' + nvidia.com/dynamo-operator-origin-version: $var:operator-version + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-disaggregated-multi-disagg-965397d6c4380e14 + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-decode-{{worker-hash}} + namespace: upgrade-latest-vllm-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + clusterIP: $ignore + clusterIPs: + - $ignore + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: system + port: 9090 + protocol: TCP + targetPort: system + selector: + disaggregatedset.x-k8s.io/name: workers + disaggregatedset.x-k8s.io/revision: $var:ds-revision + disaggregatedset.x-k8s.io/role: decode + role: leader + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + nvidia.com/dynamo-managed-service-metadata: '{"labels":["nvidia.com/dynamo-component","nvidia.com/dynamo-discovery-backend","nvidia.com/dynamo-discovery-enabled","nvidia.com/dynamo-graph-deployment-name","nvidia.com/dynamo-namespace"],"annotations":["nvidia.com/dynamo-operator-origin-version"]}' + nvidia.com/dynamo-operator-origin-version: $var:operator-version + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-disaggregated-multi-disagg-965397d6c4380e14 + name: workers-frontend + namespace: upgrade-latest-vllm-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + clusterIP: $ignore + clusterIPs: + - $ignore + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: http + port: 8000 + protocol: TCP + targetPort: http + selector: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-component-type: frontend + nvidia.com/dynamo-namespace: upgrade-latest-vllm-disaggregated-multi-disagg-965397d6c4380e14 + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + nvidia.com/dynamo-managed-service-metadata: '{"labels":["nvidia.com/dynamo-component","nvidia.com/dynamo-discovery-backend","nvidia.com/dynamo-discovery-enabled","nvidia.com/dynamo-graph-deployment-name","nvidia.com/dynamo-namespace","nvidia.com/dynamo-worker-hash"],"annotations":["nvidia.com/dynamo-operator-origin-version"]}' + nvidia.com/dynamo-operator-origin-version: $var:operator-version + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-disaggregated-multi-disagg-965397d6c4380e14 + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-prefill-{{worker-hash}} + namespace: upgrade-latest-vllm-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + clusterIP: $ignore + clusterIPs: + - $ignore + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: system + port: 9090 + protocol: TCP + targetPort: system + selector: + disaggregatedset.x-k8s.io/name: workers + disaggregatedset.x-k8s.io/revision: $var:ds-revision + disaggregatedset.x-k8s.io/role: prefill + role: leader + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery + namespace: upgrade-latest-vllm-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + generation: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-disaggregated-multi-disagg-965397d6c4380e14 + name: workers-frontend + namespace: upgrade-latest-vllm-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: workers-frontend + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + nvidia.com/selector: workers-frontend + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-component-type: frontend + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-disaggregated-multi-disagg-965397d6c4380e14 + nvidia.com/metrics-enabled: 'true' + nvidia.com/selector: workers-frontend + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + env: + - name: DYNAMO_PORT + value: '8000' + - name: DYN_COMPONENT + value: frontend + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_HTTP_PORT + value: '8000' + - name: DYN_NAMESPACE + value: upgrade-latest-vllm-disaggregated-multi-disagg-965397d6c4380e14 + - name: DYN_NAMESPACE_PREFIX + value: upgrade-latest-vllm-disaggregated-multi-disagg-965397d6c4380e14 + - name: DYN_NAMESPACE_PREFIX_STRICT + value: 'true' + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-vllm-disaggregated-multi-disaggregatedset + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.uid + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /live + port: http + scheme: HTTP + initialDelaySeconds: 15 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: main + ports: + - containerPort: 8000 + name: http + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: http + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 3 + resources: {} + startupProbe: + failureThreshold: 30 + httpGet: + path: /live + port: http + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 1000 + serviceAccount: workers-k8s-service-discovery + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory +status: + $ignore: true +--- +apiVersion: disaggregatedset.x-k8s.io/v1 +kind: DisaggregatedSet +metadata: + creationTimestamp: $ignore + generation: $ignore + labels: + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/selector: workers + name: workers + namespace: upgrade-latest-vllm-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + roles: + - name: prefill + spec: + leaderWorkerTemplate: + leaderTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-component-type: prefill + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-disaggregated-multi-disagg-965397d6c4380e14 + nvidia.com/dynamo-sub-component-type: prefill + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: leader + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - prefill + command: + - python3 + - -m + - dynamo.vllm + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: prefill + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-vllm-disaggregated-multi-disagg-965397d6c4380e14 + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-vllm-disaggregated-multi-disaggregatedset + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + fieldPath: status.podIP + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.6.0 + livenessProbe: + failureThreshold: 3 + httpGet: + path: /live + port: system + periodSeconds: 5 + timeoutSeconds: 4 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: system + periodSeconds: 10 + timeoutSeconds: 4 + resources: + limits: + nvidia.com/gpu: '1' + startupProbe: + failureThreshold: 720 + httpGet: + path: /live + port: system + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + restartPolicy: RecreateGroupOnPodRestart + size: 2 + workerTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-component-type: prefill + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-disaggregated-multi-disagg-965397d6c4380e14 + nvidia.com/dynamo-sub-component-type: prefill + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: worker + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - prefill + command: + - python3 + - -m + - dynamo.vllm + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: prefill + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-vllm-disaggregated-multi-disagg-965397d6c4380e14 + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-vllm-disaggregated-multi-disaggregatedset + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + fieldPath: status.podIP + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.6.0 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + resources: + limits: + nvidia.com/gpu: '1' + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + replicas: 2 + rolloutStrategy: + type: RollingUpdate + startupPolicy: LeaderCreated + - name: decode + spec: + leaderWorkerTemplate: + leaderTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + labels: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-component-type: decode + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-disaggregated-multi-disagg-965397d6c4380e14 + nvidia.com/dynamo-sub-component-type: decode + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: leader + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - decode + command: + - python3 + - -m + - dynamo.vllm + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: decode + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-vllm-disaggregated-multi-disagg-965397d6c4380e14 + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-vllm-disaggregated-multi-disaggregatedset + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + fieldPath: status.podIP + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.6.0 + livenessProbe: + failureThreshold: 3 + httpGet: + path: /live + port: system + periodSeconds: 5 + timeoutSeconds: 4 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: system + periodSeconds: 10 + timeoutSeconds: 4 + resources: + limits: + nvidia.com/gpu: '1' + startupProbe: + failureThreshold: 720 + httpGet: + path: /live + port: system + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + restartPolicy: RecreateGroupOnPodRestart + size: 2 + workerTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + labels: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-component-type: decode + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-disaggregated-multi-disagg-965397d6c4380e14 + nvidia.com/dynamo-sub-component-type: decode + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + role: worker + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - decode + command: + - python3 + - -m + - dynamo.vllm + env: + - name: DYNAMO_LEADER_ADDRESS + value: $(LWS_LEADER_ADDRESS) + - name: DYNAMO_RANK + value: $(LWS_WORKER_INDEX) + - name: DYN_COMPONENT + value: decode + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-vllm-disaggregated-multi-disagg-965397d6c4380e14 + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-vllm-disaggregated-multi-disaggregatedset + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + fieldPath: status.podIP + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.6.0 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + resources: + limits: + nvidia.com/gpu: '1' + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + replicas: 2 + rolloutStrategy: + type: RollingUpdate + startupPolicy: LeaderCreated + slices: 1 +--- +apiVersion: nvidia.com/v1beta1 +kind: DynamoComponentDeployment +metadata: + annotations: + nvidia.com/dcd-spec: '{"version":1,"spec":{},"rawJSON":[{"path":"serviceName","nil":true}]}' + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-vllm-disaggregated-multi-disagg-965397d6c4380e14 + name: workers-frontend + namespace: upgrade-latest-vllm-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: vllm + name: frontend + podTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + name: main + replicas: 1 + type: frontend +status: + component: + $strict: false + componentKind: Deployment + componentNames: + - workers-frontend + runtimeNamespace: upgrade-latest-vllm-disaggregated-multi-disagg-965397d6c4380e14 + conditions: + - lastTransitionTime: $ignore + message: $ignore + reason: $ignore + status: 'False' + type: Available + - lastTransitionTime: $ignore + message: DynamoComponent is not ready + reason: ComponentNotReady + status: 'False' + type: DynamoComponentReady + observedGeneration: 1 +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: 0d454cc8266ac38ed6453ee2ef3cf6d9dd237faebc63af22f67ca911e106796c + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery-role + namespace: upgrade-latest-vllm-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +rules: +- apiGroups: + - '' + resources: + - endpoints + - pods + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - nvidia.com + resources: + - dynamoworkermetadatas + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: 9251414205e3bfb5e9c5f5223d11b85f17e0a4d323934b03f661c483490fa376 + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery-binding + namespace: upgrade-latest-vllm-disaggregated-multi-disaggregatedset + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: workers-k8s-service-discovery-role +subjects: +- kind: ServiceAccount + name: workers-k8s-service-discovery + namespace: upgrade-latest-vllm-disaggregated-multi-disaggregatedset diff --git a/deploy/operator/internal/controller/upgrade_clustertest_test.go b/deploy/operator/internal/controller/upgrade_clustertest_test.go index ada564aa07d6..86e1f1a1ba63 100644 --- a/deploy/operator/internal/controller/upgrade_clustertest_test.go +++ b/deploy/operator/internal/controller/upgrade_clustertest_test.go @@ -246,7 +246,7 @@ func TestClusterOperatorUpgrade(t *testing.T) { operatorConfig.MPI.SSHSecretNamespace = namespace require.Empty(t, configvalidation.ValidateOperatorConfiguration(operatorConfig)) env.StartManager(func(mgr ctrl.Manager) error { - opts := SetupOptions{Config: operatorConfig, RuntimeConfig: &common.RuntimeConfig{Gate: features.Gates{Grove: true, LWS: true}}} + opts := SetupOptions{Config: operatorConfig, RuntimeConfig: &common.RuntimeConfig{Gate: features.Gates{Grove: true, LWS: true, DisaggregatedSet: true}}} if err := SetupDynamoGraphDeployment(mgr, DynamoGraphDeploymentSetupOptions{SetupOptions: opts}); err != nil { return err } @@ -262,17 +262,19 @@ func TestClusterOperatorUpgrade(t *testing.T) { return err == nil && live.Status.ObservedGeneration == live.Generation && live.Annotations[consts.AnnotationCurrentWorkerHashV2] == want }, 90*time.Second, 200*time.Millisecond) } + dgd = getOnlyClusterDGD(t, ctx, env.Client(), namespace) var after []unstructured.Unstructured if dgd.Annotations[consts.KubeAnnotationEnableGrove] == "false" { t.Log("Wait for each DCD and its immediate workload without requiring serving Pods") require.Eventually(t, func() bool { var err error - after, err = captureUpgradeComponent(t, env, completed, len(dgd.Spec.Components)) + after, err = captureUpgradeComponent(t, env, completed, upgradeExpectedDCDs(dgd)) return err == nil }, 90*time.Second, 200*time.Millisecond, "component workload graph did not converge") } else { after = captureUpgrade(t, env) } + checkUpgradeWorkloadMatrix(t, relative, dgd, after) checkUpgradeGroveMatrix(t, relative, getOnlyClusterDGD(t, ctx, env.Client(), namespace), after) if isHistory { if err := upgradeSafetyError(transition.hashChanged, before, after); err != nil { @@ -295,37 +297,22 @@ func TestClusterOperatorUpgrade(t *testing.T) { require.NoError(t, os.MkdirAll(filepath.Dir(path), 0755)) require.NoError(t, golden.WriteManifests(path, after)) } - if !isHistory && !item.promote && strings.HasSuffix(relative, "/component") && !t.Failed() { - t.Log("An intentional worker edit must reach a newly rendered Deployment even while Pods are blocked") + if !isHistory && !item.promote && (strings.HasSuffix(relative, "/component") || strings.HasSuffix(relative, "/disaggregatedset")) && !t.Failed() { + t.Log("An intentional worker edit must reach every immediate worker template while Pods are blocked") + var workers []string current := getOnlyClusterDGD(t, ctx, env.Client(), namespace) for i := range current.Spec.Components { if current.Spec.Components[i].ComponentType == api.ComponentTypeFrontend { continue } + workers = append(workers, current.Spec.Components[i].ComponentName) container := ¤t.Spec.Components[i].PodTemplate.Spec.Containers[0] container.Env = append(container.Env, corev1.EnvVar{Name: "WORKER_REVISION", Value: "2"}) } require.NoError(t, env.Client().Update(ctx, current)) require.Eventually(t, func() bool { - for _, object := range captureUpgrade(t, env) { - if object.GetKind() != "Deployment" { - continue - } - containers, _, err := unstructured.NestedSlice(object.Object, "spec", "template", "spec", "containers") - if err != nil { - return false - } - for _, container := range containers { - for _, variable := range container.(map[string]any)["env"].([]any) { - v := variable.(map[string]any) - if v["name"] == "WORKER_REVISION" && v["value"] == "2" { - return true - } - } - } - } - return false - }, 60*time.Second, 200*time.Millisecond, "worker edit did not reach a Deployment") + return upgradeWorkerEditError(captureUpgrade(t, env), workers) == nil + }, 60*time.Second, 200*time.Millisecond, "worker edit did not reach every immediate template") require.Error(t, upgradeSafetyError(false, after, captureUpgrade(t, env))) } if isHistory && strings.HasSuffix(relative, "/sglang-1.5") && !t.Failed() { @@ -374,6 +361,7 @@ func TestClusterOperatorUpgrade(t *testing.T) { }) } t.Run("component-history-barrier", testUpgradeComponentHistoryBarrier) + t.Run("disaggregatedset-replay", testUpgradeDisaggregatedSetReplay) } // Upgrade capture and replay use the same version source as production admission. diff --git a/deploy/operator/internal/controller/upgrade_component_clustertest_test.go b/deploy/operator/internal/controller/upgrade_component_clustertest_test.go index 1bc17f9a6435..148cbd7ddd48 100644 --- a/deploy/operator/internal/controller/upgrade_component_clustertest_test.go +++ b/deploy/operator/internal/controller/upgrade_component_clustertest_test.go @@ -54,6 +54,14 @@ func (c incompatibleUpgradeClient) Get(ctx context.Context, key client.ObjectKey } func testUpgradeComponentHistoryBarrier(t *testing.T) { + for _, topology := range []string{"single", "multi"} { + t.Run(topology, func(t *testing.T) { + testUpgradeComponentHistory(t, "testdata/upgrade/latest/sglang/aggregated/"+topology+"/component/input.yaml") + }) + } +} + +func testUpgradeComponentHistory(t *testing.T, input string) { var snapshot []unstructured.Unstructured var namespace string var seedClient client.Client @@ -63,7 +71,7 @@ func testUpgradeComponentHistoryBarrier(t *testing.T) { env := clusterTestEnv.RunT(t) namespace, seedClient = env.Namespace(), env.Client() env.BlockWorkloads() - objects, err := golden.ReadManifests("testdata/upgrade/latest/sglang/aggregated/single/component/input.yaml") + objects, err := golden.ReadManifests(input) require.NoError(t, err) for i := range objects { objects[i].SetNamespace(namespace) diff --git a/deploy/operator/internal/controller/upgrade_component_test.go b/deploy/operator/internal/controller/upgrade_component_test.go index 49a2b801932a..f3d05ba63167 100644 --- a/deploy/operator/internal/controller/upgrade_component_test.go +++ b/deploy/operator/internal/controller/upgrade_component_test.go @@ -63,7 +63,7 @@ func upgradeComponentGraphReady(objects []unstructured.Unstructured, components } func TestUpgradeComponentSafety(t *testing.T) { - for _, kind := range []string{"Deployment", "LeaderWorkerSet"} { + for _, kind := range []string{"Deployment", "LeaderWorkerSet", "DisaggregatedSet"} { t.Run(kind, func(t *testing.T) { t.Log("Preserve immediate workload identity and spec independently of golden updates") old := unstructured.Unstructured{Object: map[string]any{ @@ -88,7 +88,12 @@ func TestUpgradeComponentSafety(t *testing.T) { if kind == "LeaderWorkerSet" { path = []string{"spec", "leaderWorkerTemplate", "workerTemplate", "spec", "hostname"} } - require.NoError(t, unstructured.SetNestedField(changed.Object, "changed-worker", path...)) + if kind == "DisaggregatedSet" { + path = []string{"spec", "roles"} + require.NoError(t, unstructured.SetNestedSlice(changed.Object, []any{map[string]any{"name": "changed-role"}}, path...)) + } else { + require.NoError(t, unstructured.SetNestedField(changed.Object, "changed-worker", path...)) + } require.ErrorContains(t, upgradeSafetyError(false, before, []unstructured.Unstructured{*changed}), "rollout intent") require.ErrorContains(t, upgradeSafetyError(false, before, nil), "inventory") }) @@ -129,19 +134,27 @@ func TestUpgradeComponentGraphReady(t *testing.T) { } func TestUpgradeComponentVersionBindings(t *testing.T) { - t.Log("Keep generated component identities exact across an operator version bump") - path := "testdata/upgrade/latest/sglang/aggregated/single/component/output.yaml" - previous := upgradeLatestVariables(t, path, "1.6.0") - current := upgradeLatestVariables(t, path, "1.7.0") - require.NotEqual(t, previous["worker-hash"], current["worker-hash"]) - bindings := 0 - for name, value := range current { - if !strings.HasPrefix(name, "worker-string:") { - continue - } - bindings++ - require.Contains(t, value, current["worker-hash"]) - require.NotEqual(t, previous[name], value) + for _, scenario := range []string{"aggregated/single/component", "aggregated/multi/component", "disaggregated/multi/disaggregatedset"} { + t.Run(scenario, func(t *testing.T) { + t.Log("Keep generated component identities exact across an operator version bump") + path := "testdata/upgrade/latest/sglang/" + scenario + "/output.yaml" + previous := upgradeLatestVariables(t, path, "1.6.0") + current := upgradeLatestVariables(t, path, "1.7.0") + require.NotEqual(t, previous["worker-hash"], current["worker-hash"]) + bindings := 0 + for name, value := range current { + if !strings.HasPrefix(name, "worker-string:") { + continue + } + bindings++ + require.Contains(t, value, current["worker-hash"]) + require.NotEqual(t, previous[name], value) + } + require.Positive(t, bindings, "component names must follow the reviewed DGD hash") + if strings.HasSuffix(scenario, "/disaggregatedset") { + require.NotEqual(t, "pending", current["ds-revision"]) + require.NotEqual(t, previous["ds-revision"], current["ds-revision"]) + } + }) } - require.Positive(t, bindings, "component names must follow the reviewed DGD hash") } diff --git a/deploy/operator/internal/controller/upgrade_contract_test.go b/deploy/operator/internal/controller/upgrade_contract_test.go index 5dbbacc6c42c..8e561f34a3a6 100644 --- a/deploy/operator/internal/controller/upgrade_contract_test.go +++ b/deploy/operator/internal/controller/upgrade_contract_test.go @@ -83,7 +83,7 @@ func upgradeSafetyError(generationChanged bool, before, after []unstructured.Uns return fmt.Errorf("DGD rollout intent changed: %s", diff) } } - case "Deployment", "LeaderWorkerSet": + case "Deployment", "LeaderWorkerSet", "DisaggregatedSet": if err := upgradeComponentWorkloadSafetyError(old, current); err != nil { return err } @@ -132,7 +132,7 @@ func upgradeComponentWorkloadSafetyError(old, current unstructured.Unstructured) } func isUpgradeWorkload(kind string) bool { - return kind == upgradePodCliqueSetKind || kind == "Deployment" || kind == "LeaderWorkerSet" + return kind == upgradePodCliqueSetKind || kind == "Deployment" || kind == "LeaderWorkerSet" || kind == "DisaggregatedSet" } func upgradeWorkerTemplates(object unstructured.Unstructured) (map[string]any, error) { @@ -222,7 +222,7 @@ func TestUpgradeScenarioNamespaces(t *testing.T) { // Hash expectations come from the frozen golden spec, never from candidate output. func upgradeLatestVariables(t *testing.T, path, version string) golden.Variables { t.Helper() - variables := golden.Variables{"operator-version": version, "worker-hash": "pending", "pcs-spec-hash": "pending"} + variables := golden.Variables{"operator-version": version, "worker-hash": "pending", "pcs-spec-hash": "pending", "ds-revision": "pending"} raw, err := golden.ReadManifests(path) require.NoError(t, err) for _, object := range raw { @@ -253,7 +253,7 @@ func upgradeLatestVariables(t *testing.T, path, version string) golden.Variables upgradeWorkerStringVariables(object.Object, variables["worker-hash"], variables) } - // Rebind the golden PCS template before deriving its exact spec digest. + // Derive provider revisions from reviewed templates after rebinding worker identity. objects, err = golden.ReadManifestsWithVariables(path, variables) require.NoError(t, err) for _, object := range objects { @@ -261,6 +261,11 @@ func upgradeLatestVariables(t *testing.T, path, version string) golden.Variables variables["pcs-spec-hash"], err = common.GetSpecHash(&object) require.NoError(t, err) } + if object.GetKind() == "DisaggregatedSet" { + spec := unstructured.Unstructured{Object: map[string]any{"spec": object.Object["spec"]}} + variables["ds-revision"], err = disaggregatedSetTargetRevision(&spec) + require.NoError(t, err) + } } return variables } diff --git a/deploy/operator/internal/controller/upgrade_disaggregatedset_clustertest_test.go b/deploy/operator/internal/controller/upgrade_disaggregatedset_clustertest_test.go new file mode 100644 index 000000000000..35fcb627bfc1 --- /dev/null +++ b/deploy/operator/internal/controller/upgrade_disaggregatedset_clustertest_test.go @@ -0,0 +1,105 @@ +//go:build clustertest && upgradecontract + +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "testing" + "time" + + common "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/golden" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + disaggregatedsetv1 "sigs.k8s.io/lws/api/disaggregatedset/v1" +) + +// Exercise a populated DS graph without fabricating released historical provenance. +func testUpgradeDisaggregatedSetReplay(t *testing.T) { + var snapshot []unstructured.Unstructured + var namespace string + var seedClient client.Client + t.Run("capture", func(t *testing.T) { + t.Log("Capture production DS lowering and its auxiliary frontend without downstream controllers") + env := clusterTestEnv.RunT(t) + namespace, seedClient = env.Namespace(), env.Client() + env.BlockWorkloads() + objects, err := golden.ReadManifests("testdata/upgrade/latest/sglang/disaggregated/multi/disaggregatedset/input.yaml") + require.NoError(t, err) + for i := range objects { + objects[i].SetNamespace(namespace) + require.NoError(t, seedClient.Create(t.Context(), &objects[i])) + } + completed := &upgradeComponentReconciles{} + env.StartManager(func(mgr ctrl.Manager) error { + opts := SetupOptions{Config: clusterTestRestrictedConfig(namespace), RuntimeConfig: &common.RuntimeConfig{Gate: features.Gates{Grove: true, LWS: true, DisaggregatedSet: true}}} + if err := SetupDynamoGraphDeployment(mgr, DynamoGraphDeploymentSetupOptions{SetupOptions: opts}); err != nil { + return err + } + return setupUpgradeComponentController(mgr, opts, completed) + }) + require.Eventually(t, func() bool { + var err error + snapshot, err = captureUpgradeComponent(t, env, completed, 1) + return err == nil + }, 90*time.Second, 100*time.Millisecond) + }) + require.NotEmpty(t, snapshot) + + t.Run("takeover", func(t *testing.T) { + t.Log("Restore the concrete DS graph with the workload quota already active") + require.NoError(t, seedClient.Create(t.Context(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}})) + env := clusterTestEnv.RunTInNamespace(t, namespace) + env.BlockWorkloads() + require.NoError(t, golden.ApplySnapshot(t.Context(), env.Client(), namespace, snapshot)) + before := captureUpgrade(t, env) + completed := &upgradeComponentReconciles{} + baseline := completedUpgradeReconciles(t) + env.StartManager(func(mgr ctrl.Manager) error { + opts := SetupOptions{Config: clusterTestRestrictedConfig(namespace), RuntimeConfig: &common.RuntimeConfig{Gate: features.Gates{Grove: true, LWS: true, DisaggregatedSet: true}}} + if err := SetupDynamoGraphDeployment(mgr, DynamoGraphDeploymentSetupOptions{SetupOptions: opts}); err != nil { + return err + } + return setupUpgradeComponentController(mgr, opts, completed) + }) + require.Eventually(t, func() bool { return completedUpgradeReconciles(t) > baseline }, 90*time.Second, 100*time.Millisecond) + var after []unstructured.Unstructured + require.Eventually(t, func() bool { + var err error + after, err = captureUpgradeComponent(t, env, completed, 1) + return err == nil + }, 90*time.Second, 100*time.Millisecond) + require.NoError(t, upgradeSafetyError(false, before, after)) + checkUpgradeWorkloadMatrix(t, "latest/sglang/disaggregated/multi/disaggregatedset", getOnlyClusterDGD(t, t.Context(), env.Client(), namespace), after) + + t.Log("The upstream DS admission handler rejects a role rollout strategy accepted by the LWS schema") + var workloads disaggregatedsetv1.DisaggregatedSetList + require.NoError(t, env.Client().List(t.Context(), &workloads, client.InNamespace(namespace))) + require.Len(t, workloads.Items, 1) + invalid := workloads.Items[0].DeepCopy() + invalid.Spec.Roles[0].Spec.RolloutStrategy.Type = "Recreate" + require.ErrorContains(t, env.Client().Update(t.Context(), invalid), "vdisaggregatedset.kb.io") + + t.Log("The independent assertion rejects changed DS worker templates without a golden comparison") + changed := make([]unstructured.Unstructured, len(after)) + for i := range after { + changed[i] = *after[i].DeepCopy() + if changed[i].GetKind() != "DisaggregatedSet" { + continue + } + roles, _, err := unstructured.NestedSlice(changed[i].Object, "spec", "roles") + require.NoError(t, err) + role := roles[0].(map[string]any) + require.NoError(t, unstructured.SetNestedField(role, "incompatible", "spec", "leaderWorkerTemplate", "workerTemplate", "spec", "hostname")) + require.NoError(t, unstructured.SetNestedSlice(changed[i].Object, roles, "spec", "roles")) + } + require.ErrorContains(t, upgradeSafetyError(false, before, changed), "rollout intent") + }) +} diff --git a/deploy/operator/internal/controller/upgrade_workload_edit_test.go b/deploy/operator/internal/controller/upgrade_workload_edit_test.go new file mode 100644 index 000000000000..f3d5c560f72e --- /dev/null +++ b/deploy/operator/internal/controller/upgrade_workload_edit_test.go @@ -0,0 +1,108 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// Every edited component must reach its immediate workload, including both multinode roles. +func upgradeWorkerEditError(objects []unstructured.Unstructured, workers []string) error { + edited := map[string]bool{} + for _, object := range objects { + component := object.GetLabels()["nvidia.com/dynamo-component"] + switch object.GetKind() { + case "Deployment": + edited[component] = edited[component] || upgradeTemplateHasEdit(object.Object, "spec", "template") + case "LeaderWorkerSet": + edited[component] = edited[component] || upgradeLWSHasEdit(object.Object) + case "DisaggregatedSet": + roles, _, err := unstructured.NestedSlice(object.Object, "spec", "roles") + if err != nil { + return err + } + for _, value := range roles { + role := value.(map[string]any) + name := role["name"].(string) + edited[name] = edited[name] || upgradeLWSHasEdit(role) + } + } + } + for _, worker := range workers { + if !edited[worker] { + return fmt.Errorf("worker edit has not reached all templates for %s", worker) + } + } + return nil +} + +func upgradeLWSHasEdit(object map[string]any) bool { + return upgradeTemplateHasEdit(object, "spec", "leaderWorkerTemplate", "leaderTemplate") && + upgradeTemplateHasEdit(object, "spec", "leaderWorkerTemplate", "workerTemplate") +} + +func upgradeTemplateHasEdit(object map[string]any, path ...string) bool { + containers, _, err := unstructured.NestedSlice(object, append(path, "spec", "containers")...) + if err != nil { + return false + } + for _, value := range containers { + container := value.(map[string]any) + if container["name"] != "main" { + continue + } + variables, _, err := unstructured.NestedSlice(container, "env") + if err != nil { + return false + } + for _, value := range variables { + variable := value.(map[string]any) + if variable["name"] == "WORKER_REVISION" && variable["value"] == "2" { + return true + } + } + } + return false +} + +func TestUpgradeMultinodeWorkerEdit(t *testing.T) { + for _, kind := range []string{"LeaderWorkerSet", "DisaggregatedSet"} { + for _, missing := range []string{"none", "leader", "worker", "decode"} { + t.Run(kind+"/"+missing, func(t *testing.T) { + t.Log("Require the edit in both templates of every worker component") + var objects []unstructured.Unstructured + var roles []any + for _, name := range []string{"prefill", "decode"} { + role := map[string]any{"name": name, "spec": map[string]any{}} + for _, template := range []string{"leader", "worker"} { + pod := map[string]any{"spec": map[string]any{"containers": []any{map[string]any{"name": "main"}}}} + if missing != template && missing != name { + require.NoError(t, unstructured.SetNestedSlice(pod, []any{map[string]any{ + "name": "main", "env": []any{map[string]any{"name": "WORKER_REVISION", "value": "2"}}, + }}, "spec", "containers")) + } + require.NoError(t, unstructured.SetNestedMap(role, pod, "spec", "leaderWorkerTemplate", template+"Template")) + } + roles = append(roles, role) + objects = append(objects, unstructured.Unstructured{Object: map[string]any{ + "kind": kind, "metadata": map[string]any{"labels": map[string]any{"nvidia.com/dynamo-component": name}}, "spec": role["spec"], + }}) + } + if kind == "DisaggregatedSet" { + objects = []unstructured.Unstructured{{Object: map[string]any{"kind": kind, "spec": map[string]any{"roles": roles}}}} + } + err := upgradeWorkerEditError(objects, []string{"prefill", "decode"}) + if missing == "none" { + require.NoError(t, err) + } else { + require.Error(t, err) + } + }) + } + } +} diff --git a/deploy/operator/internal/controller/upgrade_workload_matrix_test.go b/deploy/operator/internal/controller/upgrade_workload_matrix_test.go new file mode 100644 index 000000000000..1b555f25a5c7 --- /dev/null +++ b/deploy/operator/internal/controller/upgrade_workload_matrix_test.go @@ -0,0 +1,114 @@ +//go:build clustertest && upgradecontract + +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "path/filepath" + "strings" + "testing" + + api "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func upgradeExpectedDCDs(dgd *api.DynamoGraphDeployment) int { + count := 0 + for _, component := range dgd.Spec.Components { + if dgd.Annotations["nvidia.com/workload-provider"] != "disaggregatedset" || !dynamo.IsWorkerComponent(string(component.ComponentType)) { + count++ + } + } + return count +} + +// Canonical paths must select their provider and topology independently of accepted goldens. +func checkUpgradeWorkloadMatrix(t *testing.T, relative string, dgd *api.DynamoGraphDeployment, objects []unstructured.Unstructured) { + t.Helper() + parts := strings.Split(filepath.ToSlash(relative), "/") + if parts[0] == "history" { + parts = parts[1:] + } + if len(parts) != 5 || parts[4] == "grove" { + return + } + t.Log("Check the canonical Component/DisaggregatedSet topology independently of the golden") + require.Equal(t, parts[1], dgd.Spec.BackendFramework) + require.Equal(t, parts[4], dgd.Annotations["nvidia.com/workload-provider"]) + workers := []string{"worker"} + if parts[2] == "disaggregated" { + workers = []string{"prefill", "decode"} + } + require.Len(t, dgd.Spec.Components, len(workers)+1) + for _, component := range dgd.Spec.Components { + if component.ComponentType == api.ComponentTypeFrontend { + require.Equal(t, "frontend", component.ComponentName) + require.Nil(t, component.Multinode) + continue + } + require.Contains(t, workers, component.ComponentName) + require.Equal(t, component.ComponentName, string(component.ComponentType)) + if parts[3] == "multi" { + require.NotNil(t, component.Multinode) + require.EqualValues(t, 2, component.Multinode.NodeCount) + } else { + require.Nil(t, component.Multinode) + } + } + + // Only the operator's immediate workloads belong to this inventory. + counts := map[string]int{} + for _, object := range objects { + counts[object.GetKind()]++ + if object.GetKind() != "DisaggregatedSet" { + continue + } + require.True(t, metav1.IsControlledBy(&object, dgd)) + roles, found, err := unstructured.NestedSlice(object.Object, "spec", "roles") + require.NoError(t, err) + require.True(t, found) + require.Len(t, roles, len(workers)) + seen := map[string]bool{} + for _, value := range roles { + role := value.(map[string]any) + name := role["name"].(string) + require.Contains(t, workers, name) + require.False(t, seen[name], "duplicate DS role %s", name) + seen[name] = true + size, found, err := unstructured.NestedInt64(role, "spec", "leaderWorkerTemplate", "size") + require.NoError(t, err) + require.True(t, found) + require.EqualValues(t, 2, size) + for _, template := range []string{"leaderTemplate", "workerTemplate"} { + containers, found, err := unstructured.NestedSlice(role, "spec", "leaderWorkerTemplate", template, "spec", "containers") + require.NoError(t, err) + require.True(t, found) + require.NotEmpty(t, containers) + } + } + } + require.Zero(t, counts["PodCliqueSet"]) + if parts[4] == "disaggregatedset" { + require.Equal(t, "disaggregated", parts[2]) + require.Equal(t, "multi", parts[3]) + require.Equal(t, 1, counts["DisaggregatedSet"]) + require.Equal(t, 1, counts["DynamoComponentDeployment"]) + require.Equal(t, 1, counts["Deployment"]) + require.Zero(t, counts["LeaderWorkerSet"], "DS descendants must not be actuated or captured") + } else { + require.Zero(t, counts["DisaggregatedSet"]) + require.Equal(t, len(workers)+1, counts["DynamoComponentDeployment"]) + if parts[3] == "multi" { + require.Equal(t, len(workers), counts["LeaderWorkerSet"]) + require.Equal(t, 1, counts["Deployment"]) + } else { + require.Zero(t, counts["LeaderWorkerSet"]) + require.Equal(t, len(workers)+1, counts["Deployment"]) + } + } +} diff --git a/deploy/operator/internal/testing/mocks/lws/lws.go b/deploy/operator/internal/testing/mocks/lws/lws.go index efc7c2655690..b9346c6240ef 100644 --- a/deploy/operator/internal/testing/mocks/lws/lws.go +++ b/deploy/operator/internal/testing/mocks/lws/lws.go @@ -11,6 +11,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" lwswebhooks "sigs.k8s.io/lws/pkg/webhooks" + dswebhooks "sigs.k8s.io/lws/pkg/webhooks/disaggregatedset" ) const ( @@ -22,13 +23,16 @@ const ( lwsValidatingPath = "/validate-leaderworkerset-x-k8s-io-v1-leaderworkerset" ) -// Setup registers the production LeaderWorkerSet admission handlers. +// Setup registers the production LeaderWorkerSet and DisaggregatedSet admission handlers. func Setup(mgr ctrl.Manager) error { - return lwswebhooks.SetupLeaderWorkerSetWebhook(mgr) + if err := lwswebhooks.SetupLeaderWorkerSetWebhook(mgr); err != nil { + return err + } + return dswebhooks.SetupDisaggregatedSetWebhook(mgr) } -// Configurations returns the admission registrations generated from the -// LeaderWorkerSet webhook markers, restricted to LeaderWorkerSet. +// Configurations returns the workload admission registrations from upstream markers. +// Pod webhooks stay excluded because these tests stop at immediate workloads. func Configurations() webhookconfig.Configurations { fail := admissionregistrationv1.Fail none := admissionregistrationv1.SideEffectClassNone @@ -65,6 +69,15 @@ func Configurations() webhookconfig.Configurations { SideEffects: &none, Rules: []admissionregistrationv1.RuleWithOperations{rule}, ClientConfig: serviceClientConfig(lwsValidatingPath), + }, { + Name: "vdisaggregatedset.kb.io", + AdmissionReviewVersions: []string{"v1"}, + FailurePolicy: &fail, + SideEffects: &none, + Rules: []admissionregistrationv1.RuleWithOperations{{Operations: operations, Rule: admissionregistrationv1.Rule{ + APIGroups: []string{"disaggregatedset.x-k8s.io"}, APIVersions: []string{"v1"}, Resources: []string{"disaggregatedsets"}, + }}}, + ClientConfig: serviceClientConfig("/validate-disaggregatedset-x-k8s-io-v1-disaggregatedset"), }}, }}, } diff --git a/deploy/operator/internal/testing/mocks/lws/lws_test.go b/deploy/operator/internal/testing/mocks/lws/lws_test.go index 013d8a645377..b50649bb192b 100644 --- a/deploy/operator/internal/testing/mocks/lws/lws_test.go +++ b/deploy/operator/internal/testing/mocks/lws/lws_test.go @@ -7,20 +7,21 @@ package lws import "testing" -func TestConfigurationsContainOnlyLeaderWorkerSetWebhooks(t *testing.T) { - t.Log("Build the focused registrations from the upstream LeaderWorkerSet webhook markers") +func TestConfigurationsContainOnlyWorkloadWebhooks(t *testing.T) { + t.Log("Build the focused registrations from the upstream workload webhook markers") configurations := Configurations() if len(configurations.Mutating) != 1 || len(configurations.Mutating[0].Webhooks) != 1 { t.Fatalf("mutating registrations = %#v, want one configuration with one webhook", configurations.Mutating) } - if len(configurations.Validating) != 1 || len(configurations.Validating[0].Webhooks) != 1 { - t.Fatalf("validating registrations = %#v, want one configuration with one webhook", configurations.Validating) + if len(configurations.Validating) != 1 || len(configurations.Validating[0].Webhooks) != 2 { + t.Fatalf("validating registrations = %#v, want one configuration with two webhooks", configurations.Validating) } - t.Log("Verify both registrations retain their service paths and target only LeaderWorkerSets") + t.Log("Verify registrations retain service paths and exclude Pod admission") for name, path := range map[string]*string{ - "mutating": configurations.Mutating[0].Webhooks[0].ClientConfig.Service.Path, - "validating": configurations.Validating[0].Webhooks[0].ClientConfig.Service.Path, + "mutating": configurations.Mutating[0].Webhooks[0].ClientConfig.Service.Path, + "validating": configurations.Validating[0].Webhooks[0].ClientConfig.Service.Path, + "disaggregatedset": configurations.Validating[0].Webhooks[1].ClientConfig.Service.Path, } { if path == nil || *path == "" { t.Fatalf("%s webhook path is empty", name) @@ -29,12 +30,14 @@ func TestConfigurationsContainOnlyLeaderWorkerSetWebhooks(t *testing.T) { for _, rule := range []struct { name string resources []string + want string }{ - {name: "mutating", resources: configurations.Mutating[0].Webhooks[0].Rules[0].Resources}, - {name: "validating", resources: configurations.Validating[0].Webhooks[0].Rules[0].Resources}, + {name: "mutating", resources: configurations.Mutating[0].Webhooks[0].Rules[0].Resources, want: lwsResource}, + {name: "validating", resources: configurations.Validating[0].Webhooks[0].Rules[0].Resources, want: lwsResource}, + {name: "disaggregatedset", resources: configurations.Validating[0].Webhooks[1].Rules[0].Resources, want: "disaggregatedsets"}, } { - if len(rule.resources) != 1 || rule.resources[0] != lwsResource { - t.Fatalf("%s webhook resources = %v, want [%s]", rule.name, rule.resources, lwsResource) + if len(rule.resources) != 1 || rule.resources[0] != rule.want { + t.Fatalf("%s webhook resources = %v, want [%s]", rule.name, rule.resources, rule.want) } } } From 047ced9f3dafe9685e6b925030ba408bbd0e0aec Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 21 Oct 2026 12:25:23 +0200 Subject: [PATCH 2/6] operator/tests: require semantic golden review Document semantic review of controller goldens, normalization and hash bindings. Protect rollout-relevant expectations and keep independent upgrade safety checks and historical inputs intact. Signed-off-by: Dr. Stefan Schimanski --- deploy/operator/internal/controller/AGENTS.md | 6 ++++ .../internal/controller/testdata/README.md | 32 +++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 deploy/operator/internal/controller/testdata/README.md diff --git a/deploy/operator/internal/controller/AGENTS.md b/deploy/operator/internal/controller/AGENTS.md index bbee311cf416..6a7363049366 100644 --- a/deploy/operator/internal/controller/AGENTS.md +++ b/deploy/operator/internal/controller/AGENTS.md @@ -20,3 +20,9 @@ SPDX-License-Identifier: Apache-2.0 - Do not share DGD, DGDR, or other test fixtures between tests. Every fixture must be owned by one test. - Keep fixtures in local variables or constants so their ownership remains visible. - Prefer a shared construction DSL when complex fixtures repeat boilerplate. The DSL should describe the object at a domain level; share the DSL, never the constructed fixture. + +# clusterenv golden review + +- Before changing clusterenv goldens, their normalization, or hash bindings, read [the golden review rules](testdata/README.md). These rules cover all controller goldens, including DCD, DGD, DGDR, and upgrade scenarios. +- Every change to an existing golden requires a field-by-field semantic review. Never replace expected output, recompute expected hashes, or add wildcards merely to make a failing test pass. Regeneration does not establish correctness. +- Treat changes to rollout-relevant hashes, generated names, selectors, and pod templates as potential production rollouts, even when DGD intent and runtime images are unchanged. Investigate the production change first; preserve compatibility or document and validate an intentional migration. Explain each semantic golden change and its upgrade consequences in the PR. diff --git a/deploy/operator/internal/controller/testdata/README.md b/deploy/operator/internal/controller/testdata/README.md new file mode 100644 index 000000000000..d1f527610ee7 --- /dev/null +++ b/deploy/operator/internal/controller/testdata/README.md @@ -0,0 +1,32 @@ + + +# Reviewing clusterenv goldens + +> [!WARNING] +> Every change to an existing clusterenv golden needs a detailed semantic review. +> Replacing expected output with newly observed output can hide a production +> regression. A passing test after regeneration does not establish correctness. + +These rules apply to all controller goldens here, including DCD, DGD, DGDR, and +upgrade scenarios. Review every changed field against the intended behavior and +identify the production change that caused it. Do not bulk-accept `.new` files. + +Pay particular attention to worker-generation hashes, generated resource names, +selectors, and pod-template labels, annotations, environment, and specs. Changes +can replace or roll running workloads even when the DGD intent and runtime images +have not changed. Distinguish an internal bookkeeping digest from a value that +selects an active generation or triggers a rollout; do not assume a hash-only diff +is harmless. + +Never update, recompute, normalize away, or wildcard rollout-relevant expectations +merely to accommodate changed controller output. Investigate the production +change first and preserve compatibility. An intentional behavior change needs an +explanation of affected workloads, upgrade consequences, and any migration or +opt-in, with focused validation. Explain each semantic golden change in the PR; +regeneration alone is not a justification. + +Keep independent safety assertions intact. Historical inputs remain frozen and +advance only through the documented [upgrade history workflow](upgrade/README.md). From 383bd8b0dab5c9571f33c0cf900a35a12cee5dcf Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 21 Oct 2026 12:28:58 +0200 Subject: [PATCH 3/6] fixup! operator/tests: cover multinode LWS and DS upgrades Signed-off-by: Dr. Stefan Schimanski --- .../upgrade_disaggregatedset_clustertest_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/deploy/operator/internal/controller/upgrade_disaggregatedset_clustertest_test.go b/deploy/operator/internal/controller/upgrade_disaggregatedset_clustertest_test.go index 35fcb627bfc1..966082b63eb1 100644 --- a/deploy/operator/internal/controller/upgrade_disaggregatedset_clustertest_test.go +++ b/deploy/operator/internal/controller/upgrade_disaggregatedset_clustertest_test.go @@ -16,9 +16,11 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/intstr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" disaggregatedsetv1 "sigs.k8s.io/lws/api/disaggregatedset/v1" + leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1" ) // Exercise a populated DS graph without fabricating released historical provenance. @@ -79,12 +81,14 @@ func testUpgradeDisaggregatedSetReplay(t *testing.T) { require.NoError(t, upgradeSafetyError(false, before, after)) checkUpgradeWorkloadMatrix(t, "latest/sglang/disaggregated/multi/disaggregatedset", getOnlyClusterDGD(t, t.Context(), env.Client(), namespace), after) - t.Log("The upstream DS admission handler rejects a role rollout strategy accepted by the LWS schema") + t.Log("The upstream DS admission handler rejects a schema-valid rollout that cannot make progress") var workloads disaggregatedsetv1.DisaggregatedSetList require.NoError(t, env.Client().List(t.Context(), &workloads, client.InNamespace(namespace))) require.Len(t, workloads.Items, 1) invalid := workloads.Items[0].DeepCopy() - invalid.Spec.Roles[0].Spec.RolloutStrategy.Type = "Recreate" + invalid.Spec.Roles[0].Spec.RolloutStrategy.RollingUpdateConfiguration = &leaderworkersetv1.RollingUpdateConfiguration{ + MaxSurge: intstr.FromInt(0), MaxUnavailable: intstr.FromInt(0), + } require.ErrorContains(t, env.Client().Update(t.Context(), invalid), "vdisaggregatedset.kb.io") t.Log("The independent assertion rejects changed DS worker templates without a golden comparison") From 6c44d9e104e97db198204c15cafd4928773a1941 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 21 Oct 2026 17:13:08 +0200 Subject: [PATCH 4/6] fixup! operator/tests: cover multinode LWS and DS upgrades Check rendered LWS sizes against each DGD component independently of accepted goldens. Add negative coverage for incorrect sizes and incomplete workload inventory, return contextual errors for malformed DS roles, and link the DEP reference in the fixture README. Signed-off-by: Dr. Stefan Schimanski --- .../controller/testdata/upgrade/README.md | 3 +- .../controller/upgrade_workload_edit_test.go | 33 ++++++- .../upgrade_workload_matrix_test.go | 1 + .../upgrade_workload_topology_test.go | 98 +++++++++++++++++++ 4 files changed, 131 insertions(+), 4 deletions(-) create mode 100644 deploy/operator/internal/controller/upgrade_workload_topology_test.go diff --git a/deploy/operator/internal/controller/testdata/upgrade/README.md b/deploy/operator/internal/controller/testdata/upgrade/README.md index 44a7189312a8..1fa11e8f6334 100644 --- a/deploy/operator/internal/controller/testdata/upgrade/README.md +++ b/deploy/operator/internal/controller/testdata/upgrade/README.md @@ -53,7 +53,8 @@ Formal released history still advances only through the pinned-release workflow. These cases complete 22 of the 27 canonical matrix entries. The five remaining single-node Component cases and the missing-origin historical case remain in -#16650; the incomplete release-matrix gate stays enforced. +[#16650](https://github.com/ai-dynamo/dynamo/issues/16650); the incomplete +release-matrix gate stays enforced. ## Storage and production lifecycle diff --git a/deploy/operator/internal/controller/upgrade_workload_edit_test.go b/deploy/operator/internal/controller/upgrade_workload_edit_test.go index f3d5c560f72e..fa487eaf48c5 100644 --- a/deploy/operator/internal/controller/upgrade_workload_edit_test.go +++ b/deploy/operator/internal/controller/upgrade_workload_edit_test.go @@ -26,9 +26,15 @@ func upgradeWorkerEditError(objects []unstructured.Unstructured, workers []strin if err != nil { return err } - for _, value := range roles { - role := value.(map[string]any) - name := role["name"].(string) + for i, value := range roles { + role, ok := value.(map[string]any) + if !ok { + return fmt.Errorf("DisaggregatedSet %s role %d is not an object", object.GetName(), i) + } + name, ok := role["name"].(string) + if !ok { + return fmt.Errorf("DisaggregatedSet %s role %d has no string name", object.GetName(), i) + } edited[name] = edited[name] || upgradeLWSHasEdit(role) } } @@ -106,3 +112,24 @@ func TestUpgradeMultinodeWorkerEdit(t *testing.T) { } } } + +func TestUpgradeWorkerEditMalformedRoles(t *testing.T) { + for _, test := range []struct { + name string + role any + want string + }{ + {name: "not-an-object", role: "invalid", want: "role 0 is not an object"}, + {name: "missing-name", role: map[string]any{}, want: "role 0 has no string name"}, + {name: "invalid-name", role: map[string]any{"name": int64(1)}, want: "role 0 has no string name"}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Malformed DS roles report context instead of panicking during polling") + object := unstructured.Unstructured{Object: map[string]any{ + "kind": "DisaggregatedSet", "metadata": map[string]any{"name": "workers"}, + "spec": map[string]any{"roles": []any{test.role}}, + }} + require.ErrorContains(t, upgradeWorkerEditError([]unstructured.Unstructured{object}, []string{"prefill"}), test.want) + }) + } +} diff --git a/deploy/operator/internal/controller/upgrade_workload_matrix_test.go b/deploy/operator/internal/controller/upgrade_workload_matrix_test.go index 1b555f25a5c7..21af9e34f0a3 100644 --- a/deploy/operator/internal/controller/upgrade_workload_matrix_test.go +++ b/deploy/operator/internal/controller/upgrade_workload_matrix_test.go @@ -105,6 +105,7 @@ func checkUpgradeWorkloadMatrix(t *testing.T, relative string, dgd *api.DynamoGr require.Equal(t, len(workers)+1, counts["DynamoComponentDeployment"]) if parts[3] == "multi" { require.Equal(t, len(workers), counts["LeaderWorkerSet"]) + require.NoError(t, upgradeLWSSizeError(dgd, objects)) require.Equal(t, 1, counts["Deployment"]) } else { require.Zero(t, counts["LeaderWorkerSet"]) diff --git a/deploy/operator/internal/controller/upgrade_workload_topology_test.go b/deploy/operator/internal/controller/upgrade_workload_topology_test.go new file mode 100644 index 000000000000..6560534339c4 --- /dev/null +++ b/deploy/operator/internal/controller/upgrade_workload_topology_test.go @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "fmt" + "testing" + + api "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// Compare every multinode component's rendered LWS size against its DGD intent, not its golden. +func upgradeLWSSizeError(dgd *api.DynamoGraphDeployment, objects []unstructured.Unstructured) error { + for _, component := range dgd.Spec.Components { + if !component.IsMultinode() { + continue + } + + // Match the logical component even when workload names carry a worker hash. + matches := 0 + for _, object := range objects { + if object.GetKind() != "LeaderWorkerSet" || object.GetLabels()[consts.KubeLabelDynamoComponent] != component.ComponentName { + continue + } + matches++ + size, found, err := unstructured.NestedInt64(object.Object, "spec", "leaderWorkerTemplate", "size") + if err != nil { + return fmt.Errorf("LeaderWorkerSet %s for component %s has invalid size: %w", object.GetName(), component.ComponentName, err) + } + if !found { + return fmt.Errorf("LeaderWorkerSet %s for component %s has no leaderWorkerTemplate.size", object.GetName(), component.ComponentName) + } + if size != int64(component.Multinode.NodeCount) { + return fmt.Errorf("LeaderWorkerSet %s for component %s has size %d, want %d", object.GetName(), component.ComponentName, size, component.Multinode.NodeCount) + } + } + if matches != 1 { + return fmt.Errorf("component %s has %d LeaderWorkerSets, want 1", component.ComponentName, matches) + } + } + return nil +} + +func TestUpgradeLWSSize(t *testing.T) { + for _, test := range []struct { + name string + sizes []any + duplicate bool + want string + }{ + {name: "correct", sizes: []any{int64(2), int64(3)}}, + {name: "lowered-to-single-node", sizes: []any{int64(1), int64(3)}, want: "has size 1, want 2"}, + {name: "per-component-size", sizes: []any{int64(2), int64(2)}, want: "has size 2, want 3"}, + {name: "missing-size", sizes: []any{int64(2), nil}, want: "has no leaderWorkerTemplate.size"}, + {name: "invalid-size", sizes: []any{int64(2), "3"}, want: "has invalid size"}, + {name: "missing-component", sizes: []any{int64(2)}, want: "component decode has 0 LeaderWorkerSets"}, + {name: "duplicate-component", sizes: []any{int64(2), int64(3)}, duplicate: true, want: "component prefill has 2 LeaderWorkerSets"}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Require one correctly sized LWS for each multinode component independently of goldens") + dgd := &api.DynamoGraphDeployment{Spec: api.DynamoGraphDeploymentSpec{ + Components: []api.DynamoComponentDeploymentSharedSpec{ + {ComponentName: "frontend", ComponentType: api.ComponentTypeFrontend}, + {ComponentName: "prefill", ComponentType: api.ComponentTypePrefill, Multinode: &api.MultinodeSpec{NodeCount: 2}}, + {ComponentName: "decode", ComponentType: api.ComponentTypeDecode, Multinode: &api.MultinodeSpec{NodeCount: 3}}, + }, + }} + var objects []unstructured.Unstructured + for i, size := range test.sizes { + name := dgd.Spec.Components[i+1].ComponentName + object := unstructured.Unstructured{Object: map[string]any{ + "kind": "LeaderWorkerSet", "metadata": map[string]any{ + "name": "workers-" + name, "labels": map[string]any{consts.KubeLabelDynamoComponent: name}, + }, + }} + if size != nil { + require.NoError(t, unstructured.SetNestedField(object.Object, size, "spec", "leaderWorkerTemplate", "size")) + } + objects = append(objects, object) + } + if test.duplicate { + objects = append(objects, *objects[0].DeepCopy()) + } + + t.Log("A wrong rendered topology must fail even when that output is accepted as a golden") + err := upgradeLWSSizeError(dgd, objects) + if test.want == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, test.want) + } + }) + } +} From 64d8fbc5d5ee1bd2e3e7eff29f47d2aa24011c67 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 21 Oct 2026 18:52:16 +0200 Subject: [PATCH 5/6] operator/ci: clarify lowering and upgrade job name Signed-off-by: Dr. Stefan Schimanski --- .github/workflows/operator-upgrade-contract.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/operator-upgrade-contract.yml b/.github/workflows/operator-upgrade-contract.yml index a276ecdedc38..b55bb560ac17 100644 --- a/.github/workflows/operator-upgrade-contract.yml +++ b/.github/workflows/operator-upgrade-contract.yml @@ -46,7 +46,7 @@ jobs: output.write(f'operator={str(changed).lower()}\nbase={base}\n') PY replay: - name: Historical Grove takeover + name: Operator lowering and upgrade tests needs: changes if: needs.changes.outputs.operator == 'true' runs-on: ubuntu-latest From a62f0ae71383b040fcbd0e119fbb789f6b9deb00 Mon Sep 17 00:00:00 2001 From: xianlubird Date: Mon, 26 Oct 2026 17:10:17 +0800 Subject: [PATCH 6/6] test(operator): cover SGLang single-node disaggregated upgrades Signed-off-by: xianlubird --- .../controller/testdata/upgrade/README.md | 2 +- .../disaggregated/single/component/input.yaml | 65 + .../single/component/output.yaml | 1045 +++++++++++++++++ .../controller/upgrade_clustertest_test.go | 7 +- .../controller/upgrade_component_test.go | 22 +- .../controller/upgrade_workload_edit_test.go | 42 +- 6 files changed, 1167 insertions(+), 16 deletions(-) create mode 100644 deploy/operator/internal/controller/testdata/upgrade/latest/sglang/disaggregated/single/component/input.yaml create mode 100644 deploy/operator/internal/controller/testdata/upgrade/latest/sglang/disaggregated/single/component/output.yaml diff --git a/deploy/operator/internal/controller/testdata/upgrade/README.md b/deploy/operator/internal/controller/testdata/upgrade/README.md index 1fa11e8f6334..e5cdfcd7416f 100644 --- a/deploy/operator/internal/controller/testdata/upgrade/README.md +++ b/deploy/operator/internal/controller/testdata/upgrade/README.md @@ -51,7 +51,7 @@ The populated graph replay controls cover Deployment, LWS and DS graphs without claiming that these freshly captured test graphs came from an older release. Formal released history still advances only through the pinned-release workflow. -These cases complete 22 of the 27 canonical matrix entries. The five remaining +These cases complete 23 of the 27 canonical matrix entries. The four remaining single-node Component cases and the missing-origin historical case remain in [#16650](https://github.com/ai-dynamo/dynamo/issues/16650); the incomplete release-matrix gate stays enforced. diff --git a/deploy/operator/internal/controller/testdata/upgrade/latest/sglang/disaggregated/single/component/input.yaml b/deploy/operator/internal/controller/testdata/upgrade/latest/sglang/disaggregated/single/component/input.yaml new file mode 100644 index 000000000000..145a420288d9 --- /dev/null +++ b/deploy/operator/internal/controller/testdata/upgrade/latest/sglang/disaggregated/single/component/input.yaml @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + name: workers + namespace: upgrade-latest-sglang-disaggregated-single-component + annotations: + nvidia.com/enable-grove: 'false' + nvidia.com/dynamo-operator-origin-version: 0.0.0 +spec: + backendFramework: sglang + components: + - name: frontend + type: frontend + replicas: 1 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + command: + - python3 + - -m + - dynamo.frontend + - name: prefill + type: prefill + replicas: 2 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + command: + - python3 + - -m + - dynamo.sglang + args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - prefill + resources: + limits: + nvidia.com/gpu: '1' + - name: decode + type: decode + replicas: 2 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + command: + - python3 + - -m + - dynamo.sglang + args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - decode + resources: + limits: + nvidia.com/gpu: '1' diff --git a/deploy/operator/internal/controller/testdata/upgrade/latest/sglang/disaggregated/single/component/output.yaml b/deploy/operator/internal/controller/testdata/upgrade/latest/sglang/disaggregated/single/component/output.yaml new file mode 100644 index 000000000000..03546b76bc6f --- /dev/null +++ b/deploy/operator/internal/controller/testdata/upgrade/latest/sglang/disaggregated/single/component/output.yaml @@ -0,0 +1,1045 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + generation: $ignore + labels: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-single-component-workers + nvidia.com/dynamo-sub-component-type: decode + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-decode-{{worker-hash}} + namespace: upgrade-latest-sglang-disaggregated-single-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: $var:worker-string:workers-decode-{{worker-hash}} + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + progressDeadlineSeconds: 600 + replicas: 2 + revisionHistoryLimit: 10 + selector: + matchLabels: + nvidia.com/selector: $var:worker-string:workers-decode-{{worker-hash}} + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-component-type: decode + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-single-component-workers + nvidia.com/dynamo-sub-component-type: decode + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + nvidia.com/selector: $var:worker-string:workers-decode-{{worker-hash}} + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - decode + command: + - python3 + - -m + - dynamo.sglang + env: + - name: DYN_COMPONENT + value: decode + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-sglang-disaggregated-single-component-workers + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-sglang-disaggregated-single-component + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.uid + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /live + port: system + scheme: HTTP + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 4 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: system + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 4 + resources: + limits: + nvidia.com/gpu: '1' + startupProbe: + failureThreshold: 720 + httpGet: + path: /live + port: system + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 1000 + serviceAccount: workers-k8s-service-discovery + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory +status: + $ignore: true +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + generation: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-single-component-workers + name: workers-frontend + namespace: upgrade-latest-sglang-disaggregated-single-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: workers-frontend + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + nvidia.com/selector: workers-frontend + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-component-type: frontend + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-single-component-workers + nvidia.com/metrics-enabled: 'true' + nvidia.com/selector: workers-frontend + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + env: + - name: DYNAMO_PORT + value: '8000' + - name: DYN_COMPONENT + value: frontend + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_HTTP_PORT + value: '8000' + - name: DYN_NAMESPACE + value: upgrade-latest-sglang-disaggregated-single-component-workers + - name: DYN_NAMESPACE_PREFIX + value: upgrade-latest-sglang-disaggregated-single-component-workers + - name: DYN_NAMESPACE_PREFIX_STRICT + value: 'true' + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-sglang-disaggregated-single-component + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.uid + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /live + port: http + scheme: HTTP + initialDelaySeconds: 15 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: main + ports: + - containerPort: 8000 + name: http + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: http + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 3 + resources: {} + startupProbe: + failureThreshold: 30 + httpGet: + path: /live + port: http + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 1000 + serviceAccount: workers-k8s-service-discovery + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory +status: + $ignore: true +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + generation: $ignore + labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-single-component-workers + nvidia.com/dynamo-sub-component-type: prefill + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-prefill-{{worker-hash}} + namespace: upgrade-latest-sglang-disaggregated-single-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: $var:worker-string:workers-prefill-{{worker-hash}} + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + progressDeadlineSeconds: 600 + replicas: 2 + revisionHistoryLimit: 10 + selector: + matchLabels: + nvidia.com/selector: $var:worker-string:workers-prefill-{{worker-hash}} + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-component-type: prefill + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-single-component-workers + nvidia.com/dynamo-sub-component-type: prefill + nvidia.com/dynamo-worker-hash: $var:worker-hash + nvidia.com/metrics-enabled: 'true' + nvidia.com/selector: $var:worker-string:workers-prefill-{{worker-hash}} + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - prefill + command: + - python3 + - -m + - dynamo.sglang + env: + - name: DYN_COMPONENT + value: prefill + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: '20380' + - name: DYN_HEALTH_CHECK_ENABLED + value: 'true' + - name: DYN_NAMESPACE + value: upgrade-latest-sglang-disaggregated-single-component-workers + - name: DYN_NAMESPACE_WORKER_SUFFIX + value: $var:worker-hash + - name: DYN_PARENT_DGD_K8S_NAME + value: workers + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: upgrade-latest-sglang-disaggregated-single-component + - name: DYN_SYSTEM_ENABLED + value: 'true' + - name: DYN_SYSTEM_PORT + value: '9090' + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: NIXL_TELEMETRY_ENABLE + value: n + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: '19090' + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.uid + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /live + port: system + scheme: HTTP + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 4 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: system + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 4 + resources: + limits: + nvidia.com/gpu: '1' + startupProbe: + failureThreshold: 720 + httpGet: + path: /live + port: system + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 1000 + serviceAccount: workers-k8s-service-discovery + serviceAccountName: workers-k8s-service-discovery + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory +status: + $ignore: true +--- +apiVersion: nvidia.com/v1beta1 +kind: DynamoComponentDeployment +metadata: + annotations: + nvidia.com/dcd-spec: '{"version":1,"spec":{"resources":{"limits":{"gpu":"1"}},"extraPodSpec":{"mainContainer":{"name":"","resources":{}}}},"rawJSON":[{"path":"serviceName","nil":true}]}' + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + labels: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-single-component-workers + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-decode-{{worker-hash}} + namespace: upgrade-latest-sglang-disaggregated-single-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: sglang + name: decode + podTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + labels: + nvidia.com/dynamo-worker-hash: $var:worker-hash + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - decode + command: + - python3 + - -m + - dynamo.sglang + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + name: main + resources: + limits: + nvidia.com/gpu: '1' + replicas: 2 + type: decode +status: + component: + $strict: false + componentKind: Deployment + componentNames: + - $var:worker-string:workers-decode-{{worker-hash}} + runtimeNamespace: $var:worker-string:upgrade-latest-sglang-disaggregated-single-component-workers-{{worker-hash}} + servedModelName: Qwen/Qwen3-0.6B + conditions: + - lastTransitionTime: $ignore + message: $ignore + reason: $ignore + status: 'False' + type: Available + - lastTransitionTime: $ignore + message: DynamoComponent is not ready + reason: ComponentNotReady + status: 'False' + type: DynamoComponentReady + observedGeneration: 1 +--- +apiVersion: nvidia.com/v1beta1 +kind: DynamoComponentDeployment +metadata: + annotations: + nvidia.com/dcd-spec: '{"version":1,"spec":{},"rawJSON":[{"path":"serviceName","nil":true}]}' + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-single-component-workers + name: workers-frontend + namespace: upgrade-latest-sglang-disaggregated-single-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: sglang + name: frontend + podTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + name: main + replicas: 1 + type: frontend +status: + component: + $strict: false + componentKind: Deployment + componentNames: + - workers-frontend + runtimeNamespace: upgrade-latest-sglang-disaggregated-single-component-workers + conditions: + - lastTransitionTime: $ignore + message: $ignore + reason: $ignore + status: 'False' + type: Available + - lastTransitionTime: $ignore + message: DynamoComponent is not ready + reason: ComponentNotReady + status: 'False' + type: DynamoComponentReady + observedGeneration: 1 +--- +apiVersion: nvidia.com/v1beta1 +kind: DynamoComponentDeployment +metadata: + annotations: + nvidia.com/dcd-spec: '{"version":1,"spec":{"resources":{"limits":{"gpu":"1"}},"extraPodSpec":{"mainContainer":{"name":"","resources":{}}}},"rawJSON":[{"path":"serviceName","nil":true}]}' + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-single-component-workers + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-prefill-{{worker-hash}} + namespace: upgrade-latest-sglang-disaggregated-single-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: sglang + name: prefill + podTemplate: + metadata: + annotations: + nvidia.com/dynamo-operator-origin-version: $var:operator-version + labels: + nvidia.com/dynamo-worker-hash: $var:worker-hash + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - prefill + command: + - python3 + - -m + - dynamo.sglang + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + name: main + resources: + limits: + nvidia.com/gpu: '1' + replicas: 2 + type: prefill +status: + component: + $strict: false + componentKind: Deployment + componentNames: + - $var:worker-string:workers-prefill-{{worker-hash}} + runtimeNamespace: $var:worker-string:upgrade-latest-sglang-disaggregated-single-component-workers-{{worker-hash}} + servedModelName: Qwen/Qwen3-0.6B + conditions: + - lastTransitionTime: $ignore + message: $ignore + reason: $ignore + status: 'False' + type: Available + - lastTransitionTime: $ignore + message: DynamoComponent is not ready + reason: ComponentNotReady + status: 'False' + type: DynamoComponentReady + observedGeneration: 1 +--- +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + annotations: + nvidia.com/current-worker-hash-v2: $var:worker-hash + nvidia.com/dynamo-operator-origin-version: $var:operator-version + nvidia.com/enable-grove: 'false' + nvidia.com/workload-provider: component + creationTimestamp: $ignore + finalizers: + - nvidia.com/finalizer + generation: $ignore + name: workers + namespace: upgrade-latest-sglang-disaggregated-single-component + resourceVersion: $ignore + uid: $ignore +spec: + backendFramework: sglang + components: + - name: frontend + podTemplate: + spec: + containers: + - command: + - python3 + - -m + - dynamo.frontend + image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0 + name: main + replicas: 1 + type: frontend + - name: prefill + podTemplate: + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - prefill + command: + - python3 + - -m + - dynamo.sglang + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + name: main + resources: + limits: + nvidia.com/gpu: '1' + replicas: 2 + type: prefill + - name: decode + podTemplate: + spec: + containers: + - args: + - --model-path + - Qwen/Qwen3-0.6B + - --disaggregation-mode + - decode + command: + - python3 + - -m + - dynamo.sglang + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0 + name: main + resources: + limits: + nvidia.com/gpu: '1' + replicas: 2 + type: decode +status: + components: + decode: + $strict: false + componentKind: Deployment + componentNames: + - $var:worker-string:workers-decode-{{worker-hash}} + runtimeNamespace: $var:worker-string:upgrade-latest-sglang-disaggregated-single-component-workers-{{worker-hash}} + servedModelName: Qwen/Qwen3-0.6B + frontend: + $strict: false + componentKind: Deployment + componentNames: + - workers-frontend + runtimeNamespace: upgrade-latest-sglang-disaggregated-single-component-workers + prefill: + $strict: false + componentKind: Deployment + componentNames: + - $var:worker-string:workers-prefill-{{worker-hash}} + runtimeNamespace: $var:worker-string:upgrade-latest-sglang-disaggregated-single-component-workers-{{worker-hash}} + servedModelName: Qwen/Qwen3-0.6B + conditions: + - lastTransitionTime: $ignore + message: '$var:worker-string:Resources not ready: workers-decode-{{worker-hash}}: + Component deployment not ready - Available condition not true; workers-frontend: + Component deployment not ready - Available condition not true; workers-prefill-{{worker-hash}}: + Component deployment not ready - Available condition not true' + observedGeneration: 1 + reason: some_resources_are_not_ready + status: 'False' + type: Ready + observedGeneration: 1 + state: pending +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: 0d454cc8266ac38ed6453ee2ef3cf6d9dd237faebc63af22f67ca911e106796c + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery-role + namespace: upgrade-latest-sglang-disaggregated-single-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +rules: +- apiGroups: + - '' + resources: + - endpoints + - pods + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - nvidia.com + resources: + - dynamoworkermetadatas + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: 71c0ad80d1baaed6e4a0485b5fc7c4617c888c970e41aa3d1e54e03c3fff01bb + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery-binding + namespace: upgrade-latest-sglang-disaggregated-single-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: workers-k8s-service-discovery-role +subjects: +- kind: ServiceAccount + name: workers-k8s-service-discovery + namespace: upgrade-latest-sglang-disaggregated-single-component +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-single-component-workers + nvidia.com/dynamo-sub-component-type: decode + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-decode-{{worker-hash}} + namespace: upgrade-latest-sglang-disaggregated-single-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: $var:worker-string:workers-decode-{{worker-hash}} + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + clusterIP: $ignore + clusterIPs: + - $ignore + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: system + port: 9090 + protocol: TCP + targetPort: system + selector: + nvidia.com/dynamo-component: decode + nvidia.com/dynamo-component-type: decode + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-single-component-workers + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-single-component-workers + name: workers-frontend + namespace: upgrade-latest-sglang-disaggregated-single-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: workers-frontend + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + clusterIP: $ignore + clusterIPs: + - $ignore + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: http + port: 8000 + protocol: TCP + targetPort: http + selector: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-component-type: frontend + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-single-component-workers + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: $ignore + creationTimestamp: $ignore + labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-discovery-backend: kubernetes + nvidia.com/dynamo-discovery-enabled: 'true' + nvidia.com/dynamo-graph-deployment-name: workers + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-single-component-workers + nvidia.com/dynamo-sub-component-type: prefill + nvidia.com/dynamo-worker-hash: $var:worker-hash + name: $var:worker-string:workers-prefill-{{worker-hash}} + namespace: upgrade-latest-sglang-disaggregated-single-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoComponentDeployment + name: $var:worker-string:workers-prefill-{{worker-hash}} + uid: $ignore + resourceVersion: $ignore + uid: $ignore +spec: + clusterIP: $ignore + clusterIPs: + - $ignore + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: system + port: 9090 + protocol: TCP + targetPort: system + selector: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-component-type: prefill + nvidia.com/dynamo-namespace: upgrade-latest-sglang-disaggregated-single-component-workers + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + nvidia.com/last-applied-generation: '1' + nvidia.com/last-applied-hash: 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a + creationTimestamp: $ignore + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/managed-by: dynamo-operator + app.kubernetes.io/name: workers-k8s-service-discovery + name: workers-k8s-service-discovery + namespace: upgrade-latest-sglang-disaggregated-single-component + ownerReferences: + - apiVersion: nvidia.com/v1beta1 + blockOwnerDeletion: true + controller: true + kind: DynamoGraphDeployment + name: workers + uid: $ignore + resourceVersion: $ignore + uid: $ignore diff --git a/deploy/operator/internal/controller/upgrade_clustertest_test.go b/deploy/operator/internal/controller/upgrade_clustertest_test.go index 86e1f1a1ba63..e9a39efb39c0 100644 --- a/deploy/operator/internal/controller/upgrade_clustertest_test.go +++ b/deploy/operator/internal/controller/upgrade_clustertest_test.go @@ -23,6 +23,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/clusterenv" "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/golden" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -309,9 +310,11 @@ func TestClusterOperatorUpgrade(t *testing.T) { container := ¤t.Spec.Components[i].PodTemplate.Spec.Containers[0] container.Env = append(container.Env, corev1.EnvVar{Name: "WORKER_REVISION", Value: "2"}) } + desiredHash, err := dynamo.ComputeDGDWorkersSpecHash(current) + require.NoError(t, err) require.NoError(t, env.Client().Update(ctx, current)) - require.Eventually(t, func() bool { - return upgradeWorkerEditError(captureUpgrade(t, env), workers) == nil + require.EventuallyWithT(t, func(c *assert.CollectT) { + assert.NoError(c, upgradeWorkerEditError(captureUpgrade(t, env), workers, desiredHash)) }, 60*time.Second, 200*time.Millisecond, "worker edit did not reach every immediate template") require.Error(t, upgradeSafetyError(false, after, captureUpgrade(t, env))) } diff --git a/deploy/operator/internal/controller/upgrade_component_test.go b/deploy/operator/internal/controller/upgrade_component_test.go index f3d05ba63167..5808064e4a5e 100644 --- a/deploy/operator/internal/controller/upgrade_component_test.go +++ b/deploy/operator/internal/controller/upgrade_component_test.go @@ -83,18 +83,22 @@ func TestUpgradeComponentSafety(t *testing.T) { changed := old.DeepCopy() changed.SetGeneration(2) require.ErrorContains(t, upgradeSafetyError(false, before, []unstructured.Unstructured{*changed}), "generation") - changed = old.DeepCopy() - path := []string{"spec", "template", "spec", "hostname"} + paths := [][]string{{"spec", "template", "spec", "hostname"}} if kind == "LeaderWorkerSet" { - path = []string{"spec", "leaderWorkerTemplate", "workerTemplate", "spec", "hostname"} + paths = [][]string{ + {"spec", "leaderWorkerTemplate", "leaderTemplate", "spec", "hostname"}, + {"spec", "leaderWorkerTemplate", "workerTemplate", "spec", "hostname"}, + } } - if kind == "DisaggregatedSet" { - path = []string{"spec", "roles"} - require.NoError(t, unstructured.SetNestedSlice(changed.Object, []any{map[string]any{"name": "changed-role"}}, path...)) - } else { - require.NoError(t, unstructured.SetNestedField(changed.Object, "changed-worker", path...)) + for _, path := range paths { + changed = old.DeepCopy() + if kind == "DisaggregatedSet" { + require.NoError(t, unstructured.SetNestedSlice(changed.Object, []any{map[string]any{"name": "changed-role"}}, "spec", "roles")) + } else { + require.NoError(t, unstructured.SetNestedField(changed.Object, "changed-worker", path...)) + } + require.ErrorContains(t, upgradeSafetyError(false, before, []unstructured.Unstructured{*changed}), "rollout intent") } - require.ErrorContains(t, upgradeSafetyError(false, before, []unstructured.Unstructured{*changed}), "rollout intent") require.ErrorContains(t, upgradeSafetyError(false, before, nil), "inventory") }) } diff --git a/deploy/operator/internal/controller/upgrade_workload_edit_test.go b/deploy/operator/internal/controller/upgrade_workload_edit_test.go index fa487eaf48c5..e21d85f42ffc 100644 --- a/deploy/operator/internal/controller/upgrade_workload_edit_test.go +++ b/deploy/operator/internal/controller/upgrade_workload_edit_test.go @@ -12,10 +12,23 @@ import ( ) // Every edited component must reach its immediate workload, including both multinode roles. -func upgradeWorkerEditError(objects []unstructured.Unstructured, workers []string) error { +func upgradeWorkerEditError(objects []unstructured.Unstructured, workers []string, desiredHash string) error { edited := map[string]bool{} + matched := map[string]int{} for _, object := range objects { component := object.GetLabels()["nvidia.com/dynamo-component"] + + // Component workloads must belong to the requested worker generation. + if object.GetKind() == "Deployment" || object.GetKind() == "LeaderWorkerSet" { + if object.GetLabels()["nvidia.com/dynamo-worker-hash"] != desiredHash { + continue + } + matched[component]++ + if matched[component] > 1 { + return fmt.Errorf("component %s has multiple workloads for worker hash %s", component, desiredHash) + } + } + switch object.GetKind() { case "Deployment": edited[component] = edited[component] || upgradeTemplateHasEdit(object.Object, "spec", "template") @@ -96,13 +109,13 @@ func TestUpgradeMultinodeWorkerEdit(t *testing.T) { } roles = append(roles, role) objects = append(objects, unstructured.Unstructured{Object: map[string]any{ - "kind": kind, "metadata": map[string]any{"labels": map[string]any{"nvidia.com/dynamo-component": name}}, "spec": role["spec"], + "kind": kind, "metadata": map[string]any{"labels": map[string]any{"nvidia.com/dynamo-component": name, "nvidia.com/dynamo-worker-hash": "new"}}, "spec": role["spec"], }}) } if kind == "DisaggregatedSet" { objects = []unstructured.Unstructured{{Object: map[string]any{"kind": kind, "spec": map[string]any{"roles": roles}}}} } - err := upgradeWorkerEditError(objects, []string{"prefill", "decode"}) + err := upgradeWorkerEditError(objects, []string{"prefill", "decode"}, "new") if missing == "none" { require.NoError(t, err) } else { @@ -129,7 +142,28 @@ func TestUpgradeWorkerEditMalformedRoles(t *testing.T) { "kind": "DisaggregatedSet", "metadata": map[string]any{"name": "workers"}, "spec": map[string]any{"roles": []any{test.role}}, }} - require.ErrorContains(t, upgradeWorkerEditError([]unstructured.Unstructured{object}, []string{"prefill"}), test.want) + require.ErrorContains(t, upgradeWorkerEditError([]unstructured.Unstructured{object}, []string{"prefill"}, "new"), test.want) }) } } + +func TestUpgradeWorkerEditGeneration(t *testing.T) { + t.Log("An edited Deployment from an old generation must not satisfy a new worker edit") + object := unstructured.Unstructured{Object: map[string]any{ + "kind": "Deployment", "metadata": map[string]any{"labels": map[string]any{ + "nvidia.com/dynamo-component": "prefill", "nvidia.com/dynamo-worker-hash": "old", + }}, + }} + require.NoError(t, unstructured.SetNestedSlice(object.Object, []any{map[string]any{ + "name": "main", "env": []any{map[string]any{"name": "WORKER_REVISION", "value": "2"}}, + }}, "spec", "template", "spec", "containers")) + require.NoError(t, upgradeWorkerEditError([]unstructured.Unstructured{object}, []string{"prefill"}, "old")) + require.Error(t, upgradeWorkerEditError([]unstructured.Unstructured{object}, []string{"prefill"}, "new")) + + t.Log("Require exactly one edited workload for each worker in the requested generation") + current := object.DeepCopy() + current.SetLabels(map[string]string{"nvidia.com/dynamo-component": "prefill", "nvidia.com/dynamo-worker-hash": "new"}) + require.NoError(t, upgradeWorkerEditError([]unstructured.Unstructured{object, *current}, []string{"prefill"}, "new")) + require.Error(t, upgradeWorkerEditError([]unstructured.Unstructured{*current, *current}, []string{"prefill"}, "new")) + require.Error(t, upgradeWorkerEditError([]unstructured.Unstructured{*current}, []string{"prefill", "decode"}, "new")) +}