test(operator): cover SGLang single-node disaggregated upgrades

.github/workflows/operator-upgrade-contract.yml

@@ -46,7 +46,7 @@ jobs:
               output.write(f'operator={str(changed).lower()}\nbase={base}\n')
           PY
   replay:
-    name: Historical Grove takeover
+    name: Operator lowering and upgrade tests
     needs: changes
     if: needs.changes.outputs.operator == 'true'
     runs-on: ubuntu-latest

deploy/operator/hack/generate_upgrade_fixtures.py

@@ -260,7 +260,7 @@ def golden_objects(objects, latest=False):
                 annotations = metadata.get("annotations", {})
                 if "nvidia.com/last-applied-hash" in annotations:
                     annotations["nvidia.com/last-applied-hash"] = "$ignore"
-            if obj["kind"] in ("Deployment", "LeaderWorkerSet"):
+            if obj["kind"] in ("Deployment", "LeaderWorkerSet") and "status" in obj:
                 # Workload-controller availability is outside the operator lowering boundary.
                 obj["status"] = {"$ignore": True}
             if obj["kind"] in ("DynamoComponentDeployment", "DynamoGraphDeployment"):
@@ -328,6 +328,8 @@ def bind_latest_variables(value):
         for key, child in value.items():
             if key == "nvidia.com/dynamo-operator-origin-version":
                 value[key] = "$var:operator-version"
+            elif key == "disaggregatedset.x-k8s.io/revision":
+                value[key] = "$var:ds-revision"
             elif key in (
                 "nvidia.com/current-worker-hash-v2",
                 "nvidia.com/dynamo-worker-hash",

deploy/operator/hack/test_upgrade_component_status.py

@@ -10,6 +10,38 @@
 
 
 class UpgradeComponentStatusTest(unittest.TestCase):
+    def test_absent_workload_status_is_not_manufactured(self):
+        path = (
+            fixtures.FIXTURES / "latest/sglang/aggregated/multi/component/output.yaml"
+        )
+        objects = list(yaml.safe_load_all(path.read_text()))
+        workload = next(obj for obj in objects if obj["kind"] == "LeaderWorkerSet")
+        workload.pop("status", None)
+
+        # No LWS controller runs at the immediate-workload boundary.
+        normalized = fixtures.golden_objects(objects, latest=True)
+        self.assertNotIn(
+            "status",
+            next(obj for obj in normalized if obj["kind"] == "LeaderWorkerSet"),
+        )
+
+        # An observed runtime status is ignored without changing the historical capture.
+        workload["status"] = {"readyReplicas": 0}
+        normalized = fixtures.golden_objects(objects, latest=True)
+        self.assertEqual(
+            next(obj for obj in normalized if obj["kind"] == "LeaderWorkerSet")[
+                "status"
+            ],
+            {"$ignore": True},
+        )
+        historical = fixtures.golden_objects(objects)
+        self.assertEqual(
+            next(obj for obj in historical if obj["kind"] == "LeaderWorkerSet")[
+                "status"
+            ],
+            {"readyReplicas": 0},
+        )
+
     def test_runtime_status_does_not_change_component_golden(self):
         path = (
             fixtures.FIXTURES / "latest/sglang/aggregated/single/component/output.yaml"

deploy/operator/internal/controller/AGENTS.md

@@ -20,3 +20,9 @@ SPDX-License-Identifier: Apache-2.0
 - Do not share DGD, DGDR, or other test fixtures between tests. Every fixture must be owned by one test.
 - Keep fixtures in local variables or constants so their ownership remains visible.
 - Prefer a shared construction DSL when complex fixtures repeat boilerplate. The DSL should describe the object at a domain level; share the DSL, never the constructed fixture.
+
+# clusterenv golden review
+
+- Before changing clusterenv goldens, their normalization, or hash bindings, read [the golden review rules](testdata/README.md). These rules cover all controller goldens, including DCD, DGD, DGDR, and upgrade scenarios.
+- Every change to an existing golden requires a field-by-field semantic review. Never replace expected output, recompute expected hashes, or add wildcards merely to make a failing test pass. Regeneration does not establish correctness.
+- Treat changes to rollout-relevant hashes, generated names, selectors, and pod templates as potential production rollouts, even when DGD intent and runtime images are unchanged. Investigate the production change first; preserve compatibility or document and validate an intentional migration. Explain each semantic golden change and its upgrade consequences in the PR.

deploy/operator/internal/controller/clusterenv_helpers_test.go

@@ -29,6 +29,7 @@ import (
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/utils/ptr"
 	gaiev1 "sigs.k8s.io/gateway-api-inference-extension/api/v1"
+	disaggregatedsetv1 "sigs.k8s.io/lws/api/disaggregatedset/v1"
 	lwsscheme "sigs.k8s.io/lws/client-go/clientset/versioned/scheme"
 	vcbatchv1alpha1 "volcano.sh/apis/pkg/apis/batch/v1alpha1"
 	volcanov1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
@@ -44,6 +45,7 @@ func newClusterTestScheme() *runtime.Scheme {
 	utilruntime.Must(monitoringv1.AddToScheme(scheme))
 	utilruntime.Must(istioclientsetscheme.AddToScheme(scheme))
 	utilruntime.Must(lwsscheme.AddToScheme(scheme))
+	utilruntime.Must(disaggregatedsetv1.AddToScheme(scheme))
 	utilruntime.Must(gaiev1.Install(scheme))
 	utilruntime.Must(appsv1.AddToScheme(scheme))
 	utilruntime.Must(admissionregistrationv1.AddToScheme(scheme))

deploy/operator/internal/controller/clusterenv_test.go

@@ -58,7 +58,7 @@ func runClusterTests(m *testing.M, prepare func(context.Context, client.Client)
 	operatorConfig := &configv1alpha1.OperatorConfiguration{}
 	configv1alpha1.SetDefaultsOperatorConfiguration(operatorConfig)
 	operatorConfig.GPU.DiscoveryEnabled = ptr.To(false)
-	runtimeConfig := &commoncontroller.RuntimeConfig{Gate: features.Gates{Grove: true, LWS: true}}
+	runtimeConfig := &commoncontroller.RuntimeConfig{Gate: features.Gates{Grove: true, LWS: true, DisaggregatedSet: true}}
 	additionalAdmission := lwsmock.Configurations()
 	groveAdmission := grovemock.Configurations()
 	additionalAdmission.Mutating = append(additionalAdmission.Mutating, groveAdmission.Mutating...)

deploy/operator/internal/controller/testdata/README.md

@@ -0,0 +1,32 @@
+<!--
+SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+-->
+
+# Reviewing clusterenv goldens
+
+> [!WARNING]
+> Every change to an existing clusterenv golden needs a detailed semantic review.
+> Replacing expected output with newly observed output can hide a production
+> regression. A passing test after regeneration does not establish correctness.
+
+These rules apply to all controller goldens here, including DCD, DGD, DGDR, and
+upgrade scenarios. Review every changed field against the intended behavior and
+identify the production change that caused it. Do not bulk-accept `.new` files.
+
+Pay particular attention to worker-generation hashes, generated resource names,
+selectors, and pod-template labels, annotations, environment, and specs. Changes
+can replace or roll running workloads even when the DGD intent and runtime images
+have not changed. Distinguish an internal bookkeeping digest from a value that
+selects an active generation or triggers a rollout; do not assume a hash-only diff
+is harmless.
+
+Never update, recompute, normalize away, or wildcard rollout-relevant expectations
+merely to accommodate changed controller output. Investigate the production
+change first and preserve compatibility. An intentional behavior change needs an
+explanation of affected workloads, upgrade consequences, and any migration or
+opt-in, with focused validation. Explain each semantic golden change in the PR;
+regeneration alone is not a justification.
+
+Keep independent safety assertions intact. Historical inputs remain frozen and
+advance only through the documented [upgrade history workflow](upgrade/README.md).

deploy/operator/internal/controller/testdata/upgrade/README.md

@@ -33,6 +33,29 @@ PodSpec changes remain protected. The independent detector test rejects a new
 generation even if the output is identical. An intentional worker env edit must
 still reach a new workload generation after takeover.
 
+## Multinode Component and DisaggregatedSet coverage
+
+The canonical suite includes aggregated and disaggregated multinode Component
+scenarios for vLLM, SGLang, and TensorRT-LLM, plus one disaggregated multinode
+DisaggregatedSet scenario per engine. Component scenarios stop at DCD-owned
+LeaderWorkerSets; DisaggregatedSet scenarios stop at the DisaggregatedSet itself
+and the frontend's auxiliary DCD/Deployment. The LWS/DS controllers and serving
+Pods do not run. Dependency CRDs and production admission handlers come from the
+pinned LWS module already used by the shared clustertest setup.
+
+Independent checks enforce the selected provider, component/role inventory and
+multinode shape. Historical safety also protects the complete DisaggregatedSet
+spec, identity and generation. Intentional worker edits must reach every edited
+component and both leader/worker templates, including embedded DS role templates.
+The populated graph replay controls cover Deployment, LWS and DS graphs without
+claiming that these freshly captured test graphs came from an older release.
+Formal released history still advances only through the pinned-release workflow.
+
+These cases complete 23 of the 27 canonical matrix entries. The four remaining
+single-node Component cases and the missing-origin historical case remain in
+[#16650](https://github.com/ai-dynamo/dynamo/issues/16650); the incomplete
+release-matrix gate stays enforced.
+
 ## Storage and production lifecycle
 
 Run this suite only against a dedicated cluster with no installed controllers or
@@ -142,11 +165,11 @@ git -C "$release_tree" restore --source="$harness" -- \
   deploy/operator/internal/controller/clusterenv_helpers_test.go \
   deploy/operator/internal/controller/clusterenv_test.go \
   deploy/operator/internal/controller/clusterenv_main_test.go \
-  deploy/operator/internal/controller/upgrade_clustertest_test.go \
-  deploy/operator/internal/controller/upgrade_contract_test.go \
+  ':(glob)deploy/operator/internal/controller/upgrade_*_test.go' \
   deploy/operator/internal/controller/testdata/upgrade \
   deploy/operator/internal/testing/clusterenv \
-  deploy/operator/internal/testing/golden
+  deploy/operator/internal/testing/golden \
+  deploy/operator/internal/testing/mocks
 make -C "$release_tree/deploy/operator" clustertest-upgrade-fixtures-regen \
   UPGRADE_RELEASE_TAG=v1.6.0
 ```
@@ -209,9 +232,9 @@ across releases; namespace collisions across all roles are rejected. Long
 namespace names include a digest to remain within Kubernetes' length limit.
 
 Latest origin annotations use `$var:operator-version`, bound exactly to the
-operator chart's `appVersion` by the golden matcher. Worker hashes and the PCS
-spec digest also depend on origin: their exact bindings are computed from the
-reviewed golden DGD/PCS specifications with that version, never from candidate
+operator chart's `appVersion` by the golden matcher. Worker hashes, the PCS
+spec digest and the DS role revision also depend on origin: their exact bindings
+are computed from the reviewed golden DGD/PCS/DS specifications with that version, never from candidate
 output. The independent CREATE origin assertion remains active. All other
 lowering fields remain exact, and historical origins and hashes remain concrete. Accepting a golden never bypasses rollout
 safety assertions.

deploy/operator/internal/controller/testdata/upgrade/latest/sglang/aggregated/multi/component/input.yaml

@@ -0,0 +1,45 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+apiVersion: nvidia.com/v1beta1
+kind: DynamoGraphDeployment
+metadata:
+  name: workers
+  namespace: upgrade-latest-sglang-aggregated-multi-component
+  annotations:
+    nvidia.com/enable-grove: 'false'
+    nvidia.com/dynamo-operator-origin-version: 0.0.0
+spec:
+  backendFramework: sglang
+  components:
+  - name: frontend
+    type: frontend
+    replicas: 1
+    podTemplate:
+      spec:
+        containers:
+        - name: main
+          image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.6.0
+          command:
+          - python3
+          - -m
+          - dynamo.frontend
+  - name: worker
+    type: worker
+    replicas: 2
+    podTemplate:
+      spec:
+        containers:
+        - name: main
+          image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.6.0
+          command:
+          - python3
+          - -m
+          - dynamo.sglang
+          args:
+          - --model-path
+          - Qwen/Qwen3-0.6B
+          resources:
+            limits:
+              nvidia.com/gpu: '1'
+    multinode:
+      nodeCount: 2