From 023a7945aa751d95508db87a4b2a245fef2c80c2 Mon Sep 17 00:00:00 2001 From: Julien Mancuso <161955438+julienmancuso@users.noreply.github.com> Date: Mon, 2 Nov 2026 10:07:14 -0700 Subject: [PATCH 1/4] chore(deps): bump Grove to v0.2.0-beta1 Signed-off-by: Julien Mancuso --- deploy/helm/charts/platform/Chart.yaml | 2 +- deploy/helm/charts/platform/README.md | 27 +++++++++++++++++-- deploy/helm/charts/platform/README.md.gotmpl | 23 ++++++++++++++++ .../platform/files/grove-preflight/check.py | 7 ++--- .../platform/tests/grove_preflight_test.yaml | 2 +- .../platform/tests/test_grove_preflight.py | 20 ++++++++------ deploy/helm/charts/platform/values.yaml | 2 +- deploy/operator/Makefile | 2 +- deploy/operator/go.mod | 2 +- deploy/operator/go.sum | 4 +-- .../internal/dynamo/grove_schema_test.go | 2 +- 11 files changed, 72 insertions(+), 21 deletions(-) diff --git a/deploy/helm/charts/platform/Chart.yaml b/deploy/helm/charts/platform/Chart.yaml index ffa145b99bac..1dbe2132d8b7 100644 --- a/deploy/helm/charts/platform/Chart.yaml +++ b/deploy/helm/charts/platform/Chart.yaml @@ -25,7 +25,7 @@ kubeVersion: ">=1.30.0-0" home: https://nvidia.com annotations: # One scalar drives both the bundled dependency and the external-Grove preflight. - dynamo.nvidia.com/grove-version: &groveVersion v0.1.0-alpha.15 + dynamo.nvidia.com/grove-version: &groveVersion v0.2.0-beta1 dependencies: - name: dynamo-operator version: 1.7.0 diff --git a/deploy/helm/charts/platform/README.md b/deploy/helm/charts/platform/README.md index d7b5c5f2e2a2..80d81989376a 100644 --- a/deploy/helm/charts/platform/README.md +++ b/deploy/helm/charts/platform/README.md @@ -173,6 +173,28 @@ sidecar guides. **Existing deployments:** No existing workload rolls. Components without a `runtime` init container keep running the engine and Dynamo together in `main`. +#### Dependency compatibility + +##### Grove v0.2.0-beta1 + +**Change:** The bundled Grove chart and Go API dependency are now `v0.2.0-beta1`. +Grove stores resources as `grove.io/v1beta1` and continues serving the +`grove.io/v1alpha1` API used by Dynamo. The external Grove preflight accepts beta +release tags and requires the pinned version or later. + +**Affected:** All Grove-backed deployments upgrading to Dynamo 1.7.0. + +**Action:** Upgrade externally managed Grove and matching CRDs first, with +`crdInstaller.upgradeCRDs=true`. The platform chart enables this setting for +bundled Grove, replacing `grove.crdInstaller.enabled`. The API server must be able +to reach Grove's conversion webhook. Read the +[Grove upgrade and downgrade guide](https://github.com/ai-dynamo/grove/blob/v0.2.0-beta1/docs/user-guide/upgrading-to-v1beta1-v0.2.0-beta1.md) +before upgrading; rollback requires migrating stored resources back to `v1alpha1`. + +**Existing deployments:** Grove converts existing alpha resources without requiring +manifest changes. An unset clique or scaling-group `scaleConfig.minReplicas` now +defaults to `minAvailable` rather than `replicas`; existing explicit minima are retained. + ### v1.6.0 #### Runtime Compatibility @@ -522,7 +544,7 @@ Kubernetes: `>=1.30.0-0` | file://components/operator | dynamo-operator | 1.7.0 | | https://charts.bitnami.com/bitnami | etcd | 12.0.18 | | https://nats-io.github.io/k8s/helm/charts/ | nats | 1.3.2 | -| oci://ghcr.io/ai-dynamo/grove | grove(grove-charts) | v0.1.0-alpha.15 | +| oci://ghcr.io/ai-dynamo/grove | grove(grove-charts) | v0.2.0-beta1 | | oci://ghcr.io/ai-dynamo/snapshot | snapshot | 0.2.0 | | oci://ghcr.io/kai-scheduler/kai-scheduler | kai-scheduler | v0.17.0 | @@ -611,7 +633,7 @@ Kubernetes: `>=1.30.0-0` | dynamo-operator.checkpoint.enabled | bool | `false` | Whether to enable checkpoint/restore functionality | | grove.tolerations | list | `[]` | Node tolerations for Grove pods | | grove.affinity | object | `{}` | Affinity for Grove pods | -| grove.crdInstaller.enabled | bool | `true` | Upgrade Grove CRDs before the bundled operator starts. | +| grove.crdInstaller.upgradeCRDs | bool | `true` | Upgrade Grove CRDs before the bundled operator starts. | | grove.config.server.healthProbes.enable | bool | `true` | Enable Grove's webhook-aware liveness and readiness probes. The readiness endpoint stays false until certificates and the webhook server are ready. | | kai-scheduler.global.tolerations | list | `[]` | Node tolerations for kai-scheduler pods | | kai-scheduler.global.affinity | object | `{}` | Affinity for kai-scheduler pods | @@ -685,6 +707,7 @@ For **production environments**, Kai Scheduler and Grove should be installed sep | 1.4.x | >= v0.13.4 | >= v0.1.0-alpha.12-rc1 | | 1.5.x | >= v0.17.0 | >= v0.1.0-alpha.13 | | 1.6.x | >= v0.17.0 | >= v0.1.0-alpha.15 | +| 1.7.x | >= v0.17.0 | >= v0.2.0-beta1 | Upgrade Grove in lockstep with Dynamo while Grove APIs are not stable. See the [v1.4.0 upgrade notes](#v140) for the topology API transition and the diff --git a/deploy/helm/charts/platform/README.md.gotmpl b/deploy/helm/charts/platform/README.md.gotmpl index eb1c02ef5a67..24ee251cccab 100644 --- a/deploy/helm/charts/platform/README.md.gotmpl +++ b/deploy/helm/charts/platform/README.md.gotmpl @@ -173,6 +173,28 @@ sidecar guides. **Existing deployments:** No existing workload rolls. Components without a `runtime` init container keep running the engine and Dynamo together in `main`. +#### Dependency compatibility + +##### Grove v0.2.0-beta1 + +**Change:** The bundled Grove chart and Go API dependency are now `v0.2.0-beta1`. +Grove stores resources as `grove.io/v1beta1` and continues serving the +`grove.io/v1alpha1` API used by Dynamo. The external Grove preflight accepts beta +release tags and requires the pinned version or later. + +**Affected:** All Grove-backed deployments upgrading to Dynamo 1.7.0. + +**Action:** Upgrade externally managed Grove and matching CRDs first, with +`crdInstaller.upgradeCRDs=true`. The platform chart enables this setting for +bundled Grove, replacing `grove.crdInstaller.enabled`. The API server must be able +to reach Grove's conversion webhook. Read the +[Grove upgrade and downgrade guide](https://github.com/ai-dynamo/grove/blob/v0.2.0-beta1/docs/user-guide/upgrading-to-v1beta1-v0.2.0-beta1.md) +before upgrading; rollback requires migrating stored resources back to `v1alpha1`. + +**Existing deployments:** Grove converts existing alpha resources without requiring +manifest changes. An unset clique or scaling-group `scaleConfig.minReplicas` now +defaults to `minAvailable` rather than `replicas`; existing explicit minima are retained. + ### v1.6.0 #### Runtime Compatibility @@ -585,6 +607,7 @@ For **production environments**, Kai Scheduler and Grove should be installed sep | 1.4.x | >= v0.13.4 | >= v0.1.0-alpha.12-rc1 | | 1.5.x | >= v0.17.0 | >= v0.1.0-alpha.13 | | 1.6.x | >= v0.17.0 | >= v0.1.0-alpha.15 | +| 1.7.x | >= v0.17.0 | >= v0.2.0-beta1 | Upgrade Grove in lockstep with Dynamo while Grove APIs are not stable. See the [v1.4.0 upgrade notes](#v140) for the topology API transition and the diff --git a/deploy/helm/charts/platform/files/grove-preflight/check.py b/deploy/helm/charts/platform/files/grove-preflight/check.py index 28eea3a26158..3cc8005975da 100644 --- a/deploy/helm/charts/platform/files/grove-preflight/check.py +++ b/deploy/helm/charts/platform/files/grove-preflight/check.py @@ -12,14 +12,15 @@ def version(tag): - # Recognize Grove alpha releases and stable releases; custom/RC tags are unverifiable. - match = re.fullmatch(r"v?(\d+)\.(\d+)\.(\d+)(?:-alpha\.(\d+))?", tag) + # Recognize Grove alpha, beta, and stable releases; custom/RC tags are unverifiable. + match = re.fullmatch(r"v?(\d+)\.(\d+)\.(\d+)(?:-(alpha\.|beta)(\d+))?", tag) if not match: raise ValueError( f"Unverifiable Grove version {tag!r}; use a release-tagged image." ) return tuple(int(n) for n in match.group(1, 2, 3)) + ( - int(match[4]) if match[4] else float("inf"), + {"alpha.": 0, "beta": 1, None: 2}[match[4]], + int(match[5] or 0), ) diff --git a/deploy/helm/charts/platform/tests/grove_preflight_test.yaml b/deploy/helm/charts/platform/tests/grove_preflight_test.yaml index 147b7013b753..630852f5d079 100644 --- a/deploy/helm/charts/platform/tests/grove_preflight_test.yaml +++ b/deploy/helm/charts/platform/tests/grove_preflight_test.yaml @@ -38,7 +38,7 @@ tests: - matchRegex: {path: 'spec.template.spec.containers[0].args[0]', pattern: 'def check\('} - matchRegex: path: spec.template.spec.containers[0].args[1] - pattern: '^v?[0-9]+\.[0-9]+\.[0-9]+(-alpha\.[0-9]+)?$' + pattern: '^v?[0-9]+\.[0-9]+\.[0-9]+(-(alpha\.|beta)[0-9]+)?$' - it: keeps the compatibility check visible in long hook names set: global.grove.enabled: true diff --git a/deploy/helm/charts/platform/tests/test_grove_preflight.py b/deploy/helm/charts/platform/tests/test_grove_preflight.py index 1fc5ba8dfb68..43ed7eb910ff 100644 --- a/deploy/helm/charts/platform/tests/test_grove_preflight.py +++ b/deploy/helm/charts/platform/tests/test_grove_preflight.py @@ -23,13 +23,15 @@ class GrovePreflightTest(unittest.TestCase): - def test_orders_alpha_versions_numerically_before_stable(self): - self.assertLess( - preflight.version("v1.2.3-alpha.9"), preflight.version("v1.2.3-alpha.10") - ) - self.assertLess( - preflight.version("v1.2.3-alpha.10"), preflight.version("v1.2.3") - ) + def test_orders_alpha_beta_and_stable_versions(self): + for earlier, later in ( + ("v1.2.3-alpha.9", "v1.2.3-alpha.10"), + ("v1.2.3-alpha.10", "v1.2.3-beta1"), + ("v1.2.3-beta9", "v1.2.3-beta10"), + ("v1.2.3-beta10", "v1.2.3"), + ): + with self.subTest(earlier=earlier, later=later): + self.assertLess(preflight.version(earlier), preflight.version(later)) def test_uses_the_supplied_minimum(self): higher = f"v{self.major + 1}.0.0" @@ -37,7 +39,7 @@ def test_uses_the_supplied_minimum(self): preflight.check(self.get, higher) def setUp(self): - self.major, minor, patch, _ = preflight.version(MINIMUM) + self.major, minor, patch, *_ = preflight.version(MINIMUM) self.stable = f"v{self.major}.{minor}.{patch}" self.container = { "name": "grove-operator", @@ -63,6 +65,8 @@ def test_accepts_release_and_rejects_old_or_unverifiable_images(self): self.container["image"] = "registry:5000/grove:" + tag preflight.check(self.get, MINIMUM) for image in ( + "grove:v0.1.0-alpha.15", + "grove:v0.2.0-alpha.17", "grove:v0.0.0-alpha.0", "grove:v0.0.0", "grove:latest", diff --git a/deploy/helm/charts/platform/values.yaml b/deploy/helm/charts/platform/values.yaml index a68de01cca42..3bea2054ac66 100644 --- a/deploy/helm/charts/platform/values.yaml +++ b/deploy/helm/charts/platform/values.yaml @@ -318,7 +318,7 @@ grove: affinity: {} crdInstaller: # -- Upgrade Grove CRDs before the bundled operator starts. - enabled: true + upgradeCRDs: true config: server: healthProbes: diff --git a/deploy/operator/Makefile b/deploy/operator/Makefile index 002a263cb8bd..ce76a0119aae 100644 --- a/deploy/operator/Makefile +++ b/deploy/operator/Makefile @@ -25,7 +25,7 @@ CLUSTERTEST_PLATFORM ?= linux/$(GOARCH) CLUSTERTEST_LWS_VERSION = $(shell go list -m -f '{{.Version}}' sigs.k8s.io/lws) CLUSTERTEST_LWS_CRD_DIR = $(shell go env GOMODCACHE)/sigs.k8s.io/lws@$(CLUSTERTEST_LWS_VERSION)/config/crd/bases CLUSTERTEST_GROVE_VERSION = $(shell go list -m -f '{{.Version}}' github.com/ai-dynamo/grove/operator/api) -CLUSTERTEST_GROVE_CRD_DIR = $(shell go env GOMODCACHE)/github.com/ai-dynamo/grove/operator/api@$(CLUSTERTEST_GROVE_VERSION)/core/v1alpha1/crds +CLUSTERTEST_GROVE_CRD_DIR = $(shell go env GOMODCACHE)/github.com/ai-dynamo/grove/operator/api@$(CLUSTERTEST_GROVE_VERSION)/core/crds CLUSTERTEST_VOLCANO_VERSION = $(shell go list -m -f '{{.Version}}' volcano.sh/apis) CLUSTERTEST_VOLCANO_PODGROUP_CRD_URL = https://raw.githubusercontent.com/volcano-sh/volcano/$(CLUSTERTEST_VOLCANO_VERSION)/config/crd/volcano/bases/scheduling.volcano.sh_podgroups.yaml diff --git a/deploy/operator/go.mod b/deploy/operator/go.mod index 97becd0f1dc2..897f58b14c47 100644 --- a/deploy/operator/go.mod +++ b/deploy/operator/go.mod @@ -6,7 +6,7 @@ require ( capnproto.org/go/capnp/v3 v3.1.0-alpha.2 emperror.dev/errors v0.8.1 github.com/Masterminds/semver/v3 v3.4.0 - github.com/ai-dynamo/grove/operator/api v0.1.0-alpha.15 + github.com/ai-dynamo/grove/operator/api v0.2.0-beta1 github.com/ai-dynamo/modelexpress/modelexpress_client/go v0.0.0-20261002171923-ac6d03a87a55 github.com/ai-dynamo/snapshot/api v0.2.0 github.com/bsm/gomega v1.27.10 diff --git a/deploy/operator/go.sum b/deploy/operator/go.sum index 630464257781..12a498fddfff 100644 --- a/deploy/operator/go.sum +++ b/deploy/operator/go.sum @@ -18,8 +18,8 @@ github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= -github.com/ai-dynamo/grove/operator/api v0.1.0-alpha.15 h1:7OD4K8Ihjdehd1fHzAxV6e2bMWzsFaox2UuAZ32qnDU= -github.com/ai-dynamo/grove/operator/api v0.1.0-alpha.15/go.mod h1:pTafNPyD9ic9WPyfZ6PeoClWbIkzWmMUTORG1tdhNFk= +github.com/ai-dynamo/grove/operator/api v0.2.0-beta1 h1:i+/GdpillY8cbv5j86CNiPiZZti/9OdrLdDiQnvWMyg= +github.com/ai-dynamo/grove/operator/api v0.2.0-beta1/go.mod h1:JZgOne08H1JomHIQGudSrSQMXByAAm0mcpfuVP1YV48= github.com/ai-dynamo/modelexpress/modelexpress_client/go v0.0.0-20261002171923-ac6d03a87a55 h1:Kk9yUPloFZuSSry7P79fXrI7rAvfW/ywjaeOGeE2LEo= github.com/ai-dynamo/modelexpress/modelexpress_client/go v0.0.0-20261002171923-ac6d03a87a55/go.mod h1:gmv3oYWWMcnFRaNH3Ij7oaXZPo5Xkrd0rdN0ry6TMkQ= github.com/ai-dynamo/snapshot/api v0.2.0 h1:+Nt4lhF0iiyjbNmOAd0F2lBZVKQb41wfcg1ZOP/iM1E= diff --git a/deploy/operator/internal/dynamo/grove_schema_test.go b/deploy/operator/internal/dynamo/grove_schema_test.go index 0bed1402faff..29b2f03af0db 100644 --- a/deploy/operator/internal/dynamo/grove_schema_test.go +++ b/deploy/operator/internal/dynamo/grove_schema_test.go @@ -11,8 +11,8 @@ import ( configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" controller_common "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + grovecrds "github.com/ai-dynamo/grove/operator/api/core/crds" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - grovecrds "github.com/ai-dynamo/grove/operator/api/core/v1alpha1/crds" "github.com/stretchr/testify/require" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" From 12b4139128f62f3fd7e01836f59fc0601bff930b Mon Sep 17 00:00:00 2001 From: Julien Mancuso <161955438+julienmancuso@users.noreply.github.com> Date: Mon, 2 Nov 2026 10:46:22 -0700 Subject: [PATCH 2/4] fix(ci): support Grove beta webhooks in deployment tests Signed-off-by: Julien Mancuso --- .../actions/setup-dynamo-operator/action.yml | 6 +- deploy/helm/charts/platform/README.md | 5 +- deploy/helm/charts/platform/README.md.gotmpl | 5 +- .../platform/tests/test_grove_version.py | 51 ++++++++++ deploy/operator/Makefile | 7 +- .../hack/prepare_grove_test_crds/main.go | 66 +++++++++++++ .../hack/prepare_grove_test_crds/main_test.go | 92 +++++++++++++++++++ .../internal/testing/clusterenv/DESIGN.md | 7 ++ 8 files changed, 232 insertions(+), 7 deletions(-) create mode 100644 deploy/operator/hack/prepare_grove_test_crds/main.go create mode 100644 deploy/operator/hack/prepare_grove_test_crds/main_test.go diff --git a/.github/actions/setup-dynamo-operator/action.yml b/.github/actions/setup-dynamo-operator/action.yml index d6a30bc5c116..21c0379766f1 100644 --- a/.github/actions/setup-dynamo-operator/action.yml +++ b/.github/actions/setup-dynamo-operator/action.yml @@ -784,11 +784,15 @@ runs: -o json) if ! jq -e --arg webhook_name "${webhook_name}" ' any(.webhooks[]; + (.matchPolicy // "Equivalent") as $matchPolicy | .name == $webhook_name and .failurePolicy == "Fail" and any(.rules[]; (.apiGroups | index("grove.io")) != null - and (.apiVersions | index("v1alpha1")) != null + and ( + (.apiVersions | index("v1alpha1")) != null + or ($matchPolicy == "Equivalent" and (.apiVersions | index("v1beta1")) != null) + ) and (.resources | index("podcliquesets")) != null and (.operations | index("CREATE")) != null ) diff --git a/deploy/helm/charts/platform/README.md b/deploy/helm/charts/platform/README.md index 80d81989376a..a60cde7504db 100644 --- a/deploy/helm/charts/platform/README.md +++ b/deploy/helm/charts/platform/README.md @@ -189,7 +189,10 @@ release tags and requires the pinned version or later. bundled Grove, replacing `grove.crdInstaller.enabled`. The API server must be able to reach Grove's conversion webhook. Read the [Grove upgrade and downgrade guide](https://github.com/ai-dynamo/grove/blob/v0.2.0-beta1/docs/user-guide/upgrading-to-v1beta1-v0.2.0-beta1.md) -before upgrading; rollback requires migrating stored resources back to `v1alpha1`. +before upgrading. Run the downgrade migration back to `v1alpha1` while the +`v0.2.0-beta1` operator is still running, then roll back to Grove +`v0.1.0-alpha.16`, rather than the previously bundled `v0.1.0-alpha.15`. +Earlier CRDs can drop fields written by newer releases. **Existing deployments:** Grove converts existing alpha resources without requiring manifest changes. An unset clique or scaling-group `scaleConfig.minReplicas` now diff --git a/deploy/helm/charts/platform/README.md.gotmpl b/deploy/helm/charts/platform/README.md.gotmpl index 24ee251cccab..c92556c6a2a1 100644 --- a/deploy/helm/charts/platform/README.md.gotmpl +++ b/deploy/helm/charts/platform/README.md.gotmpl @@ -189,7 +189,10 @@ release tags and requires the pinned version or later. bundled Grove, replacing `grove.crdInstaller.enabled`. The API server must be able to reach Grove's conversion webhook. Read the [Grove upgrade and downgrade guide](https://github.com/ai-dynamo/grove/blob/v0.2.0-beta1/docs/user-guide/upgrading-to-v1beta1-v0.2.0-beta1.md) -before upgrading; rollback requires migrating stored resources back to `v1alpha1`. +before upgrading. Run the downgrade migration back to `v1alpha1` while the +`v0.2.0-beta1` operator is still running, then roll back to Grove +`v0.1.0-alpha.16`, rather than the previously bundled `v0.1.0-alpha.15`. +Earlier CRDs can drop fields written by newer releases. **Existing deployments:** Grove converts existing alpha resources without requiring manifest changes. An unset clique or scaling-group `scaleConfig.minReplicas` now diff --git a/deploy/helm/charts/platform/tests/test_grove_version.py b/deploy/helm/charts/platform/tests/test_grove_version.py index 4ba6872d3d74..95b3938d3830 100644 --- a/deploy/helm/charts/platform/tests/test_grove_version.py +++ b/deploy/helm/charts/platform/tests/test_grove_version.py @@ -1,6 +1,9 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import copy +import json +import re import shutil import subprocess import tempfile @@ -10,6 +13,54 @@ ROOT = Path(__file__).resolve().parents[1] +@unittest.skipUnless(shutil.which("jq"), "requires jq CLI") +class GroveWebhookConfigurationTest(unittest.TestCase): + def test_alpha_requests_are_covered_by_fail_closed_webhooks(self): + """Accept equivalent beta rules without accepting fail-open or exact beta rules.""" + action = ( + ROOT.parents[3] / ".github/actions/setup-dynamo-operator/action.yml" + ).read_text() + query = re.search( + r"(?s)if ! jq -e --arg webhook_name.*? '\n(.*?)\n ' <<<", + action, + ).group(1) + webhook = { + "name": "pcs.defaulting.webhooks.grove.io", + "failurePolicy": "Fail", + "rules": [ + { + "apiGroups": ["grove.io"], + "apiVersions": ["v1alpha1"], + "resources": ["podcliquesets"], + "operations": ["CREATE"], + } + ], + } + for version, policy, failure, accepted in ( + ("v1alpha1", "Exact", "Fail", True), + ("v1beta1", "Equivalent", "Fail", True), + ("v1beta1", None, "Fail", True), + ("v1beta1", "Exact", "Fail", False), + ("v1beta1", "Equivalent", "Ignore", False), + ("v1beta2", "Equivalent", "Fail", False), + ): + with self.subTest(version=version, policy=policy, failure=failure): + candidate = copy.deepcopy(webhook) + candidate["failurePolicy"] = failure + if policy is not None: + candidate["matchPolicy"] = policy + candidate["rules"][0]["apiVersions"] = [version] + result = subprocess.run( + ["jq", "-e", "--arg", "webhook_name", webhook["name"], query], + input=json.dumps({"webhooks": [candidate]}), + capture_output=True, + text=True, + timeout=5, + check=False, + ) + self.assertEqual(result.returncode, 0 if accepted else 1, result.stderr) + + @unittest.skipUnless(shutil.which("helm"), "requires Helm CLI") class GroveVersionTest(unittest.TestCase): def helm(self, *args): diff --git a/deploy/operator/Makefile b/deploy/operator/Makefile index ce76a0119aae..7af4283fdb08 100644 --- a/deploy/operator/Makefile +++ b/deploy/operator/Makefile @@ -24,8 +24,6 @@ CLUSTERTEST_OPERATOR_IMAGE ?= $(DYNAMO_CLUSTERTEST_OPERATOR_IMAGE) CLUSTERTEST_PLATFORM ?= linux/$(GOARCH) CLUSTERTEST_LWS_VERSION = $(shell go list -m -f '{{.Version}}' sigs.k8s.io/lws) CLUSTERTEST_LWS_CRD_DIR = $(shell go env GOMODCACHE)/sigs.k8s.io/lws@$(CLUSTERTEST_LWS_VERSION)/config/crd/bases -CLUSTERTEST_GROVE_VERSION = $(shell go list -m -f '{{.Version}}' github.com/ai-dynamo/grove/operator/api) -CLUSTERTEST_GROVE_CRD_DIR = $(shell go env GOMODCACHE)/github.com/ai-dynamo/grove/operator/api@$(CLUSTERTEST_GROVE_VERSION)/core/crds CLUSTERTEST_VOLCANO_VERSION = $(shell go list -m -f '{{.Version}}' volcano.sh/apis) CLUSTERTEST_VOLCANO_PODGROUP_CRD_URL = https://raw.githubusercontent.com/volcano-sh/volcano/$(CLUSTERTEST_VOLCANO_VERSION)/config/crd/volcano/bases/scheduling.volcano.sh_podgroups.yaml @@ -344,8 +342,9 @@ setup-clustertest-crds-in-cluster: kubectl ## Install shared CRDs into the expli $(KUBECTL) --kubeconfig '$(CLUSTERTEST_KUBECONFIG)' --context "$${DYNAMO_CLUSTERTEST_CONTEXT}" apply --server-side -f config/crd/bases go mod download sigs.k8s.io/lws@$(CLUSTERTEST_LWS_VERSION) $(KUBECTL) --kubeconfig '$(CLUSTERTEST_KUBECONFIG)' --context "$${DYNAMO_CLUSTERTEST_CONTEXT}" apply --server-side -f '$(CLUSTERTEST_LWS_CRD_DIR)' - go mod download github.com/ai-dynamo/grove/operator/api@$(CLUSTERTEST_GROVE_VERSION) - $(KUBECTL) --kubeconfig '$(CLUSTERTEST_KUBECONFIG)' --context "$${DYNAMO_CLUSTERTEST_CONTEXT}" apply --server-side -f '$(CLUSTERTEST_GROVE_CRD_DIR)' + # Dependency controllers do not run in clustertests; retain the pinned alpha schema without webhook conversion. + go run ./hack/prepare_grove_test_crds | \ + $(KUBECTL) --kubeconfig '$(CLUSTERTEST_KUBECONFIG)' --context "$${DYNAMO_CLUSTERTEST_CONTEXT}" apply --server-side -f - curl --fail --location --silent --show-error --max-time 60 --retry 3 '$(CLUSTERTEST_VOLCANO_PODGROUP_CRD_URL)' | \ $(KUBECTL) --kubeconfig '$(CLUSTERTEST_KUBECONFIG)' --context "$${DYNAMO_CLUSTERTEST_CONTEXT}" apply --server-side -f - $(KUBECTL) --kubeconfig '$(CLUSTERTEST_KUBECONFIG)' --context "$${DYNAMO_CLUSTERTEST_CONTEXT}" wait --for=condition=Established crd --all --timeout=2m diff --git a/deploy/operator/hack/prepare_grove_test_crds/main.go b/deploy/operator/hack/prepare_grove_test_crds/main.go new file mode 100644 index 000000000000..522928d78ff5 --- /dev/null +++ b/deploy/operator/hack/prepare_grove_test_crds/main.go @@ -0,0 +1,66 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Emit pinned Grove schemas for tests that do not run the Grove controller. +package main + +import ( + "fmt" + "os" + + grovecrds "github.com/ai-dynamo/grove/operator/api/core/crds" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "sigs.k8s.io/yaml" +) + +func main() { + // Fail before emitting anything if the pinned API no longer serves Dynamo's version. + crds, err := testCRDs() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + // Stream manifests directly into the explicitly selected cluster's installer. + for _, crd := range crds { + data, err := yaml.Marshal(crd) + if err == nil { + _, err = fmt.Fprintf(os.Stdout, "---\n%s", data) + } + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + } +} + +func testCRDs() ([]*apiextensionsv1.CustomResourceDefinition, error) { + // Preserve each pinned alpha schema while removing the dependency on Grove's converter. + result := make([]*apiextensionsv1.CustomResourceDefinition, 0, 5) + for _, manifest := range []string{ + grovecrds.PodCliqueSetCRD(), + grovecrds.PodCliqueCRD(), + grovecrds.PodCliqueScalingGroupCRD(), + grovecrds.ClusterTopologyCRD(), + grovecrds.PodGangMapCRD(), + } { + crd := &apiextensionsv1.CustomResourceDefinition{} + if err := yaml.Unmarshal([]byte(manifest), crd); err != nil { + return nil, err + } + versions := make([]apiextensionsv1.CustomResourceDefinitionVersion, 0, 1) + for _, version := range crd.Spec.Versions { + if version.Name == "v1alpha1" && version.Served { + version.Storage = true + versions = append(versions, version) + } + } + if len(versions) != 1 { + return nil, fmt.Errorf("%s: expected one served v1alpha1 schema", crd.Name) + } + crd.Spec.Versions = versions + crd.Spec.Conversion = &apiextensionsv1.CustomResourceConversion{Strategy: apiextensionsv1.NoneConverter} + result = append(result, crd) + } + return result, nil +} diff --git a/deploy/operator/hack/prepare_grove_test_crds/main_test.go b/deploy/operator/hack/prepare_grove_test_crds/main_test.go new file mode 100644 index 000000000000..bb4653e689cb --- /dev/null +++ b/deploy/operator/hack/prepare_grove_test_crds/main_test.go @@ -0,0 +1,92 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "testing" + + grovecrds "github.com/ai-dynamo/grove/operator/api/core/crds" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/yaml" +) + +func TestCRDsPreservePinnedAlphaSchema(t *testing.T) { + t.Log("Generate all five Grove test CRDs from the pinned module") + crds, err := testCRDs() + require.NoError(t, err) + require.Len(t, crds, 5) + for _, crd := range crds { + require.Len(t, crd.Spec.Versions, 1, crd.Name) + require.Equal(t, "v1alpha1", crd.Spec.Versions[0].Name, crd.Name) + require.True(t, crd.Spec.Versions[0].Served, crd.Name) + require.True(t, crd.Spec.Versions[0].Storage, crd.Name) + require.Equal(t, apiextensionsv1.NoneConverter, crd.Spec.Conversion.Strategy, crd.Name) + require.Nil(t, crd.Spec.Conversion.Webhook, crd.Name) + } + + t.Log("Check that the PCS schema, defaults, CEL rules, and subresources are unchanged") + var original apiextensionsv1.CustomResourceDefinition + require.NoError(t, yaml.Unmarshal([]byte(grovecrds.PodCliqueSetCRD()), &original)) + for _, version := range original.Spec.Versions { + if version.Name == "v1alpha1" { + version.Storage = true + require.Equal(t, version, crds[0].Spec.Versions[0]) + } + } +} + +func TestPodCliqueSetRequestsWithoutGroveConversionService(t *testing.T) { + if os.Getenv("KUBEBUILDER_ASSETS") == "" { + t.Skip("requires local envtest assets") + } + + t.Log("Install the prepared Grove CRDs into an API server with no Grove operator") + crds, err := testCRDs() + require.NoError(t, err) + env := &envtest.Environment{CRDs: crds} + config, err := env.Start() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, env.Stop()) }) + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + require.NoError(t, grovev1alpha1.AddToScheme(scheme)) + c, err := client.New(config, client.Options{Scheme: scheme}) + require.NoError(t, err) + require.NoError(t, c.Create(t.Context(), &corev1.Namespace{Name: "grove-test"})) + + t.Log("Create the alpha PodCliqueSet used by the cluster-test workload provider") + pcs := &grovev1alpha1.PodCliqueSet{ + Name: "workers", Namespace: "grove-test", + Spec: grovev1alpha1.PodCliqueSetSpec{ + Replicas: 1, + Template: grovev1alpha1.PodCliqueSetTemplateSpec{ + Cliques: []*grovev1alpha1.PodCliqueTemplateSpec{{ + Name: "worker", + Spec: grovev1alpha1.PodCliqueSpec{ + RoleName: "worker", Replicas: 1, MinAvailable: new(int32(1)), + PodSpec: corev1.PodSpec{Containers: []corev1.Container{{Name: "worker", Image: "registry.k8s.io/pause:3.10"}}}, + }, + }}, + }, + }, + } + require.NoError(t, c.Create(t.Context(), pcs)) + + t.Log("Read, list, update, and delete without a conversion webhook service") + observed := &grovev1alpha1.PodCliqueSet{} + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(pcs), observed)) + var list grovev1alpha1.PodCliqueSetList + require.NoError(t, c.List(t.Context(), &list, client.InNamespace(pcs.Namespace))) + require.Len(t, list.Items, 1) + observed.Spec.Replicas = 2 + require.NoError(t, c.Update(t.Context(), observed)) + require.NoError(t, c.Delete(t.Context(), observed)) +} diff --git a/deploy/operator/internal/testing/clusterenv/DESIGN.md b/deploy/operator/internal/testing/clusterenv/DESIGN.md index b0a3f9614573..e3fdb6306418 100644 --- a/deploy/operator/internal/testing/clusterenv/DESIGN.md +++ b/deploy/operator/internal/testing/clusterenv/DESIGN.md @@ -75,6 +75,13 @@ PodCliqueSet handlers at Grove's production paths. Grove CRD schema validation and OpenAPI defaulting still run in the API server; the transparent handlers are replaced by Grove's production `Setup` API once it is released. +The shared setup derives Grove test CRDs from the pinned Go API bundle through +`hack/prepare_grove_test_crds`. It retains only `v1alpha1` as the served and storage +version, preserving that version's schema, CEL validation, and OpenAPI defaults. +Conversion is disabled because this harness does not run the Grove operator. +The deployment smoke tests install the published Grove chart and exercise its +real beta conversion and admission webhooks. + The webhook manager is independent of all per-test controller managers. No operator controller runs merely because a test uses admission or conversion. The proxy image is configurable with `WebhookProxyImage`; the controller suite From 904720575b08fa46c62eacefc7a1e112e385f827 Mon Sep 17 00:00:00 2001 From: Julien Mancuso <161955438+julienmancuso@users.noreply.github.com> Date: Mon, 2 Nov 2026 12:36:10 -0700 Subject: [PATCH 3/4] fix(operator): accept Grove beta controller owner references Grove v0.2.0-beta1 creates children with grove.io/v1beta1 controller references. Conversion to Dynamo's alpha client types preserves those references, so alpha-only owner filters drop child readiness events. Planner startup can then wait indefinitely on stale DGD worker readiness. Accept both served Grove versions in DGD event routing, LPX request indexing, and released Agent Pod ownership verification. Preserve existing kind, controller, namespace, and UID checks. Cover beta and mixed-version ownership in the existing regression tables. Validation: regression tests fail before the fix and pass afterward; full operator Go suite including envtest passed; CI-pinned golangci-lint reports 0 issues; pre-commit passed. Signed-off-by: Julien Mancuso --- .../controller/dgd_grove_watch_setup.go | 10 +++- .../controller/dgd_grove_watch_setup_test.go | 51 +++++++++++++++---- .../dynamographdeployment_controller_test.go | 26 +++++++--- .../controller/lpx/pipeline_request.go | 10 +++- .../controller/lpx/pipeline_request_test.go | 21 +++++--- .../operator/internal/controller/lpx/watch.go | 6 ++- 6 files changed, 96 insertions(+), 28 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_grove_watch_setup.go b/deploy/operator/internal/controller/dgd_grove_watch_setup.go index 0762d21b5ff2..8197e49dc50e 100644 --- a/deploy/operator/internal/controller/dgd_grove_watch_setup.go +++ b/deploy/operator/internal/controller/dgd_grove_watch_setup.go @@ -23,6 +23,7 @@ import ( nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + grovev1beta1 "github.com/ai-dynamo/grove/operator/api/core/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" @@ -145,7 +146,10 @@ func (s *groveWatchSetup) resolveOwningPodCliqueSet( podClique *grovev1alpha1.PodClique, ) (*grovev1alpha1.PodCliqueSet, bool) { controllerRef := metav1.GetControllerOf(podClique) - if controllerRef == nil || controllerRef.APIVersion != grovev1alpha1.SchemeGroupVersion.String() { + // Conversion to our alpha client types preserves beta controller owner references. + if controllerRef == nil || + (controllerRef.APIVersion != grovev1alpha1.SchemeGroupVersion.String() && + controllerRef.APIVersion != grovev1beta1.SchemeGroupVersion.String()) { return nil, false } @@ -171,8 +175,10 @@ func (s *groveWatchSetup) resolvePodCliqueSetOwner( pcsg *grovev1alpha1.PodCliqueScalingGroup, ) (*grovev1alpha1.PodCliqueSet, bool) { controllerRef := metav1.GetControllerOf(pcsg) + // Both served Grove versions identify the same owner hierarchy. if controllerRef == nil || - controllerRef.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || + (controllerRef.APIVersion != grovev1alpha1.SchemeGroupVersion.String() && + controllerRef.APIVersion != grovev1beta1.SchemeGroupVersion.String()) || controllerRef.Kind != "PodCliqueSet" { return nil, false } diff --git a/deploy/operator/internal/controller/dgd_grove_watch_setup_test.go b/deploy/operator/internal/controller/dgd_grove_watch_setup_test.go index 156c4bb1a816..77c097f0265a 100644 --- a/deploy/operator/internal/controller/dgd_grove_watch_setup_test.go +++ b/deploy/operator/internal/controller/dgd_grove_watch_setup_test.go @@ -9,6 +9,7 @@ import ( nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + grovev1beta1 "github.com/ai-dynamo/grove/operator/api/core/v1beta1" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -32,7 +33,7 @@ func TestGroveWatchMapsOnlyDGDControllerOwnedPodCliques(t *testing.T) { } } - newObjects := func() (*grovev1alpha1.PodClique, *grovev1alpha1.PodCliqueScalingGroup, *grovev1alpha1.PodCliqueSet) { + newObjects := func(apiVersion string) (*grovev1alpha1.PodClique, *grovev1alpha1.PodCliqueScalingGroup, *grovev1alpha1.PodCliqueSet) { pcs := &grovev1alpha1.PodCliqueSet{ Name: "engine-pcs", Namespace: "workloads", @@ -45,13 +46,13 @@ func TestGroveWatchMapsOnlyDGDControllerOwnedPodCliques(t *testing.T) { Namespace: "workloads", UID: "group-uid", OwnerReferences: []metav1.OwnerReference{controllerRef( - grovev1alpha1.SchemeGroupVersion.String(), "PodCliqueSet", pcs.Name, pcs.UID, + apiVersion, "PodCliqueSet", pcs.Name, pcs.UID, )}} clique := &grovev1alpha1.PodClique{ Name: "role", Namespace: "workloads", OwnerReferences: []metav1.OwnerReference{controllerRef( - grovev1alpha1.SchemeGroupVersion.String(), "PodCliqueScalingGroup", group.Name, group.UID, + apiVersion, "PodCliqueScalingGroup", group.Name, group.UID, )}} return clique, group, pcs } @@ -66,7 +67,7 @@ func TestGroveWatchMapsOnlyDGDControllerOwnedPodCliques(t *testing.T) { name: "direct PCS ownership", mutate: func(clique *grovev1alpha1.PodClique, _ *grovev1alpha1.PodCliqueScalingGroup, pcs *grovev1alpha1.PodCliqueSet) []client.Object { clique.OwnerReferences = []metav1.OwnerReference{controllerRef( - grovev1alpha1.SchemeGroupVersion.String(), "PodCliqueSet", pcs.Name, pcs.UID, + clique.OwnerReferences[0].APIVersion, "PodCliqueSet", pcs.Name, pcs.UID, )} return []client.Object{pcs} }, @@ -79,6 +80,32 @@ func TestGroveWatchMapsOnlyDGDControllerOwnedPodCliques(t *testing.T) { }, want: want, }, + { + name: "mixed-version PCSG-mediated ownership", + mutate: func(clique *grovev1alpha1.PodClique, group *grovev1alpha1.PodCliqueScalingGroup, pcs *grovev1alpha1.PodCliqueSet) []client.Object { + if clique.OwnerReferences[0].APIVersion == grovev1alpha1.SchemeGroupVersion.String() { + group.OwnerReferences[0].APIVersion = grovev1beta1.SchemeGroupVersion.String() + } else { + group.OwnerReferences[0].APIVersion = grovev1alpha1.SchemeGroupVersion.String() + } + return []client.Object{group, pcs} + }, + want: want, + }, + { + name: "foreign PodClique owner API group", + mutate: func(clique *grovev1alpha1.PodClique, group *grovev1alpha1.PodCliqueScalingGroup, pcs *grovev1alpha1.PodCliqueSet) []client.Object { + clique.OwnerReferences[0].APIVersion = "foreign.example/v1beta1" + return []client.Object{group, pcs} + }, + }, + { + name: "foreign scaling group owner API group", + mutate: func(_ *grovev1alpha1.PodClique, group *grovev1alpha1.PodCliqueScalingGroup, pcs *grovev1alpha1.PodCliqueSet) []client.Object { + group.OwnerReferences[0].APIVersion = "foreign.example/v1beta1" + return []client.Object{group, pcs} + }, + }, { name: "LGD-owned PCS", mutate: func(_ *grovev1alpha1.PodClique, group *grovev1alpha1.PodCliqueScalingGroup, pcs *grovev1alpha1.PodCliqueSet) []client.Object { @@ -133,11 +160,15 @@ func TestGroveWatchMapsOnlyDGDControllerOwnedPodCliques(t *testing.T) { } for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - clique, group, pcs := newObjects() - objects := test.mutate(clique, group, pcs) - reader := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() - require.Equal(t, test.want, newGroveWatchSetup(reader).mapPodCliqueToRequests(t.Context(), clique)) - }) + for _, apiVersion := range []string{grovev1alpha1.SchemeGroupVersion.String(), grovev1beta1.SchemeGroupVersion.String()} { + t.Run(test.name+"/"+apiVersion, func(t *testing.T) { + t.Log("Observe converted Grove children with their original owner-reference versions") + clique, group, pcs := newObjects(apiVersion) + objects := test.mutate(clique, group, pcs) + reader := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + t.Log("Map readiness updates only through the exact DGD-owned resource hierarchy") + require.Equal(t, test.want, newGroveWatchSetup(reader).mapPodCliqueToRequests(t.Context(), clique)) + }) + } } } diff --git a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go index ecffe4649f37..d0a7a8e7ba91 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go @@ -34,6 +34,7 @@ import ( grovecommon "github.com/ai-dynamo/grove/operator/api/common" groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + grovev1beta1 "github.com/ai-dynamo/grove/operator/api/core/v1beta1" snapshotv1alpha1 "github.com/ai-dynamo/snapshot/api/v1alpha1" "github.com/onsi/gomega" "github.com/stretchr/testify/assert" @@ -3667,13 +3668,24 @@ func TestGroveWatchSetup_MapPodCliqueScalingGroupToRequests(t *testing.T) { Config: &configv1alpha1.OperatorConfiguration{}, RuntimeConfig: &controller_common.RuntimeConfig{}, } - reqs := newGroveWatchSetup(r.Client). - mapPodCliqueScalingGroupToRequests(context.Background(), tt.obj) - - g.Expect(reqs).To(gomega.HaveLen(tt.wantRequests)) - if tt.wantRequests == 1 { - g.Expect(reqs[0].Name).To(gomega.Equal(tt.wantName)) - g.Expect(reqs[0].Namespace).To(gomega.Equal(tt.wantNs)) + for _, apiVersion := range []string{grovev1alpha1.SchemeGroupVersion.String(), grovev1beta1.SchemeGroupVersion.String()} { + t.Logf("Map scaling-group readiness with %s owner references", apiVersion) + obj := tt.obj.DeepCopyObject().(client.Object) + owners := obj.GetOwnerReferences() + for i := range owners { + if owners[i].APIVersion == grovev1alpha1.SchemeGroupVersion.String() { + owners[i].APIVersion = apiVersion + } + } + obj.SetOwnerReferences(owners) + reqs := newGroveWatchSetup(r.Client). + mapPodCliqueScalingGroupToRequests(context.Background(), obj) + + g.Expect(reqs).To(gomega.HaveLen(tt.wantRequests)) + if tt.wantRequests == 1 { + g.Expect(reqs[0].Name).To(gomega.Equal(tt.wantName)) + g.Expect(reqs[0].Namespace).To(gomega.Equal(tt.wantNs)) + } } }) } diff --git a/deploy/operator/internal/controller/lpx/pipeline_request.go b/deploy/operator/internal/controller/lpx/pipeline_request.go index 2fa237f46d29..4dc1de28a9d1 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request.go @@ -21,6 +21,7 @@ import ( lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovecommon "github.com/ai-dynamo/grove/operator/api/common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + grovev1beta1 "github.com/ai-dynamo/grove/operator/api/core/v1beta1" corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -308,7 +309,10 @@ func (r *graphReconciler) removeReleasedAgentPod(ctx context.Context, pcsgs map[ } cliqueOwner := metav1.GetControllerOf(pod) - if cliqueOwner == nil || cliqueOwner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || cliqueOwner.Kind != "PodClique" { + // Converted children can carry either served Grove version in owner references. + if cliqueOwner == nil || + (cliqueOwner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() && cliqueOwner.APIVersion != grovev1beta1.SchemeGroupVersion.String()) || + cliqueOwner.Kind != "PodClique" { return fmt.Errorf("released Agent Pod %q has no PodClique controller", pod.Name) } clique := &grovev1alpha1.PodClique{} @@ -320,7 +324,9 @@ func (r *graphReconciler) removeReleasedAgentPod(ctx context.Context, pcsgs map[ } groupOwner := metav1.GetControllerOf(clique) - if groupOwner == nil || groupOwner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || groupOwner.Kind != "PodCliqueScalingGroup" { + if groupOwner == nil || + (groupOwner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() && groupOwner.APIVersion != grovev1beta1.SchemeGroupVersion.String()) || + groupOwner.Kind != "PodCliqueScalingGroup" { return fmt.Errorf("released Agent Pod %q's PodClique has no PodCliqueScalingGroup controller", pod.Name) } if group := pcsgs[groupOwner.Name]; group == nil || group.UID != groupOwner.UID { diff --git a/deploy/operator/internal/controller/lpx/pipeline_request_test.go b/deploy/operator/internal/controller/lpx/pipeline_request_test.go index 3ca0321c611d..6d040397974b 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request_test.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request_test.go @@ -6,6 +6,7 @@ package lpx import ( + "cmp" "context" "errors" "slices" @@ -17,6 +18,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + grovev1beta1 "github.com/ai-dynamo/grove/operator/api/core/v1beta1" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -30,10 +32,13 @@ import ( func TestRemoveReleasedAgentPods(t *testing.T) { for _, tc := range []struct { name, change string + podOwnerVersion, cliqueOwnerVersion string reason lpxv1alpha1.ReleaseReason remove, wantError, deleteError, missingClique bool }{ {name: "dependency changed", remove: true}, + {name: "beta Pod owner", podOwnerVersion: grovev1beta1.SchemeGroupVersion.String(), remove: true}, + {name: "beta PodClique owner", cliqueOwnerVersion: grovev1beta1.SchemeGroupVersion.String(), remove: true}, {name: "request changed", reason: lpxv1alpha1.ReleaseReasonRequestChanged, remove: true}, {name: "binding failed", reason: lpxv1alpha1.ReleaseReasonBindingFailed, remove: true}, {name: "unbound target", change: "target", remove: true}, @@ -64,6 +69,8 @@ func TestRemoveReleasedAgentPods(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Log("Observe one released Agent Pod and its ownership beneath a trusted scaling group") pcsgs, pclq, pod := newLPXTestAgentHierarchy() + pod.OwnerReferences[0].APIVersion = cmp.Or(tc.podOwnerVersion, grovev1alpha1.SchemeGroupVersion.String()) + pclq.OwnerReferences[0].APIVersion = cmp.Or(tc.cliqueOwnerVersion, grovev1alpha1.SchemeGroupVersion.String()) request := newLPXTestReleaseRequest(pod) execution := &request.Status.Committed.Execution if tc.reason != "" { @@ -221,7 +228,7 @@ func TestGetPipelineRequests(t *testing.T) { uid types.UID want []string }{ - {name: "owned requests", namespace: "default", uid: "pcs-uid", want: []string{"first", "last"}}, + {name: "owned requests", namespace: "default", uid: "pcs-uid", want: []string{"first", "last", "beta"}}, {name: "different PCS with the same name", namespace: "default", uid: "other-uid", want: []string{"other-owner"}}, {name: "different namespace", namespace: "other", uid: "pcs-uid", want: []string{"other-namespace"}}, {name: "no owned requests", namespace: "default", uid: "missing-uid"}, @@ -232,16 +239,18 @@ func TestGetPipelineRequests(t *testing.T) { for _, request := range []struct { name, namespace string uid types.UID + apiVersion string }{ - {"last", "default", "pcs-uid"}, - {"first", "default", "pcs-uid"}, - {"other-owner", "default", "other-uid"}, - {"other-namespace", "other", "pcs-uid"}, + {"last", "default", "pcs-uid", grovev1alpha1.SchemeGroupVersion.String()}, + {"first", "default", "pcs-uid", grovev1alpha1.SchemeGroupVersion.String()}, + {"beta", "default", "pcs-uid", grovev1beta1.SchemeGroupVersion.String()}, + {"other-owner", "default", "other-uid", grovev1beta1.SchemeGroupVersion.String()}, + {"other-namespace", "other", "pcs-uid", grovev1beta1.SchemeGroupVersion.String()}, } { objects = append(objects, &lpxv1alpha1.LPUPipelineRequest{ Name: request.name, Namespace: request.namespace, OwnerReferences: []metav1.OwnerReference{{ - APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Kind: "PodCliqueSet", + APIVersion: request.apiVersion, Kind: "PodCliqueSet", Name: "pcs", UID: request.uid, Controller: new(true), }}}) } diff --git a/deploy/operator/internal/controller/lpx/watch.go b/deploy/operator/internal/controller/lpx/watch.go index 6a55e1cf54de..26c30cd3acf2 100644 --- a/deploy/operator/internal/controller/lpx/watch.go +++ b/deploy/operator/internal/controller/lpx/watch.go @@ -15,6 +15,7 @@ import ( lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + grovev1beta1 "github.com/ai-dynamo/grove/operator/api/core/v1beta1" corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -74,7 +75,10 @@ func (r *graphReconciler) setupWithManager(mgr ctrl.Manager) error { func pipelineRequestOwnerUID(obj client.Object) []string { owner := metav1.GetControllerOf(obj) - if owner == nil || owner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || owner.Kind != podCliqueSetKind { + // PCS owner references may use either served Grove version. + if owner == nil || + (owner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() && owner.APIVersion != grovev1beta1.SchemeGroupVersion.String()) || + owner.Kind != podCliqueSetKind { return nil } return []string{string(owner.UID)} From 27ad2cb36894638467713743c39bcc0472c9d879 Mon Sep 17 00:00:00 2001 From: Julien Mancuso Date: Tue, 3 Nov 2026 08:35:38 -0700 Subject: [PATCH 4/4] fix(operator): address Grove beta review feedback Signed-off-by: Julien Mancuso --- deploy/helm/charts/platform/README.md | 20 +++++++++++++++-- deploy/helm/charts/platform/README.md.gotmpl | 20 +++++++++++++++-- .../controller/dgd_grove_watch_setup_test.go | 22 +++++++++---------- .../controller/lpx/pipeline_request_test.go | 22 ++++++++++++------- .../operator/internal/controller/lpx/watch.go | 7 ++---- 5 files changed, 63 insertions(+), 28 deletions(-) diff --git a/deploy/helm/charts/platform/README.md b/deploy/helm/charts/platform/README.md index a60cde7504db..a56a7b73d6cd 100644 --- a/deploy/helm/charts/platform/README.md +++ b/deploy/helm/charts/platform/README.md @@ -187,13 +187,29 @@ release tags and requires the pinned version or later. **Action:** Upgrade externally managed Grove and matching CRDs first, with `crdInstaller.upgradeCRDs=true`. The platform chart enables this setting for bundled Grove, replacing `grove.crdInstaller.enabled`. The API server must be able -to reach Grove's conversion webhook. Read the +to reach Grove's conversion webhook. With Grove integration enabled, Dynamo's +alpha informers depend on that webhook to read beta-stored resources. If it is +unavailable during initial cache synchronization, the shared Dynamo manager can +fail to start, affecting non-Grove workloads handled by that manager too. After +startup, failed LIST/WATCH reconnects can leave Grove observations stale while +cached reads continue to work. This Grove informer dependency does not apply +when Grove integration is disabled. Read the [Grove upgrade and downgrade guide](https://github.com/ai-dynamo/grove/blob/v0.2.0-beta1/docs/user-guide/upgrading-to-v1beta1-v0.2.0-beta1.md) before upgrading. Run the downgrade migration back to `v1alpha1` while the `v0.2.0-beta1` operator is still running, then roll back to Grove -`v0.1.0-alpha.16`, rather than the previously bundled `v0.1.0-alpha.15`. +`v0.1.0-alpha.16`. Earlier CRDs can drop fields written by newer releases. +For bundled Grove, prepare the rollback platform chart with its +`dynamo.nvidia.com/grove-version` annotation and `grove` dependency pinned to +`v0.1.0-alpha.16`, then rebuild its dependencies with `helm dependency update`. +After the migration, deploy that chart with `helm upgrade` to the same release +and namespace, using saved pre-upgrade values, `global.grove.install=true`, and +`grove.crdInstaller.enabled=true`. Use client-side apply for this rollback: +Helm 3 does so by default; with Helm 4, pass `--server-side=false`. Do not roll +back to an unmodified older platform chart: Dynamo `v1.6.0` bundles `alpha.14`, +and pre-upgrade `main` bundles `alpha.15`. + **Existing deployments:** Grove converts existing alpha resources without requiring manifest changes. An unset clique or scaling-group `scaleConfig.minReplicas` now defaults to `minAvailable` rather than `replicas`; existing explicit minima are retained. diff --git a/deploy/helm/charts/platform/README.md.gotmpl b/deploy/helm/charts/platform/README.md.gotmpl index c92556c6a2a1..93b604881880 100644 --- a/deploy/helm/charts/platform/README.md.gotmpl +++ b/deploy/helm/charts/platform/README.md.gotmpl @@ -187,13 +187,29 @@ release tags and requires the pinned version or later. **Action:** Upgrade externally managed Grove and matching CRDs first, with `crdInstaller.upgradeCRDs=true`. The platform chart enables this setting for bundled Grove, replacing `grove.crdInstaller.enabled`. The API server must be able -to reach Grove's conversion webhook. Read the +to reach Grove's conversion webhook. With Grove integration enabled, Dynamo's +alpha informers depend on that webhook to read beta-stored resources. If it is +unavailable during initial cache synchronization, the shared Dynamo manager can +fail to start, affecting non-Grove workloads handled by that manager too. After +startup, failed LIST/WATCH reconnects can leave Grove observations stale while +cached reads continue to work. This Grove informer dependency does not apply +when Grove integration is disabled. Read the [Grove upgrade and downgrade guide](https://github.com/ai-dynamo/grove/blob/v0.2.0-beta1/docs/user-guide/upgrading-to-v1beta1-v0.2.0-beta1.md) before upgrading. Run the downgrade migration back to `v1alpha1` while the `v0.2.0-beta1` operator is still running, then roll back to Grove -`v0.1.0-alpha.16`, rather than the previously bundled `v0.1.0-alpha.15`. +`v0.1.0-alpha.16`. Earlier CRDs can drop fields written by newer releases. +For bundled Grove, prepare the rollback platform chart with its +`dynamo.nvidia.com/grove-version` annotation and `grove` dependency pinned to +`v0.1.0-alpha.16`, then rebuild its dependencies with `helm dependency update`. +After the migration, deploy that chart with `helm upgrade` to the same release +and namespace, using saved pre-upgrade values, `global.grove.install=true`, and +`grove.crdInstaller.enabled=true`. Use client-side apply for this rollback: +Helm 3 does so by default; with Helm 4, pass `--server-side=false`. Do not roll +back to an unmodified older platform chart: Dynamo `v1.6.0` bundles `alpha.14`, +and pre-upgrade `main` bundles `alpha.15`. + **Existing deployments:** Grove converts existing alpha resources without requiring manifest changes. An unset clique or scaling-group `scaleConfig.minReplicas` now defaults to `minAvailable` rather than `replicas`; existing explicit minima are retained. diff --git a/deploy/operator/internal/controller/dgd_grove_watch_setup_test.go b/deploy/operator/internal/controller/dgd_grove_watch_setup_test.go index 77c097f0265a..1b373e8e9bfd 100644 --- a/deploy/operator/internal/controller/dgd_grove_watch_setup_test.go +++ b/deploy/operator/internal/controller/dgd_grove_watch_setup_test.go @@ -4,6 +4,7 @@ package controller import ( + "cmp" "testing" nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" @@ -59,9 +60,10 @@ func TestGroveWatchMapsOnlyDGDControllerOwnedPodCliques(t *testing.T) { want := []ctrl.Request{{Namespace: "workloads", Name: "graph"}} tests := []struct { - name string - mutate func(*grovev1alpha1.PodClique, *grovev1alpha1.PodCliqueScalingGroup, *grovev1alpha1.PodCliqueSet) []client.Object - want []ctrl.Request + name string + groupOwnerVersion string + mutate func(*grovev1alpha1.PodClique, *grovev1alpha1.PodCliqueScalingGroup, *grovev1alpha1.PodCliqueSet) []client.Object + want []ctrl.Request }{ { name: "direct PCS ownership", @@ -74,20 +76,17 @@ func TestGroveWatchMapsOnlyDGDControllerOwnedPodCliques(t *testing.T) { want: want, }, { - name: "PCSG-mediated ownership", + name: "PCSG-mediated ownership with alpha PCS owner", + groupOwnerVersion: grovev1alpha1.SchemeGroupVersion.String(), mutate: func(_ *grovev1alpha1.PodClique, group *grovev1alpha1.PodCliqueScalingGroup, pcs *grovev1alpha1.PodCliqueSet) []client.Object { return []client.Object{group, pcs} }, want: want, }, { - name: "mixed-version PCSG-mediated ownership", - mutate: func(clique *grovev1alpha1.PodClique, group *grovev1alpha1.PodCliqueScalingGroup, pcs *grovev1alpha1.PodCliqueSet) []client.Object { - if clique.OwnerReferences[0].APIVersion == grovev1alpha1.SchemeGroupVersion.String() { - group.OwnerReferences[0].APIVersion = grovev1beta1.SchemeGroupVersion.String() - } else { - group.OwnerReferences[0].APIVersion = grovev1alpha1.SchemeGroupVersion.String() - } + name: "PCSG-mediated ownership with beta PCS owner", + groupOwnerVersion: grovev1beta1.SchemeGroupVersion.String(), + mutate: func(_ *grovev1alpha1.PodClique, group *grovev1alpha1.PodCliqueScalingGroup, pcs *grovev1alpha1.PodCliqueSet) []client.Object { return []client.Object{group, pcs} }, want: want, @@ -164,6 +163,7 @@ func TestGroveWatchMapsOnlyDGDControllerOwnedPodCliques(t *testing.T) { t.Run(test.name+"/"+apiVersion, func(t *testing.T) { t.Log("Observe converted Grove children with their original owner-reference versions") clique, group, pcs := newObjects(apiVersion) + group.OwnerReferences[0].APIVersion = cmp.Or(test.groupOwnerVersion, apiVersion) objects := test.mutate(clique, group, pcs) reader := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() t.Log("Map readiness updates only through the exact DGD-owned resource hierarchy") diff --git a/deploy/operator/internal/controller/lpx/pipeline_request_test.go b/deploy/operator/internal/controller/lpx/pipeline_request_test.go index 6d040397974b..94a1e5bef140 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request_test.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request_test.go @@ -228,7 +228,7 @@ func TestGetPipelineRequests(t *testing.T) { uid types.UID want []string }{ - {name: "owned requests", namespace: "default", uid: "pcs-uid", want: []string{"first", "last", "beta"}}, + {name: "owned requests", namespace: "default", uid: "pcs-uid", want: []string{"first", "last"}}, {name: "different PCS with the same name", namespace: "default", uid: "other-uid", want: []string{"other-owner"}}, {name: "different namespace", namespace: "other", uid: "pcs-uid", want: []string{"other-namespace"}}, {name: "no owned requests", namespace: "default", uid: "missing-uid"}, @@ -240,18 +240,24 @@ func TestGetPipelineRequests(t *testing.T) { name, namespace string uid types.UID apiVersion string + kind string + controller bool }{ - {"last", "default", "pcs-uid", grovev1alpha1.SchemeGroupVersion.String()}, - {"first", "default", "pcs-uid", grovev1alpha1.SchemeGroupVersion.String()}, - {"beta", "default", "pcs-uid", grovev1beta1.SchemeGroupVersion.String()}, - {"other-owner", "default", "other-uid", grovev1beta1.SchemeGroupVersion.String()}, - {"other-namespace", "other", "pcs-uid", grovev1beta1.SchemeGroupVersion.String()}, + {"last", "default", "pcs-uid", grovev1alpha1.SchemeGroupVersion.String(), "PodCliqueSet", true}, + {"first", "default", "pcs-uid", grovev1alpha1.SchemeGroupVersion.String(), "PodCliqueSet", true}, + {"other-owner", "default", "other-uid", grovev1alpha1.SchemeGroupVersion.String(), "PodCliqueSet", true}, + {"other-namespace", "other", "pcs-uid", grovev1alpha1.SchemeGroupVersion.String(), "PodCliqueSet", true}, + {"beta", "default", "pcs-uid", grovev1beta1.SchemeGroupVersion.String(), "PodCliqueSet", true}, + {"foreign-group", "default", "pcs-uid", "foreign.example/v1alpha1", "PodCliqueSet", true}, + {"unsupported-version", "default", "pcs-uid", "grove.io/v2", "PodCliqueSet", true}, + {"wrong-kind", "default", "pcs-uid", grovev1alpha1.SchemeGroupVersion.String(), "PodClique", true}, + {"non-controller", "default", "pcs-uid", grovev1alpha1.SchemeGroupVersion.String(), "PodCliqueSet", false}, } { objects = append(objects, &lpxv1alpha1.LPUPipelineRequest{ Name: request.name, Namespace: request.namespace, OwnerReferences: []metav1.OwnerReference{{ - APIVersion: request.apiVersion, Kind: "PodCliqueSet", - Name: "pcs", UID: request.uid, Controller: new(true), + APIVersion: request.apiVersion, Kind: request.kind, + Name: "pcs", UID: request.uid, Controller: new(request.controller), }}}) } r := &graphReconciler{Client: newLPXTestClient(t, objects...)} diff --git a/deploy/operator/internal/controller/lpx/watch.go b/deploy/operator/internal/controller/lpx/watch.go index 26c30cd3acf2..2d8705e1f4f1 100644 --- a/deploy/operator/internal/controller/lpx/watch.go +++ b/deploy/operator/internal/controller/lpx/watch.go @@ -15,7 +15,6 @@ import ( lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - grovev1beta1 "github.com/ai-dynamo/grove/operator/api/core/v1beta1" corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -75,10 +74,8 @@ func (r *graphReconciler) setupWithManager(mgr ctrl.Manager) error { func pipelineRequestOwnerUID(obj client.Object) []string { owner := metav1.GetControllerOf(obj) - // PCS owner references may use either served Grove version. - if owner == nil || - (owner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() && owner.APIVersion != grovev1beta1.SchemeGroupVersion.String()) || - owner.Kind != podCliqueSetKind { + // Dynamo authors these references with its alpha PCS client. + if owner == nil || owner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || owner.Kind != podCliqueSetKind { return nil } return []string{string(owner.UID)}