julienmancuso · Nov 3, 2026 · Nov 2, 2026 · Nov 2, 2026 · Nov 2, 2026 · Nov 3, 2026
@@ -784,11 +784,15 @@ runs:
             -o json)
           if ! jq -e --arg webhook_name "${webhook_name}" '
             any(.webhooks[];
+              (.matchPolicy // "Equivalent") as $matchPolicy |
               .name == $webhook_name
               and .failurePolicy == "Fail"
               and any(.rules[];
                 (.apiGroups | index("grove.io")) != null
-                and (.apiVersions | index("v1alpha1")) != null
+                and (
+                  (.apiVersions | index("v1alpha1")) != null
+                  or ($matchPolicy == "Equivalent" and (.apiVersions | index("v1beta1")) != null)
+                )
                 and (.resources | index("podcliquesets")) != null
                 and (.operations | index("CREATE")) != null
               )

@@ -25,7 +25,7 @@ kubeVersion: ">=1.30.0-0"
 home: https://nvidia.com
 annotations:
   # One scalar drives both the bundled dependency and the external-Grove preflight.
-  dynamo.nvidia.com/grove-version: &groveVersion v0.1.0-alpha.15
+  dynamo.nvidia.com/grove-version: &groveVersion v0.2.0-beta1
 dependencies:
   - name: dynamo-operator
     version: 1.7.0

@@ -173,6 +173,47 @@ sidecar guides.
 **Existing deployments:** No existing workload rolls. Components without a `runtime` init
 container keep running the engine and Dynamo together in `main`.
 
+#### Dependency compatibility
+
+##### Grove v0.2.0-beta1
+
+**Change:** The bundled Grove chart and Go API dependency are now `v0.2.0-beta1`.
+Grove stores resources as `grove.io/v1beta1` and continues serving the
+`grove.io/v1alpha1` API used by Dynamo. The external Grove preflight accepts beta
+release tags and requires the pinned version or later.
+
+**Affected:** All Grove-backed deployments upgrading to Dynamo 1.7.0.
+
+**Action:** Upgrade externally managed Grove and matching CRDs first, with
+`crdInstaller.upgradeCRDs=true`. The platform chart enables this setting for
+bundled Grove, replacing `grove.crdInstaller.enabled`. The API server must be able
+to reach Grove's conversion webhook. With Grove integration enabled, Dynamo's
+alpha informers depend on that webhook to read beta-stored resources. If it is
+unavailable during initial cache synchronization, the shared Dynamo manager can
+fail to start, affecting non-Grove workloads handled by that manager too. After
+startup, failed LIST/WATCH reconnects can leave Grove observations stale while
+cached reads continue to work. This Grove informer dependency does not apply
+when Grove integration is disabled. Read the
+[Grove upgrade and downgrade guide](https://github.com/ai-dynamo/grove/blob/v0.2.0-beta1/docs/user-guide/upgrading-to-v1beta1-v0.2.0-beta1.md)
+before upgrading. Run the downgrade migration back to `v1alpha1` while the
+`v0.2.0-beta1` operator is still running, then roll back to Grove
+`v0.1.0-alpha.16`.
+Earlier CRDs can drop fields written by newer releases.
+
+For bundled Grove, prepare the rollback platform chart with its
+`dynamo.nvidia.com/grove-version` annotation and `grove` dependency pinned to
+`v0.1.0-alpha.16`, then rebuild its dependencies with `helm dependency update`.
+After the migration, deploy that chart with `helm upgrade` to the same release
+and namespace, using saved pre-upgrade values, `global.grove.install=true`, and
+`grove.crdInstaller.enabled=true`. Use client-side apply for this rollback:
+Helm 3 does so by default; with Helm 4, pass `--server-side=false`. Do not roll
+back to an unmodified older platform chart: Dynamo `v1.6.0` bundles `alpha.14`,
+and pre-upgrade `main` bundles `alpha.15`.
+
+**Existing deployments:** Grove converts existing alpha resources without requiring
+manifest changes. An unset clique or scaling-group `scaleConfig.minReplicas` now
+defaults to `minAvailable` rather than `replicas`; existing explicit minima are retained.
+
 ### v1.6.0
 
 #### Runtime Compatibility
@@ -522,7 +563,7 @@ Kubernetes: `>=1.30.0-0`
 | file://components/operator | dynamo-operator | 1.7.0 |
 | https://charts.bitnami.com/bitnami | etcd | 12.0.18 |
 | https://nats-io.github.io/k8s/helm/charts/ | nats | 1.3.2 |
-| oci://ghcr.io/ai-dynamo/grove | grove(grove-charts) | v0.1.0-alpha.15 |
+| oci://ghcr.io/ai-dynamo/grove | grove(grove-charts) | v0.2.0-beta1 |
 | oci://ghcr.io/ai-dynamo/snapshot | snapshot | 0.2.0 |
 | oci://ghcr.io/kai-scheduler/kai-scheduler | kai-scheduler | v0.17.0 |
 
@@ -611,7 +652,7 @@ Kubernetes: `>=1.30.0-0`
 | dynamo-operator.checkpoint.enabled | bool | `false` | Whether to enable checkpoint/restore functionality |
 | grove.tolerations | list | `[]` | Node tolerations for Grove pods |
 | grove.affinity | object | `{}` | Affinity for Grove pods |
-| grove.crdInstaller.enabled | bool | `true` | Upgrade Grove CRDs before the bundled operator starts. |
+| grove.crdInstaller.upgradeCRDs | bool | `true` | Upgrade Grove CRDs before the bundled operator starts. |
 | grove.config.server.healthProbes.enable | bool | `true` | Enable Grove's webhook-aware liveness and readiness probes. The readiness endpoint stays false until certificates and the webhook server are ready. |
 | kai-scheduler.global.tolerations | list | `[]` | Node tolerations for kai-scheduler pods |
 | kai-scheduler.global.affinity | object | `{}` | Affinity for kai-scheduler pods |
@@ -685,6 +726,7 @@ For **production environments**, Kai Scheduler and Grove should be installed sep
 | 1.4.x           | >= v0.13.4    | >= v0.1.0-alpha.12-rc1 |
 | 1.5.x           | >= v0.17.0    | >= v0.1.0-alpha.13 |
 | 1.6.x           | >= v0.17.0    | >= v0.1.0-alpha.15 |
+| 1.7.x           | >= v0.17.0    | >= v0.2.0-beta1 |
 
 Upgrade Grove in lockstep with Dynamo while Grove APIs are not stable. See the
 [v1.4.0 upgrade notes](#v140) for the topology API transition and the

@@ -173,6 +173,47 @@ sidecar guides.
 **Existing deployments:** No existing workload rolls. Components without a `runtime` init
 container keep running the engine and Dynamo together in `main`.
 
+#### Dependency compatibility
+
+##### Grove v0.2.0-beta1
+
+**Change:** The bundled Grove chart and Go API dependency are now `v0.2.0-beta1`.
+Grove stores resources as `grove.io/v1beta1` and continues serving the
+`grove.io/v1alpha1` API used by Dynamo. The external Grove preflight accepts beta
+release tags and requires the pinned version or later.
+
+**Affected:** All Grove-backed deployments upgrading to Dynamo 1.7.0.
+
+**Action:** Upgrade externally managed Grove and matching CRDs first, with
+`crdInstaller.upgradeCRDs=true`. The platform chart enables this setting for
+bundled Grove, replacing `grove.crdInstaller.enabled`. The API server must be able
+to reach Grove's conversion webhook. With Grove integration enabled, Dynamo's
+alpha informers depend on that webhook to read beta-stored resources. If it is
+unavailable during initial cache synchronization, the shared Dynamo manager can
+fail to start, affecting non-Grove workloads handled by that manager too. After
+startup, failed LIST/WATCH reconnects can leave Grove observations stale while
+cached reads continue to work. This Grove informer dependency does not apply
+when Grove integration is disabled. Read the
+[Grove upgrade and downgrade guide](https://github.com/ai-dynamo/grove/blob/v0.2.0-beta1/docs/user-guide/upgrading-to-v1beta1-v0.2.0-beta1.md)
+before upgrading. Run the downgrade migration back to `v1alpha1` while the
+`v0.2.0-beta1` operator is still running, then roll back to Grove
+`v0.1.0-alpha.16`.
+Earlier CRDs can drop fields written by newer releases.
+
+For bundled Grove, prepare the rollback platform chart with its
+`dynamo.nvidia.com/grove-version` annotation and `grove` dependency pinned to
+`v0.1.0-alpha.16`, then rebuild its dependencies with `helm dependency update`.
+After the migration, deploy that chart with `helm upgrade` to the same release
+and namespace, using saved pre-upgrade values, `global.grove.install=true`, and
+`grove.crdInstaller.enabled=true`. Use client-side apply for this rollback:
+Helm 3 does so by default; with Helm 4, pass `--server-side=false`. Do not roll
+back to an unmodified older platform chart: Dynamo `v1.6.0` bundles `alpha.14`,
+and pre-upgrade `main` bundles `alpha.15`.
+
+**Existing deployments:** Grove converts existing alpha resources without requiring
+manifest changes. An unset clique or scaling-group `scaleConfig.minReplicas` now
+defaults to `minAvailable` rather than `replicas`; existing explicit minima are retained.
+
 ### v1.6.0
 
 #### Runtime Compatibility
@@ -585,6 +626,7 @@ For **production environments**, Kai Scheduler and Grove should be installed sep
 | 1.4.x           | >= v0.13.4    | >= v0.1.0-alpha.12-rc1 |
 | 1.5.x           | >= v0.17.0    | >= v0.1.0-alpha.13 |
 | 1.6.x           | >= v0.17.0    | >= v0.1.0-alpha.15 |
+| 1.7.x           | >= v0.17.0    | >= v0.2.0-beta1 |
 
 Upgrade Grove in lockstep with Dynamo while Grove APIs are not stable. See the
 [v1.4.0 upgrade notes](#v140) for the topology API transition and the

@@ -12,14 +12,15 @@
 
 
 def version(tag):
-    # Recognize Grove alpha releases and stable releases; custom/RC tags are unverifiable.
-    match = re.fullmatch(r"v?(\d+)\.(\d+)\.(\d+)(?:-alpha\.(\d+))?", tag)
+    # Recognize Grove alpha, beta, and stable releases; custom/RC tags are unverifiable.
+    match = re.fullmatch(r"v?(\d+)\.(\d+)\.(\d+)(?:-(alpha\.|beta)(\d+))?", tag)
     if not match:
         raise ValueError(
             f"Unverifiable Grove version {tag!r}; use a release-tagged image."
         )
     return tuple(int(n) for n in match.group(1, 2, 3)) + (
-        int(match[4]) if match[4] else float("inf"),
+        {"alpha.": 0, "beta": 1, None: 2}[match[4]],
+        int(match[5] or 0),
     )
 
 

@@ -38,7 +38,7 @@ tests:
       - matchRegex: {path: 'spec.template.spec.containers[0].args[0]', pattern: 'def check\('}
       - matchRegex:
           path: spec.template.spec.containers[0].args[1]
-          pattern: '^v?[0-9]+\.[0-9]+\.[0-9]+(-alpha\.[0-9]+)?$'
+          pattern: '^v?[0-9]+\.[0-9]+\.[0-9]+(-(alpha\.|beta)[0-9]+)?$'
   - it: keeps the compatibility check visible in long hook names
     set:
       global.grove.enabled: true

@@ -23,21 +23,23 @@
 
 
 class GrovePreflightTest(unittest.TestCase):
-    def test_orders_alpha_versions_numerically_before_stable(self):
-        self.assertLess(
-            preflight.version("v1.2.3-alpha.9"), preflight.version("v1.2.3-alpha.10")
-        )
-        self.assertLess(
-            preflight.version("v1.2.3-alpha.10"), preflight.version("v1.2.3")
-        )
+    def test_orders_alpha_beta_and_stable_versions(self):
+        for earlier, later in (
+            ("v1.2.3-alpha.9", "v1.2.3-alpha.10"),
+            ("v1.2.3-alpha.10", "v1.2.3-beta1"),
+            ("v1.2.3-beta9", "v1.2.3-beta10"),
+            ("v1.2.3-beta10", "v1.2.3"),
+        ):
+            with self.subTest(earlier=earlier, later=later):
+                self.assertLess(preflight.version(earlier), preflight.version(later))
 
     def test_uses_the_supplied_minimum(self):
         higher = f"v{self.major + 1}.0.0"
         with self.assertRaisesRegex(ValueError, re.escape(higher) + " or later"):
             preflight.check(self.get, higher)
 
     def setUp(self):
-        self.major, minor, patch, _ = preflight.version(MINIMUM)
+        self.major, minor, patch, *_ = preflight.version(MINIMUM)
         self.stable = f"v{self.major}.{minor}.{patch}"
         self.container = {
             "name": "grove-operator",
@@ -63,6 +65,8 @@ def test_accepts_release_and_rejects_old_or_unverifiable_images(self):
                 self.container["image"] = "registry:5000/grove:" + tag
                 preflight.check(self.get, MINIMUM)
         for image in (
+            "grove:v0.1.0-alpha.15",
+            "grove:v0.2.0-alpha.17",
             "grove:v0.0.0-alpha.0",
             "grove:v0.0.0",
             "grove:latest",

@@ -1,6 +1,9 @@
 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+import copy
+import json
+import re
 import shutil
 import subprocess
 import tempfile
@@ -10,6 +13,54 @@
 ROOT = Path(__file__).resolve().parents[1]
 
 
+@unittest.skipUnless(shutil.which("jq"), "requires jq CLI")
+class GroveWebhookConfigurationTest(unittest.TestCase):
+    def test_alpha_requests_are_covered_by_fail_closed_webhooks(self):
+        """Accept equivalent beta rules without accepting fail-open or exact beta rules."""
+        action = (
+            ROOT.parents[3] / ".github/actions/setup-dynamo-operator/action.yml"
+        ).read_text()
+        query = re.search(
+            r"(?s)if ! jq -e --arg webhook_name.*? '\n(.*?)\n          ' <<<",
+            action,
+        ).group(1)
+        webhook = {
+            "name": "pcs.defaulting.webhooks.grove.io",
+            "failurePolicy": "Fail",
+            "rules": [
+                {
+                    "apiGroups": ["grove.io"],
+                    "apiVersions": ["v1alpha1"],
+                    "resources": ["podcliquesets"],
+                    "operations": ["CREATE"],
+                }
+            ],
+        }
+        for version, policy, failure, accepted in (
+            ("v1alpha1", "Exact", "Fail", True),
+            ("v1beta1", "Equivalent", "Fail", True),
+            ("v1beta1", None, "Fail", True),
+            ("v1beta1", "Exact", "Fail", False),
+            ("v1beta1", "Equivalent", "Ignore", False),
+            ("v1beta2", "Equivalent", "Fail", False),
+        ):
+            with self.subTest(version=version, policy=policy, failure=failure):
+                candidate = copy.deepcopy(webhook)
+                candidate["failurePolicy"] = failure
+                if policy is not None:
+                    candidate["matchPolicy"] = policy
+                candidate["rules"][0]["apiVersions"] = [version]
+                result = subprocess.run(
+                    ["jq", "-e", "--arg", "webhook_name", webhook["name"], query],
+                    input=json.dumps({"webhooks": [candidate]}),
+                    capture_output=True,
+                    text=True,
+                    timeout=5,
+                    check=False,
+                )
+                self.assertEqual(result.returncode, 0 if accepted else 1, result.stderr)
+
+
 @unittest.skipUnless(shutil.which("helm"), "requires Helm CLI")
 class GroveVersionTest(unittest.TestCase):
     def helm(self, *args):

@@ -318,7 +318,7 @@ grove:
   affinity: {}
   crdInstaller:
     # -- Upgrade Grove CRDs before the bundled operator starts.
-    enabled: true
+    upgradeCRDs: true
   config:
     server:
       healthProbes:

@@ -24,8 +24,6 @@ CLUSTERTEST_OPERATOR_IMAGE ?= $(DYNAMO_CLUSTERTEST_OPERATOR_IMAGE)
 CLUSTERTEST_PLATFORM ?= linux/$(GOARCH)
 CLUSTERTEST_LWS_VERSION = $(shell go list -m -f '{{.Version}}' sigs.k8s.io/lws)
 CLUSTERTEST_LWS_CRD_DIR = $(shell go env GOMODCACHE)/sigs.k8s.io/lws@$(CLUSTERTEST_LWS_VERSION)/config/crd/bases
-CLUSTERTEST_GROVE_VERSION = $(shell go list -m -f '{{.Version}}' github.com/ai-dynamo/grove/operator/api)
-CLUSTERTEST_GROVE_CRD_DIR = $(shell go env GOMODCACHE)/github.com/ai-dynamo/grove/operator/api@$(CLUSTERTEST_GROVE_VERSION)/core/v1alpha1/crds
 CLUSTERTEST_VOLCANO_VERSION = $(shell go list -m -f '{{.Version}}' volcano.sh/apis)
 CLUSTERTEST_VOLCANO_PODGROUP_CRD_URL = https://raw.githubusercontent.com/volcano-sh/volcano/$(CLUSTERTEST_VOLCANO_VERSION)/config/crd/volcano/bases/scheduling.volcano.sh_podgroups.yaml
 
@@ -344,8 +342,9 @@ setup-clustertest-crds-in-cluster: kubectl ## Install shared CRDs into the expli
 	$(KUBECTL) --kubeconfig '$(CLUSTERTEST_KUBECONFIG)' --context "$${DYNAMO_CLUSTERTEST_CONTEXT}" apply --server-side -f config/crd/bases
 	go mod download sigs.k8s.io/lws@$(CLUSTERTEST_LWS_VERSION)
 	$(KUBECTL) --kubeconfig '$(CLUSTERTEST_KUBECONFIG)' --context "$${DYNAMO_CLUSTERTEST_CONTEXT}" apply --server-side -f '$(CLUSTERTEST_LWS_CRD_DIR)'
-	go mod download github.com/ai-dynamo/grove/operator/api@$(CLUSTERTEST_GROVE_VERSION)
-	$(KUBECTL) --kubeconfig '$(CLUSTERTEST_KUBECONFIG)' --context "$${DYNAMO_CLUSTERTEST_CONTEXT}" apply --server-side -f '$(CLUSTERTEST_GROVE_CRD_DIR)'
+	# Dependency controllers do not run in clustertests; retain the pinned alpha schema without webhook conversion.
+	go run ./hack/prepare_grove_test_crds | \
+		$(KUBECTL) --kubeconfig '$(CLUSTERTEST_KUBECONFIG)' --context "$${DYNAMO_CLUSTERTEST_CONTEXT}" apply --server-side -f -
 	curl --fail --location --silent --show-error --max-time 60 --retry 3 '$(CLUSTERTEST_VOLCANO_PODGROUP_CRD_URL)' | \
 		$(KUBECTL) --kubeconfig '$(CLUSTERTEST_KUBECONFIG)' --context "$${DYNAMO_CLUSTERTEST_CONTEXT}" apply --server-side -f -
 	$(KUBECTL) --kubeconfig '$(CLUSTERTEST_KUBECONFIG)' --context "$${DYNAMO_CLUSTERTEST_CONTEXT}" wait --for=condition=Established crd --all --timeout=2m

@@ -6,7 +6,7 @@ require (
 	capnproto.org/go/capnp/v3 v3.1.0-alpha.2
 	emperror.dev/errors v0.8.1
 	github.com/Masterminds/semver/v3 v3.4.0
-	github.com/ai-dynamo/grove/operator/api v0.1.0-alpha.15
+	github.com/ai-dynamo/grove/operator/api v0.2.0-beta1
 	github.com/ai-dynamo/modelexpress/modelexpress_client/go v0.0.0-20261002171923-ac6d03a87a55
 	github.com/ai-dynamo/snapshot/api v0.2.0
 	github.com/bsm/gomega v1.27.10

@@ -18,8 +18,8 @@ github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe
 github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0=
 github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
 github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
-github.com/ai-dynamo/grove/operator/api v0.1.0-alpha.15 h1:7OD4K8Ihjdehd1fHzAxV6e2bMWzsFaox2UuAZ32qnDU=
-github.com/ai-dynamo/grove/operator/api v0.1.0-alpha.15/go.mod h1:pTafNPyD9ic9WPyfZ6PeoClWbIkzWmMUTORG1tdhNFk=
+github.com/ai-dynamo/grove/operator/api v0.2.0-beta1 h1:i+/GdpillY8cbv5j86CNiPiZZti/9OdrLdDiQnvWMyg=
+github.com/ai-dynamo/grove/operator/api v0.2.0-beta1/go.mod h1:JZgOne08H1JomHIQGudSrSQMXByAAm0mcpfuVP1YV48=
 github.com/ai-dynamo/modelexpress/modelexpress_client/go v0.0.0-20261002171923-ac6d03a87a55 h1:Kk9yUPloFZuSSry7P79fXrI7rAvfW/ywjaeOGeE2LEo=
 github.com/ai-dynamo/modelexpress/modelexpress_client/go v0.0.0-20261002171923-ac6d03a87a55/go.mod h1:gmv3oYWWMcnFRaNH3Ij7oaXZPo5Xkrd0rdN0ry6TMkQ=
 github.com/ai-dynamo/snapshot/api v0.2.0 h1:+Nt4lhF0iiyjbNmOAd0F2lBZVKQb41wfcg1ZOP/iM1E=