diff --git a/.github/actions/setup-dynamo-operator/action.yml b/.github/actions/setup-dynamo-operator/action.yml index d6a30bc5c116..21c0379766f1 100644 --- a/.github/actions/setup-dynamo-operator/action.yml +++ b/.github/actions/setup-dynamo-operator/action.yml @@ -784,11 +784,15 @@ runs: -o json) if ! jq -e --arg webhook_name "${webhook_name}" ' any(.webhooks[]; + (.matchPolicy // "Equivalent") as $matchPolicy | .name == $webhook_name and .failurePolicy == "Fail" and any(.rules[]; (.apiGroups | index("grove.io")) != null - and (.apiVersions | index("v1alpha1")) != null + and ( + (.apiVersions | index("v1alpha1")) != null + or ($matchPolicy == "Equivalent" and (.apiVersions | index("v1beta1")) != null) + ) and (.resources | index("podcliquesets")) != null and (.operations | index("CREATE")) != null ) diff --git a/deploy/helm/charts/platform/Chart.yaml b/deploy/helm/charts/platform/Chart.yaml index ffa145b99bac..1dbe2132d8b7 100644 --- a/deploy/helm/charts/platform/Chart.yaml +++ b/deploy/helm/charts/platform/Chart.yaml @@ -25,7 +25,7 @@ kubeVersion: ">=1.30.0-0" home: https://nvidia.com annotations: # One scalar drives both the bundled dependency and the external-Grove preflight. - dynamo.nvidia.com/grove-version: &groveVersion v0.1.0-alpha.15 + dynamo.nvidia.com/grove-version: &groveVersion v0.2.0-beta1 dependencies: - name: dynamo-operator version: 1.7.0 diff --git a/deploy/helm/charts/platform/README.md b/deploy/helm/charts/platform/README.md index d7b5c5f2e2a2..a56a7b73d6cd 100644 --- a/deploy/helm/charts/platform/README.md +++ b/deploy/helm/charts/platform/README.md @@ -173,6 +173,47 @@ sidecar guides. **Existing deployments:** No existing workload rolls. Components without a `runtime` init container keep running the engine and Dynamo together in `main`. +#### Dependency compatibility + +##### Grove v0.2.0-beta1 + +**Change:** The bundled Grove chart and Go API dependency are now `v0.2.0-beta1`. +Grove stores resources as `grove.io/v1beta1` and continues serving the +`grove.io/v1alpha1` API used by Dynamo. The external Grove preflight accepts beta +release tags and requires the pinned version or later. + +**Affected:** All Grove-backed deployments upgrading to Dynamo 1.7.0. + +**Action:** Upgrade externally managed Grove and matching CRDs first, with +`crdInstaller.upgradeCRDs=true`. The platform chart enables this setting for +bundled Grove, replacing `grove.crdInstaller.enabled`. The API server must be able +to reach Grove's conversion webhook. With Grove integration enabled, Dynamo's +alpha informers depend on that webhook to read beta-stored resources. If it is +unavailable during initial cache synchronization, the shared Dynamo manager can +fail to start, affecting non-Grove workloads handled by that manager too. After +startup, failed LIST/WATCH reconnects can leave Grove observations stale while +cached reads continue to work. This Grove informer dependency does not apply +when Grove integration is disabled. Read the +[Grove upgrade and downgrade guide](https://github.com/ai-dynamo/grove/blob/v0.2.0-beta1/docs/user-guide/upgrading-to-v1beta1-v0.2.0-beta1.md) +before upgrading. Run the downgrade migration back to `v1alpha1` while the +`v0.2.0-beta1` operator is still running, then roll back to Grove +`v0.1.0-alpha.16`. +Earlier CRDs can drop fields written by newer releases. + +For bundled Grove, prepare the rollback platform chart with its +`dynamo.nvidia.com/grove-version` annotation and `grove` dependency pinned to +`v0.1.0-alpha.16`, then rebuild its dependencies with `helm dependency update`. +After the migration, deploy that chart with `helm upgrade` to the same release +and namespace, using saved pre-upgrade values, `global.grove.install=true`, and +`grove.crdInstaller.enabled=true`. Use client-side apply for this rollback: +Helm 3 does so by default; with Helm 4, pass `--server-side=false`. Do not roll +back to an unmodified older platform chart: Dynamo `v1.6.0` bundles `alpha.14`, +and pre-upgrade `main` bundles `alpha.15`. + +**Existing deployments:** Grove converts existing alpha resources without requiring +manifest changes. An unset clique or scaling-group `scaleConfig.minReplicas` now +defaults to `minAvailable` rather than `replicas`; existing explicit minima are retained. + ### v1.6.0 #### Runtime Compatibility @@ -522,7 +563,7 @@ Kubernetes: `>=1.30.0-0` | file://components/operator | dynamo-operator | 1.7.0 | | https://charts.bitnami.com/bitnami | etcd | 12.0.18 | | https://nats-io.github.io/k8s/helm/charts/ | nats | 1.3.2 | -| oci://ghcr.io/ai-dynamo/grove | grove(grove-charts) | v0.1.0-alpha.15 | +| oci://ghcr.io/ai-dynamo/grove | grove(grove-charts) | v0.2.0-beta1 | | oci://ghcr.io/ai-dynamo/snapshot | snapshot | 0.2.0 | | oci://ghcr.io/kai-scheduler/kai-scheduler | kai-scheduler | v0.17.0 | @@ -611,7 +652,7 @@ Kubernetes: `>=1.30.0-0` | dynamo-operator.checkpoint.enabled | bool | `false` | Whether to enable checkpoint/restore functionality | | grove.tolerations | list | `[]` | Node tolerations for Grove pods | | grove.affinity | object | `{}` | Affinity for Grove pods | -| grove.crdInstaller.enabled | bool | `true` | Upgrade Grove CRDs before the bundled operator starts. | +| grove.crdInstaller.upgradeCRDs | bool | `true` | Upgrade Grove CRDs before the bundled operator starts. | | grove.config.server.healthProbes.enable | bool | `true` | Enable Grove's webhook-aware liveness and readiness probes. The readiness endpoint stays false until certificates and the webhook server are ready. | | kai-scheduler.global.tolerations | list | `[]` | Node tolerations for kai-scheduler pods | | kai-scheduler.global.affinity | object | `{}` | Affinity for kai-scheduler pods | @@ -685,6 +726,7 @@ For **production environments**, Kai Scheduler and Grove should be installed sep | 1.4.x | >= v0.13.4 | >= v0.1.0-alpha.12-rc1 | | 1.5.x | >= v0.17.0 | >= v0.1.0-alpha.13 | | 1.6.x | >= v0.17.0 | >= v0.1.0-alpha.15 | +| 1.7.x | >= v0.17.0 | >= v0.2.0-beta1 | Upgrade Grove in lockstep with Dynamo while Grove APIs are not stable. See the [v1.4.0 upgrade notes](#v140) for the topology API transition and the diff --git a/deploy/helm/charts/platform/README.md.gotmpl b/deploy/helm/charts/platform/README.md.gotmpl index eb1c02ef5a67..93b604881880 100644 --- a/deploy/helm/charts/platform/README.md.gotmpl +++ b/deploy/helm/charts/platform/README.md.gotmpl @@ -173,6 +173,47 @@ sidecar guides. **Existing deployments:** No existing workload rolls. Components without a `runtime` init container keep running the engine and Dynamo together in `main`. +#### Dependency compatibility + +##### Grove v0.2.0-beta1 + +**Change:** The bundled Grove chart and Go API dependency are now `v0.2.0-beta1`. +Grove stores resources as `grove.io/v1beta1` and continues serving the +`grove.io/v1alpha1` API used by Dynamo. The external Grove preflight accepts beta +release tags and requires the pinned version or later. + +**Affected:** All Grove-backed deployments upgrading to Dynamo 1.7.0. + +**Action:** Upgrade externally managed Grove and matching CRDs first, with +`crdInstaller.upgradeCRDs=true`. The platform chart enables this setting for +bundled Grove, replacing `grove.crdInstaller.enabled`. The API server must be able +to reach Grove's conversion webhook. With Grove integration enabled, Dynamo's +alpha informers depend on that webhook to read beta-stored resources. If it is +unavailable during initial cache synchronization, the shared Dynamo manager can +fail to start, affecting non-Grove workloads handled by that manager too. After +startup, failed LIST/WATCH reconnects can leave Grove observations stale while +cached reads continue to work. This Grove informer dependency does not apply +when Grove integration is disabled. Read the +[Grove upgrade and downgrade guide](https://github.com/ai-dynamo/grove/blob/v0.2.0-beta1/docs/user-guide/upgrading-to-v1beta1-v0.2.0-beta1.md) +before upgrading. Run the downgrade migration back to `v1alpha1` while the +`v0.2.0-beta1` operator is still running, then roll back to Grove +`v0.1.0-alpha.16`. +Earlier CRDs can drop fields written by newer releases. + +For bundled Grove, prepare the rollback platform chart with its +`dynamo.nvidia.com/grove-version` annotation and `grove` dependency pinned to +`v0.1.0-alpha.16`, then rebuild its dependencies with `helm dependency update`. +After the migration, deploy that chart with `helm upgrade` to the same release +and namespace, using saved pre-upgrade values, `global.grove.install=true`, and +`grove.crdInstaller.enabled=true`. Use client-side apply for this rollback: +Helm 3 does so by default; with Helm 4, pass `--server-side=false`. Do not roll +back to an unmodified older platform chart: Dynamo `v1.6.0` bundles `alpha.14`, +and pre-upgrade `main` bundles `alpha.15`. + +**Existing deployments:** Grove converts existing alpha resources without requiring +manifest changes. An unset clique or scaling-group `scaleConfig.minReplicas` now +defaults to `minAvailable` rather than `replicas`; existing explicit minima are retained. + ### v1.6.0 #### Runtime Compatibility @@ -585,6 +626,7 @@ For **production environments**, Kai Scheduler and Grove should be installed sep | 1.4.x | >= v0.13.4 | >= v0.1.0-alpha.12-rc1 | | 1.5.x | >= v0.17.0 | >= v0.1.0-alpha.13 | | 1.6.x | >= v0.17.0 | >= v0.1.0-alpha.15 | +| 1.7.x | >= v0.17.0 | >= v0.2.0-beta1 | Upgrade Grove in lockstep with Dynamo while Grove APIs are not stable. See the [v1.4.0 upgrade notes](#v140) for the topology API transition and the diff --git a/deploy/helm/charts/platform/files/grove-preflight/check.py b/deploy/helm/charts/platform/files/grove-preflight/check.py index 28eea3a26158..3cc8005975da 100644 --- a/deploy/helm/charts/platform/files/grove-preflight/check.py +++ b/deploy/helm/charts/platform/files/grove-preflight/check.py @@ -12,14 +12,15 @@ def version(tag): - # Recognize Grove alpha releases and stable releases; custom/RC tags are unverifiable. - match = re.fullmatch(r"v?(\d+)\.(\d+)\.(\d+)(?:-alpha\.(\d+))?", tag) + # Recognize Grove alpha, beta, and stable releases; custom/RC tags are unverifiable. + match = re.fullmatch(r"v?(\d+)\.(\d+)\.(\d+)(?:-(alpha\.|beta)(\d+))?", tag) if not match: raise ValueError( f"Unverifiable Grove version {tag!r}; use a release-tagged image." ) return tuple(int(n) for n in match.group(1, 2, 3)) + ( - int(match[4]) if match[4] else float("inf"), + {"alpha.": 0, "beta": 1, None: 2}[match[4]], + int(match[5] or 0), ) diff --git a/deploy/helm/charts/platform/tests/grove_preflight_test.yaml b/deploy/helm/charts/platform/tests/grove_preflight_test.yaml index 147b7013b753..630852f5d079 100644 --- a/deploy/helm/charts/platform/tests/grove_preflight_test.yaml +++ b/deploy/helm/charts/platform/tests/grove_preflight_test.yaml @@ -38,7 +38,7 @@ tests: - matchRegex: {path: 'spec.template.spec.containers[0].args[0]', pattern: 'def check\('} - matchRegex: path: spec.template.spec.containers[0].args[1] - pattern: '^v?[0-9]+\.[0-9]+\.[0-9]+(-alpha\.[0-9]+)?$' + pattern: '^v?[0-9]+\.[0-9]+\.[0-9]+(-(alpha\.|beta)[0-9]+)?$' - it: keeps the compatibility check visible in long hook names set: global.grove.enabled: true diff --git a/deploy/helm/charts/platform/tests/test_grove_preflight.py b/deploy/helm/charts/platform/tests/test_grove_preflight.py index 1fc5ba8dfb68..43ed7eb910ff 100644 --- a/deploy/helm/charts/platform/tests/test_grove_preflight.py +++ b/deploy/helm/charts/platform/tests/test_grove_preflight.py @@ -23,13 +23,15 @@ class GrovePreflightTest(unittest.TestCase): - def test_orders_alpha_versions_numerically_before_stable(self): - self.assertLess( - preflight.version("v1.2.3-alpha.9"), preflight.version("v1.2.3-alpha.10") - ) - self.assertLess( - preflight.version("v1.2.3-alpha.10"), preflight.version("v1.2.3") - ) + def test_orders_alpha_beta_and_stable_versions(self): + for earlier, later in ( + ("v1.2.3-alpha.9", "v1.2.3-alpha.10"), + ("v1.2.3-alpha.10", "v1.2.3-beta1"), + ("v1.2.3-beta9", "v1.2.3-beta10"), + ("v1.2.3-beta10", "v1.2.3"), + ): + with self.subTest(earlier=earlier, later=later): + self.assertLess(preflight.version(earlier), preflight.version(later)) def test_uses_the_supplied_minimum(self): higher = f"v{self.major + 1}.0.0" @@ -37,7 +39,7 @@ def test_uses_the_supplied_minimum(self): preflight.check(self.get, higher) def setUp(self): - self.major, minor, patch, _ = preflight.version(MINIMUM) + self.major, minor, patch, *_ = preflight.version(MINIMUM) self.stable = f"v{self.major}.{minor}.{patch}" self.container = { "name": "grove-operator", @@ -63,6 +65,8 @@ def test_accepts_release_and_rejects_old_or_unverifiable_images(self): self.container["image"] = "registry:5000/grove:" + tag preflight.check(self.get, MINIMUM) for image in ( + "grove:v0.1.0-alpha.15", + "grove:v0.2.0-alpha.17", "grove:v0.0.0-alpha.0", "grove:v0.0.0", "grove:latest", diff --git a/deploy/helm/charts/platform/tests/test_grove_version.py b/deploy/helm/charts/platform/tests/test_grove_version.py index 4ba6872d3d74..95b3938d3830 100644 --- a/deploy/helm/charts/platform/tests/test_grove_version.py +++ b/deploy/helm/charts/platform/tests/test_grove_version.py @@ -1,6 +1,9 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import copy +import json +import re import shutil import subprocess import tempfile @@ -10,6 +13,54 @@ ROOT = Path(__file__).resolve().parents[1] +@unittest.skipUnless(shutil.which("jq"), "requires jq CLI") +class GroveWebhookConfigurationTest(unittest.TestCase): + def test_alpha_requests_are_covered_by_fail_closed_webhooks(self): + """Accept equivalent beta rules without accepting fail-open or exact beta rules.""" + action = ( + ROOT.parents[3] / ".github/actions/setup-dynamo-operator/action.yml" + ).read_text() + query = re.search( + r"(?s)if ! jq -e --arg webhook_name.*? '\n(.*?)\n ' <<<", + action, + ).group(1) + webhook = { + "name": "pcs.defaulting.webhooks.grove.io", + "failurePolicy": "Fail", + "rules": [ + { + "apiGroups": ["grove.io"], + "apiVersions": ["v1alpha1"], + "resources": ["podcliquesets"], + "operations": ["CREATE"], + } + ], + } + for version, policy, failure, accepted in ( + ("v1alpha1", "Exact", "Fail", True), + ("v1beta1", "Equivalent", "Fail", True), + ("v1beta1", None, "Fail", True), + ("v1beta1", "Exact", "Fail", False), + ("v1beta1", "Equivalent", "Ignore", False), + ("v1beta2", "Equivalent", "Fail", False), + ): + with self.subTest(version=version, policy=policy, failure=failure): + candidate = copy.deepcopy(webhook) + candidate["failurePolicy"] = failure + if policy is not None: + candidate["matchPolicy"] = policy + candidate["rules"][0]["apiVersions"] = [version] + result = subprocess.run( + ["jq", "-e", "--arg", "webhook_name", webhook["name"], query], + input=json.dumps({"webhooks": [candidate]}), + capture_output=True, + text=True, + timeout=5, + check=False, + ) + self.assertEqual(result.returncode, 0 if accepted else 1, result.stderr) + + @unittest.skipUnless(shutil.which("helm"), "requires Helm CLI") class GroveVersionTest(unittest.TestCase): def helm(self, *args): diff --git a/deploy/helm/charts/platform/values.yaml b/deploy/helm/charts/platform/values.yaml index a68de01cca42..3bea2054ac66 100644 --- a/deploy/helm/charts/platform/values.yaml +++ b/deploy/helm/charts/platform/values.yaml @@ -318,7 +318,7 @@ grove: affinity: {} crdInstaller: # -- Upgrade Grove CRDs before the bundled operator starts. - enabled: true + upgradeCRDs: true config: server: healthProbes: diff --git a/deploy/operator/Makefile b/deploy/operator/Makefile index 002a263cb8bd..7af4283fdb08 100644 --- a/deploy/operator/Makefile +++ b/deploy/operator/Makefile @@ -24,8 +24,6 @@ CLUSTERTEST_OPERATOR_IMAGE ?= $(DYNAMO_CLUSTERTEST_OPERATOR_IMAGE) CLUSTERTEST_PLATFORM ?= linux/$(GOARCH) CLUSTERTEST_LWS_VERSION = $(shell go list -m -f '{{.Version}}' sigs.k8s.io/lws) CLUSTERTEST_LWS_CRD_DIR = $(shell go env GOMODCACHE)/sigs.k8s.io/lws@$(CLUSTERTEST_LWS_VERSION)/config/crd/bases -CLUSTERTEST_GROVE_VERSION = $(shell go list -m -f '{{.Version}}' github.com/ai-dynamo/grove/operator/api) -CLUSTERTEST_GROVE_CRD_DIR = $(shell go env GOMODCACHE)/github.com/ai-dynamo/grove/operator/api@$(CLUSTERTEST_GROVE_VERSION)/core/v1alpha1/crds CLUSTERTEST_VOLCANO_VERSION = $(shell go list -m -f '{{.Version}}' volcano.sh/apis) CLUSTERTEST_VOLCANO_PODGROUP_CRD_URL = https://raw.githubusercontent.com/volcano-sh/volcano/$(CLUSTERTEST_VOLCANO_VERSION)/config/crd/volcano/bases/scheduling.volcano.sh_podgroups.yaml @@ -344,8 +342,9 @@ setup-clustertest-crds-in-cluster: kubectl ## Install shared CRDs into the expli $(KUBECTL) --kubeconfig '$(CLUSTERTEST_KUBECONFIG)' --context "$${DYNAMO_CLUSTERTEST_CONTEXT}" apply --server-side -f config/crd/bases go mod download sigs.k8s.io/lws@$(CLUSTERTEST_LWS_VERSION) $(KUBECTL) --kubeconfig '$(CLUSTERTEST_KUBECONFIG)' --context "$${DYNAMO_CLUSTERTEST_CONTEXT}" apply --server-side -f '$(CLUSTERTEST_LWS_CRD_DIR)' - go mod download github.com/ai-dynamo/grove/operator/api@$(CLUSTERTEST_GROVE_VERSION) - $(KUBECTL) --kubeconfig '$(CLUSTERTEST_KUBECONFIG)' --context "$${DYNAMO_CLUSTERTEST_CONTEXT}" apply --server-side -f '$(CLUSTERTEST_GROVE_CRD_DIR)' + # Dependency controllers do not run in clustertests; retain the pinned alpha schema without webhook conversion. + go run ./hack/prepare_grove_test_crds | \ + $(KUBECTL) --kubeconfig '$(CLUSTERTEST_KUBECONFIG)' --context "$${DYNAMO_CLUSTERTEST_CONTEXT}" apply --server-side -f - curl --fail --location --silent --show-error --max-time 60 --retry 3 '$(CLUSTERTEST_VOLCANO_PODGROUP_CRD_URL)' | \ $(KUBECTL) --kubeconfig '$(CLUSTERTEST_KUBECONFIG)' --context "$${DYNAMO_CLUSTERTEST_CONTEXT}" apply --server-side -f - $(KUBECTL) --kubeconfig '$(CLUSTERTEST_KUBECONFIG)' --context "$${DYNAMO_CLUSTERTEST_CONTEXT}" wait --for=condition=Established crd --all --timeout=2m diff --git a/deploy/operator/go.mod b/deploy/operator/go.mod index 97becd0f1dc2..897f58b14c47 100644 --- a/deploy/operator/go.mod +++ b/deploy/operator/go.mod @@ -6,7 +6,7 @@ require ( capnproto.org/go/capnp/v3 v3.1.0-alpha.2 emperror.dev/errors v0.8.1 github.com/Masterminds/semver/v3 v3.4.0 - github.com/ai-dynamo/grove/operator/api v0.1.0-alpha.15 + github.com/ai-dynamo/grove/operator/api v0.2.0-beta1 github.com/ai-dynamo/modelexpress/modelexpress_client/go v0.0.0-20261002171923-ac6d03a87a55 github.com/ai-dynamo/snapshot/api v0.2.0 github.com/bsm/gomega v1.27.10 diff --git a/deploy/operator/go.sum b/deploy/operator/go.sum index 630464257781..12a498fddfff 100644 --- a/deploy/operator/go.sum +++ b/deploy/operator/go.sum @@ -18,8 +18,8 @@ github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= -github.com/ai-dynamo/grove/operator/api v0.1.0-alpha.15 h1:7OD4K8Ihjdehd1fHzAxV6e2bMWzsFaox2UuAZ32qnDU= -github.com/ai-dynamo/grove/operator/api v0.1.0-alpha.15/go.mod h1:pTafNPyD9ic9WPyfZ6PeoClWbIkzWmMUTORG1tdhNFk= +github.com/ai-dynamo/grove/operator/api v0.2.0-beta1 h1:i+/GdpillY8cbv5j86CNiPiZZti/9OdrLdDiQnvWMyg= +github.com/ai-dynamo/grove/operator/api v0.2.0-beta1/go.mod h1:JZgOne08H1JomHIQGudSrSQMXByAAm0mcpfuVP1YV48= github.com/ai-dynamo/modelexpress/modelexpress_client/go v0.0.0-20261002171923-ac6d03a87a55 h1:Kk9yUPloFZuSSry7P79fXrI7rAvfW/ywjaeOGeE2LEo= github.com/ai-dynamo/modelexpress/modelexpress_client/go v0.0.0-20261002171923-ac6d03a87a55/go.mod h1:gmv3oYWWMcnFRaNH3Ij7oaXZPo5Xkrd0rdN0ry6TMkQ= github.com/ai-dynamo/snapshot/api v0.2.0 h1:+Nt4lhF0iiyjbNmOAd0F2lBZVKQb41wfcg1ZOP/iM1E= diff --git a/deploy/operator/hack/prepare_grove_test_crds/main.go b/deploy/operator/hack/prepare_grove_test_crds/main.go new file mode 100644 index 000000000000..522928d78ff5 --- /dev/null +++ b/deploy/operator/hack/prepare_grove_test_crds/main.go @@ -0,0 +1,66 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Emit pinned Grove schemas for tests that do not run the Grove controller. +package main + +import ( + "fmt" + "os" + + grovecrds "github.com/ai-dynamo/grove/operator/api/core/crds" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "sigs.k8s.io/yaml" +) + +func main() { + // Fail before emitting anything if the pinned API no longer serves Dynamo's version. + crds, err := testCRDs() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + // Stream manifests directly into the explicitly selected cluster's installer. + for _, crd := range crds { + data, err := yaml.Marshal(crd) + if err == nil { + _, err = fmt.Fprintf(os.Stdout, "---\n%s", data) + } + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + } +} + +func testCRDs() ([]*apiextensionsv1.CustomResourceDefinition, error) { + // Preserve each pinned alpha schema while removing the dependency on Grove's converter. + result := make([]*apiextensionsv1.CustomResourceDefinition, 0, 5) + for _, manifest := range []string{ + grovecrds.PodCliqueSetCRD(), + grovecrds.PodCliqueCRD(), + grovecrds.PodCliqueScalingGroupCRD(), + grovecrds.ClusterTopologyCRD(), + grovecrds.PodGangMapCRD(), + } { + crd := &apiextensionsv1.CustomResourceDefinition{} + if err := yaml.Unmarshal([]byte(manifest), crd); err != nil { + return nil, err + } + versions := make([]apiextensionsv1.CustomResourceDefinitionVersion, 0, 1) + for _, version := range crd.Spec.Versions { + if version.Name == "v1alpha1" && version.Served { + version.Storage = true + versions = append(versions, version) + } + } + if len(versions) != 1 { + return nil, fmt.Errorf("%s: expected one served v1alpha1 schema", crd.Name) + } + crd.Spec.Versions = versions + crd.Spec.Conversion = &apiextensionsv1.CustomResourceConversion{Strategy: apiextensionsv1.NoneConverter} + result = append(result, crd) + } + return result, nil +} diff --git a/deploy/operator/hack/prepare_grove_test_crds/main_test.go b/deploy/operator/hack/prepare_grove_test_crds/main_test.go new file mode 100644 index 000000000000..bb4653e689cb --- /dev/null +++ b/deploy/operator/hack/prepare_grove_test_crds/main_test.go @@ -0,0 +1,92 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "testing" + + grovecrds "github.com/ai-dynamo/grove/operator/api/core/crds" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/yaml" +) + +func TestCRDsPreservePinnedAlphaSchema(t *testing.T) { + t.Log("Generate all five Grove test CRDs from the pinned module") + crds, err := testCRDs() + require.NoError(t, err) + require.Len(t, crds, 5) + for _, crd := range crds { + require.Len(t, crd.Spec.Versions, 1, crd.Name) + require.Equal(t, "v1alpha1", crd.Spec.Versions[0].Name, crd.Name) + require.True(t, crd.Spec.Versions[0].Served, crd.Name) + require.True(t, crd.Spec.Versions[0].Storage, crd.Name) + require.Equal(t, apiextensionsv1.NoneConverter, crd.Spec.Conversion.Strategy, crd.Name) + require.Nil(t, crd.Spec.Conversion.Webhook, crd.Name) + } + + t.Log("Check that the PCS schema, defaults, CEL rules, and subresources are unchanged") + var original apiextensionsv1.CustomResourceDefinition + require.NoError(t, yaml.Unmarshal([]byte(grovecrds.PodCliqueSetCRD()), &original)) + for _, version := range original.Spec.Versions { + if version.Name == "v1alpha1" { + version.Storage = true + require.Equal(t, version, crds[0].Spec.Versions[0]) + } + } +} + +func TestPodCliqueSetRequestsWithoutGroveConversionService(t *testing.T) { + if os.Getenv("KUBEBUILDER_ASSETS") == "" { + t.Skip("requires local envtest assets") + } + + t.Log("Install the prepared Grove CRDs into an API server with no Grove operator") + crds, err := testCRDs() + require.NoError(t, err) + env := &envtest.Environment{CRDs: crds} + config, err := env.Start() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, env.Stop()) }) + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + require.NoError(t, grovev1alpha1.AddToScheme(scheme)) + c, err := client.New(config, client.Options{Scheme: scheme}) + require.NoError(t, err) + require.NoError(t, c.Create(t.Context(), &corev1.Namespace{Name: "grove-test"})) + + t.Log("Create the alpha PodCliqueSet used by the cluster-test workload provider") + pcs := &grovev1alpha1.PodCliqueSet{ + Name: "workers", Namespace: "grove-test", + Spec: grovev1alpha1.PodCliqueSetSpec{ + Replicas: 1, + Template: grovev1alpha1.PodCliqueSetTemplateSpec{ + Cliques: []*grovev1alpha1.PodCliqueTemplateSpec{{ + Name: "worker", + Spec: grovev1alpha1.PodCliqueSpec{ + RoleName: "worker", Replicas: 1, MinAvailable: new(int32(1)), + PodSpec: corev1.PodSpec{Containers: []corev1.Container{{Name: "worker", Image: "registry.k8s.io/pause:3.10"}}}, + }, + }}, + }, + }, + } + require.NoError(t, c.Create(t.Context(), pcs)) + + t.Log("Read, list, update, and delete without a conversion webhook service") + observed := &grovev1alpha1.PodCliqueSet{} + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(pcs), observed)) + var list grovev1alpha1.PodCliqueSetList + require.NoError(t, c.List(t.Context(), &list, client.InNamespace(pcs.Namespace))) + require.Len(t, list.Items, 1) + observed.Spec.Replicas = 2 + require.NoError(t, c.Update(t.Context(), observed)) + require.NoError(t, c.Delete(t.Context(), observed)) +} diff --git a/deploy/operator/internal/controller/dgd_grove_watch_setup.go b/deploy/operator/internal/controller/dgd_grove_watch_setup.go index 0762d21b5ff2..8197e49dc50e 100644 --- a/deploy/operator/internal/controller/dgd_grove_watch_setup.go +++ b/deploy/operator/internal/controller/dgd_grove_watch_setup.go @@ -23,6 +23,7 @@ import ( nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + grovev1beta1 "github.com/ai-dynamo/grove/operator/api/core/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" @@ -145,7 +146,10 @@ func (s *groveWatchSetup) resolveOwningPodCliqueSet( podClique *grovev1alpha1.PodClique, ) (*grovev1alpha1.PodCliqueSet, bool) { controllerRef := metav1.GetControllerOf(podClique) - if controllerRef == nil || controllerRef.APIVersion != grovev1alpha1.SchemeGroupVersion.String() { + // Conversion to our alpha client types preserves beta controller owner references. + if controllerRef == nil || + (controllerRef.APIVersion != grovev1alpha1.SchemeGroupVersion.String() && + controllerRef.APIVersion != grovev1beta1.SchemeGroupVersion.String()) { return nil, false } @@ -171,8 +175,10 @@ func (s *groveWatchSetup) resolvePodCliqueSetOwner( pcsg *grovev1alpha1.PodCliqueScalingGroup, ) (*grovev1alpha1.PodCliqueSet, bool) { controllerRef := metav1.GetControllerOf(pcsg) + // Both served Grove versions identify the same owner hierarchy. if controllerRef == nil || - controllerRef.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || + (controllerRef.APIVersion != grovev1alpha1.SchemeGroupVersion.String() && + controllerRef.APIVersion != grovev1beta1.SchemeGroupVersion.String()) || controllerRef.Kind != "PodCliqueSet" { return nil, false } diff --git a/deploy/operator/internal/controller/dgd_grove_watch_setup_test.go b/deploy/operator/internal/controller/dgd_grove_watch_setup_test.go index 156c4bb1a816..1b373e8e9bfd 100644 --- a/deploy/operator/internal/controller/dgd_grove_watch_setup_test.go +++ b/deploy/operator/internal/controller/dgd_grove_watch_setup_test.go @@ -4,11 +4,13 @@ package controller import ( + "cmp" "testing" nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + grovev1beta1 "github.com/ai-dynamo/grove/operator/api/core/v1beta1" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -32,7 +34,7 @@ func TestGroveWatchMapsOnlyDGDControllerOwnedPodCliques(t *testing.T) { } } - newObjects := func() (*grovev1alpha1.PodClique, *grovev1alpha1.PodCliqueScalingGroup, *grovev1alpha1.PodCliqueSet) { + newObjects := func(apiVersion string) (*grovev1alpha1.PodClique, *grovev1alpha1.PodCliqueScalingGroup, *grovev1alpha1.PodCliqueSet) { pcs := &grovev1alpha1.PodCliqueSet{ Name: "engine-pcs", Namespace: "workloads", @@ -45,40 +47,64 @@ func TestGroveWatchMapsOnlyDGDControllerOwnedPodCliques(t *testing.T) { Namespace: "workloads", UID: "group-uid", OwnerReferences: []metav1.OwnerReference{controllerRef( - grovev1alpha1.SchemeGroupVersion.String(), "PodCliqueSet", pcs.Name, pcs.UID, + apiVersion, "PodCliqueSet", pcs.Name, pcs.UID, )}} clique := &grovev1alpha1.PodClique{ Name: "role", Namespace: "workloads", OwnerReferences: []metav1.OwnerReference{controllerRef( - grovev1alpha1.SchemeGroupVersion.String(), "PodCliqueScalingGroup", group.Name, group.UID, + apiVersion, "PodCliqueScalingGroup", group.Name, group.UID, )}} return clique, group, pcs } want := []ctrl.Request{{Namespace: "workloads", Name: "graph"}} tests := []struct { - name string - mutate func(*grovev1alpha1.PodClique, *grovev1alpha1.PodCliqueScalingGroup, *grovev1alpha1.PodCliqueSet) []client.Object - want []ctrl.Request + name string + groupOwnerVersion string + mutate func(*grovev1alpha1.PodClique, *grovev1alpha1.PodCliqueScalingGroup, *grovev1alpha1.PodCliqueSet) []client.Object + want []ctrl.Request }{ { name: "direct PCS ownership", mutate: func(clique *grovev1alpha1.PodClique, _ *grovev1alpha1.PodCliqueScalingGroup, pcs *grovev1alpha1.PodCliqueSet) []client.Object { clique.OwnerReferences = []metav1.OwnerReference{controllerRef( - grovev1alpha1.SchemeGroupVersion.String(), "PodCliqueSet", pcs.Name, pcs.UID, + clique.OwnerReferences[0].APIVersion, "PodCliqueSet", pcs.Name, pcs.UID, )} return []client.Object{pcs} }, want: want, }, { - name: "PCSG-mediated ownership", + name: "PCSG-mediated ownership with alpha PCS owner", + groupOwnerVersion: grovev1alpha1.SchemeGroupVersion.String(), mutate: func(_ *grovev1alpha1.PodClique, group *grovev1alpha1.PodCliqueScalingGroup, pcs *grovev1alpha1.PodCliqueSet) []client.Object { return []client.Object{group, pcs} }, want: want, }, + { + name: "PCSG-mediated ownership with beta PCS owner", + groupOwnerVersion: grovev1beta1.SchemeGroupVersion.String(), + mutate: func(_ *grovev1alpha1.PodClique, group *grovev1alpha1.PodCliqueScalingGroup, pcs *grovev1alpha1.PodCliqueSet) []client.Object { + return []client.Object{group, pcs} + }, + want: want, + }, + { + name: "foreign PodClique owner API group", + mutate: func(clique *grovev1alpha1.PodClique, group *grovev1alpha1.PodCliqueScalingGroup, pcs *grovev1alpha1.PodCliqueSet) []client.Object { + clique.OwnerReferences[0].APIVersion = "foreign.example/v1beta1" + return []client.Object{group, pcs} + }, + }, + { + name: "foreign scaling group owner API group", + mutate: func(_ *grovev1alpha1.PodClique, group *grovev1alpha1.PodCliqueScalingGroup, pcs *grovev1alpha1.PodCliqueSet) []client.Object { + group.OwnerReferences[0].APIVersion = "foreign.example/v1beta1" + return []client.Object{group, pcs} + }, + }, { name: "LGD-owned PCS", mutate: func(_ *grovev1alpha1.PodClique, group *grovev1alpha1.PodCliqueScalingGroup, pcs *grovev1alpha1.PodCliqueSet) []client.Object { @@ -133,11 +159,16 @@ func TestGroveWatchMapsOnlyDGDControllerOwnedPodCliques(t *testing.T) { } for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - clique, group, pcs := newObjects() - objects := test.mutate(clique, group, pcs) - reader := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() - require.Equal(t, test.want, newGroveWatchSetup(reader).mapPodCliqueToRequests(t.Context(), clique)) - }) + for _, apiVersion := range []string{grovev1alpha1.SchemeGroupVersion.String(), grovev1beta1.SchemeGroupVersion.String()} { + t.Run(test.name+"/"+apiVersion, func(t *testing.T) { + t.Log("Observe converted Grove children with their original owner-reference versions") + clique, group, pcs := newObjects(apiVersion) + group.OwnerReferences[0].APIVersion = cmp.Or(test.groupOwnerVersion, apiVersion) + objects := test.mutate(clique, group, pcs) + reader := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + t.Log("Map readiness updates only through the exact DGD-owned resource hierarchy") + require.Equal(t, test.want, newGroveWatchSetup(reader).mapPodCliqueToRequests(t.Context(), clique)) + }) + } } } diff --git a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go index ecffe4649f37..d0a7a8e7ba91 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go @@ -34,6 +34,7 @@ import ( grovecommon "github.com/ai-dynamo/grove/operator/api/common" groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + grovev1beta1 "github.com/ai-dynamo/grove/operator/api/core/v1beta1" snapshotv1alpha1 "github.com/ai-dynamo/snapshot/api/v1alpha1" "github.com/onsi/gomega" "github.com/stretchr/testify/assert" @@ -3667,13 +3668,24 @@ func TestGroveWatchSetup_MapPodCliqueScalingGroupToRequests(t *testing.T) { Config: &configv1alpha1.OperatorConfiguration{}, RuntimeConfig: &controller_common.RuntimeConfig{}, } - reqs := newGroveWatchSetup(r.Client). - mapPodCliqueScalingGroupToRequests(context.Background(), tt.obj) - - g.Expect(reqs).To(gomega.HaveLen(tt.wantRequests)) - if tt.wantRequests == 1 { - g.Expect(reqs[0].Name).To(gomega.Equal(tt.wantName)) - g.Expect(reqs[0].Namespace).To(gomega.Equal(tt.wantNs)) + for _, apiVersion := range []string{grovev1alpha1.SchemeGroupVersion.String(), grovev1beta1.SchemeGroupVersion.String()} { + t.Logf("Map scaling-group readiness with %s owner references", apiVersion) + obj := tt.obj.DeepCopyObject().(client.Object) + owners := obj.GetOwnerReferences() + for i := range owners { + if owners[i].APIVersion == grovev1alpha1.SchemeGroupVersion.String() { + owners[i].APIVersion = apiVersion + } + } + obj.SetOwnerReferences(owners) + reqs := newGroveWatchSetup(r.Client). + mapPodCliqueScalingGroupToRequests(context.Background(), obj) + + g.Expect(reqs).To(gomega.HaveLen(tt.wantRequests)) + if tt.wantRequests == 1 { + g.Expect(reqs[0].Name).To(gomega.Equal(tt.wantName)) + g.Expect(reqs[0].Namespace).To(gomega.Equal(tt.wantNs)) + } } }) } diff --git a/deploy/operator/internal/controller/lpx/pipeline_request.go b/deploy/operator/internal/controller/lpx/pipeline_request.go index 2fa237f46d29..4dc1de28a9d1 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request.go @@ -21,6 +21,7 @@ import ( lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovecommon "github.com/ai-dynamo/grove/operator/api/common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + grovev1beta1 "github.com/ai-dynamo/grove/operator/api/core/v1beta1" corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -308,7 +309,10 @@ func (r *graphReconciler) removeReleasedAgentPod(ctx context.Context, pcsgs map[ } cliqueOwner := metav1.GetControllerOf(pod) - if cliqueOwner == nil || cliqueOwner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || cliqueOwner.Kind != "PodClique" { + // Converted children can carry either served Grove version in owner references. + if cliqueOwner == nil || + (cliqueOwner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() && cliqueOwner.APIVersion != grovev1beta1.SchemeGroupVersion.String()) || + cliqueOwner.Kind != "PodClique" { return fmt.Errorf("released Agent Pod %q has no PodClique controller", pod.Name) } clique := &grovev1alpha1.PodClique{} @@ -320,7 +324,9 @@ func (r *graphReconciler) removeReleasedAgentPod(ctx context.Context, pcsgs map[ } groupOwner := metav1.GetControllerOf(clique) - if groupOwner == nil || groupOwner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || groupOwner.Kind != "PodCliqueScalingGroup" { + if groupOwner == nil || + (groupOwner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() && groupOwner.APIVersion != grovev1beta1.SchemeGroupVersion.String()) || + groupOwner.Kind != "PodCliqueScalingGroup" { return fmt.Errorf("released Agent Pod %q's PodClique has no PodCliqueScalingGroup controller", pod.Name) } if group := pcsgs[groupOwner.Name]; group == nil || group.UID != groupOwner.UID { diff --git a/deploy/operator/internal/controller/lpx/pipeline_request_test.go b/deploy/operator/internal/controller/lpx/pipeline_request_test.go index 3ca0321c611d..94a1e5bef140 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request_test.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request_test.go @@ -6,6 +6,7 @@ package lpx import ( + "cmp" "context" "errors" "slices" @@ -17,6 +18,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + grovev1beta1 "github.com/ai-dynamo/grove/operator/api/core/v1beta1" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -30,10 +32,13 @@ import ( func TestRemoveReleasedAgentPods(t *testing.T) { for _, tc := range []struct { name, change string + podOwnerVersion, cliqueOwnerVersion string reason lpxv1alpha1.ReleaseReason remove, wantError, deleteError, missingClique bool }{ {name: "dependency changed", remove: true}, + {name: "beta Pod owner", podOwnerVersion: grovev1beta1.SchemeGroupVersion.String(), remove: true}, + {name: "beta PodClique owner", cliqueOwnerVersion: grovev1beta1.SchemeGroupVersion.String(), remove: true}, {name: "request changed", reason: lpxv1alpha1.ReleaseReasonRequestChanged, remove: true}, {name: "binding failed", reason: lpxv1alpha1.ReleaseReasonBindingFailed, remove: true}, {name: "unbound target", change: "target", remove: true}, @@ -64,6 +69,8 @@ func TestRemoveReleasedAgentPods(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Log("Observe one released Agent Pod and its ownership beneath a trusted scaling group") pcsgs, pclq, pod := newLPXTestAgentHierarchy() + pod.OwnerReferences[0].APIVersion = cmp.Or(tc.podOwnerVersion, grovev1alpha1.SchemeGroupVersion.String()) + pclq.OwnerReferences[0].APIVersion = cmp.Or(tc.cliqueOwnerVersion, grovev1alpha1.SchemeGroupVersion.String()) request := newLPXTestReleaseRequest(pod) execution := &request.Status.Committed.Execution if tc.reason != "" { @@ -232,17 +239,25 @@ func TestGetPipelineRequests(t *testing.T) { for _, request := range []struct { name, namespace string uid types.UID + apiVersion string + kind string + controller bool }{ - {"last", "default", "pcs-uid"}, - {"first", "default", "pcs-uid"}, - {"other-owner", "default", "other-uid"}, - {"other-namespace", "other", "pcs-uid"}, + {"last", "default", "pcs-uid", grovev1alpha1.SchemeGroupVersion.String(), "PodCliqueSet", true}, + {"first", "default", "pcs-uid", grovev1alpha1.SchemeGroupVersion.String(), "PodCliqueSet", true}, + {"other-owner", "default", "other-uid", grovev1alpha1.SchemeGroupVersion.String(), "PodCliqueSet", true}, + {"other-namespace", "other", "pcs-uid", grovev1alpha1.SchemeGroupVersion.String(), "PodCliqueSet", true}, + {"beta", "default", "pcs-uid", grovev1beta1.SchemeGroupVersion.String(), "PodCliqueSet", true}, + {"foreign-group", "default", "pcs-uid", "foreign.example/v1alpha1", "PodCliqueSet", true}, + {"unsupported-version", "default", "pcs-uid", "grove.io/v2", "PodCliqueSet", true}, + {"wrong-kind", "default", "pcs-uid", grovev1alpha1.SchemeGroupVersion.String(), "PodClique", true}, + {"non-controller", "default", "pcs-uid", grovev1alpha1.SchemeGroupVersion.String(), "PodCliqueSet", false}, } { objects = append(objects, &lpxv1alpha1.LPUPipelineRequest{ Name: request.name, Namespace: request.namespace, OwnerReferences: []metav1.OwnerReference{{ - APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Kind: "PodCliqueSet", - Name: "pcs", UID: request.uid, Controller: new(true), + APIVersion: request.apiVersion, Kind: request.kind, + Name: "pcs", UID: request.uid, Controller: new(request.controller), }}}) } r := &graphReconciler{Client: newLPXTestClient(t, objects...)} diff --git a/deploy/operator/internal/controller/lpx/watch.go b/deploy/operator/internal/controller/lpx/watch.go index 6a55e1cf54de..2d8705e1f4f1 100644 --- a/deploy/operator/internal/controller/lpx/watch.go +++ b/deploy/operator/internal/controller/lpx/watch.go @@ -74,6 +74,7 @@ func (r *graphReconciler) setupWithManager(mgr ctrl.Manager) error { func pipelineRequestOwnerUID(obj client.Object) []string { owner := metav1.GetControllerOf(obj) + // Dynamo authors these references with its alpha PCS client. if owner == nil || owner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || owner.Kind != podCliqueSetKind { return nil } diff --git a/deploy/operator/internal/dynamo/grove_schema_test.go b/deploy/operator/internal/dynamo/grove_schema_test.go index 0bed1402faff..29b2f03af0db 100644 --- a/deploy/operator/internal/dynamo/grove_schema_test.go +++ b/deploy/operator/internal/dynamo/grove_schema_test.go @@ -11,8 +11,8 @@ import ( configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" controller_common "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + grovecrds "github.com/ai-dynamo/grove/operator/api/core/crds" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - grovecrds "github.com/ai-dynamo/grove/operator/api/core/v1alpha1/crds" "github.com/stretchr/testify/require" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" diff --git a/deploy/operator/internal/testing/clusterenv/DESIGN.md b/deploy/operator/internal/testing/clusterenv/DESIGN.md index b0a3f9614573..e3fdb6306418 100644 --- a/deploy/operator/internal/testing/clusterenv/DESIGN.md +++ b/deploy/operator/internal/testing/clusterenv/DESIGN.md @@ -75,6 +75,13 @@ PodCliqueSet handlers at Grove's production paths. Grove CRD schema validation and OpenAPI defaulting still run in the API server; the transparent handlers are replaced by Grove's production `Setup` API once it is released. +The shared setup derives Grove test CRDs from the pinned Go API bundle through +`hack/prepare_grove_test_crds`. It retains only `v1alpha1` as the served and storage +version, preserving that version's schema, CEL validation, and OpenAPI defaults. +Conversion is disabled because this harness does not run the Grove operator. +The deployment smoke tests install the published Grove chart and exercise its +real beta conversion and admission webhooks. + The webhook manager is independent of all per-test controller managers. No operator controller runs merely because a test uses admission or conversion. The proxy image is configurable with `WebhookProxyImage`; the controller suite