feat(nvcf-compute-plane): gang and MNNVL topology scheduling, KAI/Grove primitives

deploy/helm/helm-reval/values.yaml

@@ -83,9 +83,9 @@ reval:
     enabled: false
 
   serviceConfig:
-    skipValidateObjects: false
-    skipValidateImages: false
-    skipSanitizeObjectMetadata: false
+    skipValidateObjects: true
+    skipValidateImages: true
+    skipSanitizeObjectMetadata: true
     preserveLabels: []
     preserveAnnotations: []
 

deploy/stacks/nvcf-compute-plane/Makefile

@@ -130,6 +130,9 @@ dist: clean-dist
 	# Copy helmfile.d files
 	@cp -r "$(MAKEFILE_DIR)/helmfile.d" "$(DIST_DIR)/"
 
+	# Copy local charts referenced by helmfile.d releases via relative paths
+	@cp -r "$(MAKEFILE_DIR)/charts" "$(DIST_DIR)/"
+
 	# Copy and transform README (remove internal references)
 	@sed -e 's|nvcf-ncp-internal/ncp-dev|your-org/your-team|g' \
 	     -e 's|0651155215864979/ncp-dev|YOUR_ORG/YOUR_TEAM|g' \

deploy/stacks/nvcf-compute-plane/README.md

@@ -89,13 +89,20 @@ make install CLUSTER_NAME=...   # downloads helmfile v1.1.9 + helm v3.15.4 on fi
 
 ## Optional Add-ons
 
-KAI Scheduler, Grove (topology-aware scheduling), and Dynamo (inference framework
-scheduling) are disabled by default. Grove and Dynamo require KAI Scheduler
-(release and namespace `kai-scheduler`). Enable per-environment in
-`environments/<env>.yaml`:
+The stack can install KAI Scheduler, Grove, and Dynamo. KAI provides resource
+allocation and gang placement, Grove orchestrates related Pod groups, and
+Dynamo describes inference services that Grove manages. GPU clique
+topology-aware scheduling connects those workloads to the cluster's NVLink
+topology.
+
+All add-ons are disabled by default. Grove requires KAI Scheduler, and Dynamo
+requires Grove. Topology-aware scheduling also requires KAI Scheduler. Enable
+the complete stack in `environments/<env>.yaml`:
 
 ```yaml
 addons:
+  topologyAwareScheduling:
+    enabled: true
   kaiScheduler:
     enabled: true
   groveOperator:
@@ -108,8 +115,23 @@ Override KAI component resources under `addons.kaiScheduler.<component>.resource
 (for example `addons.kaiScheduler.scheduler.resources.requests.memory`). Defaults
 are set in `helmfile.d/01-dependencies.yaml.gotmpl`.
 
-For a standalone KAI install outside this stack, follow the
-[KAI Scheduler guide](https://docs.nvidia.com/cloud-functions/current/latest/cluster-management/kai-scheduler.html).
+`addons.topologyAwareScheduling` installs cluster-scoped KAI `Topology`
+resources from `topologyAwareScheduling.topologies` when KAI is enabled.
+When Grove is also enabled, the same toggle sets Grove
+`topologyAwareScheduling.enabled=true` and installs one
+`ClusterTopologyBinding` per KAI `Topology`. The default topology labels are
+`nvidia.com/gpu.clique` and `kubernetes.io/hostname`, in that order.
+
+Enabling `addons.kaiScheduler.enabled` or `addons.dynamoOperator.enabled` also
+adds the matching NVCA feature gate. Enabling KAI, Grove, or Dynamo permits
+their workload resource types in the NVCA validation policy.
+
+See [Gang Scheduling](../../../docs/user/cluster-management/gang-scheduling.md)
+for atomic workload placement and
+[Topology-Aware Scheduling](../../../docs/user/cluster-management/topology-aware-scheduling.md)
+for GPU clique placement. See
+[KAI Scheduler](../../../docs/user/cluster-management/kai-scheduler.md) for
+queue configuration and standalone installation.
 
 ## Multi-Cluster Example
 

deploy/stacks/nvcf-compute-plane/charts/nvcf-cluster-topology/Chart.yaml

@@ -0,0 +1,23 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v2
+name: nvcf-cluster-topology
+description: >-
+  Cluster-scoped KAI Topology and Grove ClusterTopologyBinding resources for
+  NVCF MNNVL topology-aware scheduling
+type: application
+version: 0.1.0
+appVersion: "0.1.0"

deploy/stacks/nvcf-compute-plane/charts/nvcf-cluster-topology/templates/_helpers.tpl

@@ -0,0 +1,40 @@
+{{/*
+SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "nvcf-cluster-topology.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "nvcf-cluster-topology.labels" -}}
+helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+app.kubernetes.io/name: {{ include "nvcf-cluster-topology.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- with .Values.commonLabels }}
+{{ toYaml . }}
+{{- end }}
+{{- end -}}
+
+{{/*
+Derive a Grove domain name from a KAI nodeLabel: last path element with '.' removed.
+Example: nvidia.com/gpu.clique -> gpuclique; kubernetes.io/hostname -> hostname
+*/}}
+{{- define "nvcf-cluster-topology.groveDomain" -}}
+{{- $nodeLabel := . | required "nvcf-cluster-topology: every level must set nodeLabel" -}}
+{{- $nodeLabel | splitList "/" | last | replace "." "" -}}
+{{- end -}}

deploy/stacks/nvcf-compute-plane/charts/nvcf-cluster-topology/templates/cluster-topology-binding.yaml

@@ -0,0 +1,62 @@
+{{/*
+SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.groveOperator.enabled }}
+{{- $topologies := .Values.topologies | default list }}
+{{- if empty $topologies }}
+{{- fail "nvcf-cluster-topology: topologies must contain at least one entry when groveOperator.enabled is true" }}
+{{- end }}
+{{- $schedulerName := .Values.groveOperator.topologyBindingSchedulerName | default "kai-scheduler" }}
+{{- range $i, $topo := $topologies }}
+{{- $levels := $topo.levels | default list }}
+{{- if empty $levels }}
+{{- fail (printf "nvcf-cluster-topology: topology %q must set at least one level" ($topo.name | default "<unnamed>")) }}
+{{- end }}
+{{- $topoName := $topo.name | required "nvcf-cluster-topology: every topology must set name" }}
+---
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: grove.io/v1alpha1
+kind: ClusterTopologyBinding
+metadata:
+  name: {{ printf "%s-binding" $topoName }}
+  labels:
+    {{- include "nvcf-cluster-topology.labels" $ | nindent 4 }}
+spec:
+  levels:
+  {{- range $levels }}
+  - domain: {{ include "nvcf-cluster-topology.groveDomain" .nodeLabel | quote }}
+    key: {{ .nodeLabel | required "nvcf-cluster-topology: every level must set nodeLabel" | quote }}
+  {{- end }}
+  schedulerTopologyBindings:
+  - schedulerName: {{ $schedulerName | quote }}
+    topologyReference: {{ $topoName | quote }}
+{{- end }}
+{{- end }}

deploy/stacks/nvcf-compute-plane/charts/nvcf-cluster-topology/templates/topology.yaml

@@ -0,0 +1,56 @@
+{{/*
+SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.kaiScheduler.enabled }}
+{{- $topologies := .Values.topologies | default list }}
+{{- if empty $topologies }}
+{{- fail "nvcf-cluster-topology: topologies must contain at least one entry when kaiScheduler.enabled is true" }}
+{{- end }}
+{{- range $i, $topo := $topologies }}
+{{- $levels := $topo.levels | default list }}
+{{- if empty $levels }}
+{{- fail (printf "nvcf-cluster-topology: topology %q must set at least one level" ($topo.name | default "<unnamed>")) }}
+{{- end }}
+---
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kai.scheduler/v1alpha1
+kind: Topology
+metadata:
+  name: {{ $topo.name | required "nvcf-cluster-topology: every topology must set name" }}
+  labels:
+    {{- include "nvcf-cluster-topology.labels" $ | nindent 4 }}
+spec:
+  levels:
+  {{- range $levels }}
+  - nodeLabel: {{ .nodeLabel | required "nvcf-cluster-topology: every level must set nodeLabel" | quote }}
+  {{- end }}
+{{- end }}
+{{- end }}

deploy/stacks/nvcf-compute-plane/charts/nvcf-cluster-topology/values.yaml

@@ -0,0 +1,44 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Cluster-scoped Topology resources to install when kaiScheduler.enabled is
+# true. Workloads reference a Topology by name through the
+# `kai.scheduler/topology` annotation.
+#
+# Each entry needs:
+#   name:   Topology resource name
+#   levels: node labels ordered widest domain first, narrowest last
+#
+# When groveOperator.enabled is true, each topologies entry also yields a
+# ClusterTopologyBinding named <name>-binding. Grove level domain is the last
+# path element of nodeLabel with '.' removed; key is nodeLabel.
+topologies:
+- name: nvcf-mnnvl-topology
+  levels:
+  - nodeLabel: nvidia.com/gpu.clique
+  - nodeLabel: kubernetes.io/hostname
+
+# When true, render kai.scheduler/v1alpha1 Topology CRs from topologies.
+kaiScheduler:
+  enabled: false
+
+# When true, render a grove.io/v1alpha1 ClusterTopologyBinding per topologies
+# entry, derived from that entry's levels.
+# topologyBindingSchedulerName sets schedulerTopologyBindings[].schedulerName.
+groveOperator:
+  enabled: false
+  topologyBindingSchedulerName: kai-scheduler
+
+commonLabels: {}

deploy/stacks/nvcf-compute-plane/environments/base.yaml

@@ -72,18 +72,39 @@ observability:
 # =============================================================================
 addons:
 
+  # MNNVL topology-aware scheduling (disabled by default). Enable on
+  # NVLink-optimized clusters so multi-node functions can request clique-level
+  # gang placement. When true and kaiScheduler.enabled, installs KAI Topology
+  # CRs from topologies below. When true and groveOperator.enabled, also sets
+  # Grove topologyAwareScheduling.enabled and installs a ClusterTopologyBinding
+  # that references the KAI Topology. Requires addons.kaiScheduler.enabled.
+  # Function owners opt in per workload with the `kai.scheduler/topology`
+  # annotation naming one of the entries below.
+  topologyAwareScheduling:
+    enabled: false
+    topologies:
+      - name: nvcf-mnnvl-topology
+        # Ordered widest domain first, narrowest last.
+        levels:
+          - nodeLabel: nvidia.com/gpu.clique
+          - nodeLabel: kubernetes.io/hostname
+
   # KAI Scheduler (disabled by default). Required when enabling Grove or Dynamo.
+  # When enabled, NVCA also gets the KAIScheduler feature gate.
   # Per-component resource overrides: addons.kaiScheduler.<component>.resources
   # (requests/limits). Defaults live in helmfile.d/01-dependencies.yaml.gotmpl.
   kaiScheduler:
     enabled: false
 
-  # Grove topology-aware scheduling operator (disabled by default)
+  # Grove topology-aware scheduling operator (disabled by default).
+  # topologyAwareScheduling.enabled is driven by addons.topologyAwareScheduling
+  # when that top-level toggle is on. Override network settings explicitly
+  # (for example network.autoMNNVLEnabled); they are not set by the topology toggle.
   groveOperator:
     enabled: false
-    topologyAwareScheduling: {}
     network: {}
 
-  # Dynamo inference framework scheduling operator (disabled by default)
+  # Dynamo inference framework scheduling operator (disabled by default).
+  # When enabled, NVCA also gets the DynamoOperatorSupport feature gate.
   dynamoOperator:
     enabled: false